From 64d558676fe22651c8e47d9a7382b5c1da8e4fba Mon Sep 17 00:00:00 2001 From: SBrandeis <33657802+SBrandeis@users.noreply.github.com> Date: Wed, 22 Jul 2026 16:14:19 +0200 Subject: [PATCH 01/19] decode bench + oracle + rewrite encode oracle --- .github/scripts/render_pipeline_bench.py | 281 +++++++++++++++--- .github/workflows/pipeline-bench.yml | 65 +++- tokenizers/tk-encode/README.md | 36 +++ .../tk-encode/benches/pipeline_benchmark.rs | 2 +- .../tk-encode/examples/fixture_bench.rs | 258 ++++++++++++++-- .../tk-encode/src/tokenizer/pipeline.rs | 11 + tokenizers/tk-encode/tests/common/mod.rs | 55 ++++ .../tk-encode/tests/pipeline_decode_oracle.rs | 104 +++++++ tokenizers/tk-encode/tests/pipeline_oracle.rs | 132 ++++---- 9 files changed, 805 insertions(+), 139 deletions(-) create mode 100644 tokenizers/tk-encode/tests/common/mod.rs create mode 100644 tokenizers/tk-encode/tests/pipeline_decode_oracle.rs diff --git a/.github/scripts/render_pipeline_bench.py b/.github/scripts/render_pipeline_bench.py index 999b68f44..25afd3ac9 100644 --- a/.github/scripts/render_pipeline_bench.py +++ b/.github/scripts/render_pipeline_bench.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 """Render the fixture_bench JSON as charts + a markdown PR report. -Two series: `baseline` (the latest released `tokenizers` crate — the bar to beat, -drawn gray; the in-tree `Tokenizer` isn't benched, it's only the id oracle behind -`ids_match`) and `pipeline` (the experimental PipelineTokenizer, blue). Leads with +Two series: `baseline` (the latest released `tokenizers` crate — the bar to beat +AND the correctness reference, drawn gray; the in-tree `Tokenizer`, being removed, +only builds the pipeline) and `pipeline` (the experimental PipelineTokenizer, +blue). `ids_match`/`text_match` compare the pipeline against the release. Leads with three always-visible charts — per-model geomean ×speedup, memory footprint, binary size — then one collapsed
per model (per-fixture speedup, thread scaling, stage mix, numbers table). Models the pipeline can't build yet render as "not @@ -107,6 +108,23 @@ def model_speedups(model): return [v for v in (speedup(r) for r in model["results"]) if v] +def decode_speedup(row): + m = row.get("decode_mbps") or {} + b, p = m.get("baseline"), m.get("pipeline") + return p / b if b and p else None + + +def decode_model_speedups(model): + return [v for v in (decode_speedup(r) for r in model["results"]) if v] + + +def has_decode_baseline(model): + """True once any fixture carries a released-crate decode number — the decode + section renders (baseline bar + pipeline 'pending') even before the pipeline + can decode.""" + return any((r.get("decode_mbps") or {}).get("baseline") for r in model["results"]) + + def base_speedup(row, base_lookup, model_name): """This PR's pipeline throughput ÷ the base branch's, for the same (model, fixture) — the "did this PR help vs base" ratio. `None` when the base branch @@ -287,9 +305,13 @@ def overview_svg(models, subtitle_base, meta, lo, hi, baseline_label, return svg_doc(ink, height, title, subtitle, axis + "".join(body) + legend, meta) -def memory_svg(models, meta, baseline_label): +def memory_svg(models, meta, baseline_label, pass_key="encode_bytes", + pass_label="encode", title="Memory footprint"): """Per model: resident-set delta of each implementation — load footprint plus - the encode-pass delta as stacked segments, peak RSS as a tick.""" + the `pass_key`-pass delta as stacked segments, peak RSS as a tick. `pass_key` + selects the encode-pass or the decode-pass delta (same chart, both directions). + A model with no `pass_key` data for either impl is dropped (e.g. decode while + the pipeline stub means only the baseline bar is drawn).""" ink, sink = INK, SERIES_INK models = [m for m in models if isinstance(m.get("memory"), dict)] @@ -297,19 +319,25 @@ def mem(m, impl): d = m["memory"].get(impl) if not isinstance(d, dict): return None - return {k: max(0, d[k]) / 1e6 if d.get(k) is not None else None - for k in ("load_bytes", "encode_bytes", "peak_bytes")} + pick = {"load_bytes": "load_bytes", "pass": pass_key, "peak_bytes": "peak_bytes"} + return {k: max(0, d[src]) / 1e6 if d.get(src) is not None else None + for k, src in pick.items()} + + models = [m for m in models + if any((mem(m, impl) or {}).get("pass") is not None + or (mem(m, impl) or {}).get("load_bytes") is not None + for impl in ("baseline", "pipeline"))] vals = [] for m in models: for impl in ("baseline", "pipeline"): d = mem(m, impl) - if d: - vals.append((d["load_bytes"] or 0) + (d["encode_bytes"] or 0)) + if d and d["load_bytes"] is not None: + vals.append((d["load_bytes"] or 0) + (d["pass"] or 0)) if d["peak_bytes"]: vals.append(d["peak_bytes"]) if not vals: - return svg_doc(ink, 120, "Memory footprint", "no data", "", meta) + return svg_doc(ink, 120, title, "no data", "", meta) max_mb = max(vals) * 1.05 def x(v): @@ -320,7 +348,7 @@ def x(v): body = [f'MB: {escape(baseline_label)} → Pipeline', f'' - f'smaller is better · solid: after load · translucent: encode-pass delta'] + f'smaller is better · solid: after load · translucent: {escape(pass_label)}-pass delta'] y = top for m in models: cy = y + row_h / 2 @@ -330,11 +358,11 @@ def x(v): by = y + 8 for impl in ("baseline", "pipeline"): d = mem(m, impl) - if not d: + if not d or d["load_bytes"] is None: totals.append(None) by += bar_h + 3 continue - load, enc = d["load_bytes"] or 0, d["encode_bytes"] or 0 + load, enc = d["load_bytes"] or 0, d["pass"] or 0 totals.append(load + enc) body.append(f'') @@ -360,9 +388,8 @@ def x(v): ("tick", ink["primary"], "peak RSS (VmHWM)"), ]) height = y + 34 - subtitle = "resident-set delta per implementation, one process each · load + encode pass" - return svg_doc(ink, height, "Memory footprint", - subtitle, grid + "".join(body) + legend, meta) + subtitle = f"resident-set delta per implementation, one process each · load + {pass_label} pass" + return svg_doc(ink, height, title, subtitle, grid + "".join(body) + legend, meta) def chart_svg(model, subtitle_base, meta, lo, hi, baseline_label): @@ -378,7 +405,6 @@ def chart_svg(model, subtitle_base, meta, lo, hi, baseline_label): body = [f'MB/s: {escape(baseline_label)} → Pipeline'] y = top - baseline_id_note = False for key, title in GROUPS: # stable order (alphabetical) so a fixture keeps its row across runs and # lines up with the stage chart — not sorted by the (run-varying) speedup. @@ -390,12 +416,8 @@ def chart_svg(model, subtitle_base, meta, lo, hi, baseline_label): f'font-weight="600" letter-spacing="1.2" text-anchor="end" dx="-10">{title.upper()}') y += 22 for r in group_rows: - label = r["fixture"] - if r.get("ids_match_baseline") is False: - label += " †" - baseline_id_note = True body.append(f'{escape(label)}') + f'font-size="12.5" text-anchor="end">{escape(r["fixture"])}') v = speedup(r) by = y + (ROW_H - BAR_H) / 2 if v: @@ -434,12 +456,148 @@ def chart_svg(model, subtitle_base, meta, lo, hi, baseline_label): parts.append(f"geomean ×{geomean(vals):.2f} vs {baseline_label}") else: parts.append(f"{baseline_label} can’t load this model — no comparison") - if baseline_id_note: - parts.append(f"† ids differ from {baseline_label}") return svg_doc(ink, height, f'{model["model"]} — PipelineTokenizer encode throughput', " · ".join(parts), axis + "".join(body) + legend, meta, subtitle_base) +def decode_overview_svg(models, subtitle_base, meta, lo, hi, baseline_label): + """Headline decode chart, twin of `overview_svg`: per-model geomean ×speedup of + pipeline decode vs the released crate (×1.0), min–max whisker across fixtures. + Same row set as the encode overview — models whose decode isn't implemented yet + show a muted 'pending' row, ones the pipeline can't build show 'not supported'.""" + ink, sink = INK, SERIES_INK + title = "PipelineTokenizer vs latest release — decode throughput" + x = log_x(OV_GUTTER, OV_PLOT, lo, hi) + ticks = thin_ticks([t for t in TICKS if lo <= t <= hi], x, min_px=34, keep=1.0) + + top, row_h = 74, 40 + col_x = CHART_W - 16 + body = [f'fixtures · text'] + y = top + for m in models: + cy = y + row_h / 2 + body.append(f'{escape(m["model"])}') + desc = m.get("desc") or m["shape"] + body.append(f'{escape(desc)}') + vals = decode_model_speedups(m) + if vals: + g, mn, mx = geomean(vals), min(vals), max(vals) + body.append(hbar(x(1.0), x(g), cy - 7, 14, sink["pipeline"])) + body.append(f'') + for v in (mn, mx): + body.append(f'') + anchor, lx = (("start", max(x(mx), x(1.0)) + 8) if g >= 1 + else ("end", min(x(mn), x(1.0)) - 8)) + if anchor == "end" and lx - 40 < OV_GUTTER + 4: + anchor, lx = "start", max(x(mx), x(1.0)) + 8 + body.append(f'×{g:.2f}') + bad = sum(1 for r in m["results"] if r.get("text_match") is False) + right, fill = ((f"⚠ {bad} differ", ink["critical"]) if bad + else (f'{len(m["results"])} · text ok', ink["secondary"])) + body.append(f'{right}') + elif not m["results"]: + pretok = m["shape"].split("·")[-1].strip() + why = (m.get("reason") or f"no {pretok} pre-tokenizer") + body.append(f'not supported — {escape(why)}') + body.append(f'—') + elif m.get("decode_reason"): + body.append(f'decode pending — not implemented yet') + body.append(f'—') + else: + msg = f"{baseline_label} can’t decode this model — no comparison" + body.append(f'{escape(msg)}') + y += row_h + + axis = speedup_axis(ink, x, ticks, top, y + 4) + y += 30 + legend = legend_row(ink, sink, y, [ + ("swatch", "pipeline", "PipelineTokenizer decode"), + ("tick", ink["baseline"], f"×1.0 = {baseline_label}"), + ]) + height = y + 34 + subtitle = (f"geomean ×speedup per model vs {baseline_label} · " + f"whisker: min–max across fixtures · {subtitle_base}") + return svg_doc(ink, height, title, subtitle, axis + "".join(body) + legend, meta) + + +def decode_chart_svg(model, subtitle_base, meta, lo, hi, baseline_label): + """Per-fixture decode ×speedup vs the release, twin of `chart_svg`. Rows with no + pipeline decode yet show the baseline number and a blank (pending) bar.""" + ink, sink = INK, SERIES_INK + rows = model["results"] + x = log_x(GUTTER, PLOT_W, lo, hi) + ticks = thin_ticks([t for t in TICKS if lo <= t <= hi], x, min_px=34, keep=1.0) + + top = 74 + col_x = GUTTER + PLOT_W + PAD_R + COL_W - 16 + body = [f'MB/s: {escape(baseline_label)} → Pipeline'] + y = top + for key, gtitle in GROUPS: + group_rows = sorted((r for r in rows if r["group"] == key), key=lambda r: r["fixture"]) + if not group_rows: + continue + body.append(f'{gtitle.upper()}') + y += 22 + for r in group_rows: + body.append(f'{escape(r["fixture"])}') + v = decode_speedup(r) + by = y + (ROW_H - BAR_H) / 2 + if v: + body.append(hbar(x(1.0), x(v), by, BAR_H, sink["pipeline"])) + txt = f"×{v:.2f}" + fill = ink["primary"] + if r.get("text_match") is False: + txt += " ⚠ text differs" + fill = ink["critical"] + anchor, lx = ("start", max(x(1.0), x(v)) + 6) if v >= 1 else ("end", min(x(1.0), x(v)) - 6) + if anchor == "end" and lx - len(txt) * 6.7 < GUTTER + 4: + anchor, lx = "start", x(1.0) + 6 + body.append(f'{txt}') + mb = r.get("decode_mbps") or {} + body.append(f'' + f'{chain([mb.get("baseline"), mb.get("pipeline")])}') + y += ROW_H + y += 10 + + axis = speedup_axis(ink, x, ticks, top, y) + y += 26 + legend = legend_row(ink, sink, y, [ + ("swatch", "pipeline", "PipelineTokenizer decode"), + ("tick", ink["baseline"], f"×1.0 = {baseline_label}"), + ]) + height = y + 44 + + parts = [model["shape"]] + vals = decode_model_speedups(model) + if vals: + parts.append(f"geomean ×{geomean(vals):.2f} vs {baseline_label}") + elif model.get("decode_reason"): + parts.append("decode pending — not implemented yet") + else: + parts.append(f"{baseline_label} can’t decode this model — no comparison") + return svg_doc(ink, height, f'{model["model"]} — PipelineTokenizer decode throughput', + " · ".join(parts), axis + "".join(body) + legend, meta, subtitle_base) + + def has_stages(model): return any("stage_ns_per_byte" in r for r in model["results"]) @@ -647,18 +805,20 @@ def x(v): subtitle, grid + "".join(body), meta) -def has_threads(m): - t = m.get("threads") +def has_threads(m, key="threads"): + t = m.get(key) return isinstance(t, dict) and bool(t.get("counts")) -def threads_svg(model, meta, baseline_label): - """Per model: encode throughput (MB/s) at 1/2/4/8/device-max threads — pipeline - vs the release — with a per-row *ideal linear* tick (single-thread × N) on the +def threads_svg(model, meta, baseline_label, threads_key="threads", title="Thread scaling"): + """Per model: throughput (MB/s) at 1/2/4/8/device-max threads — pipeline vs the + release — with a per-row *ideal linear* tick (single-thread × N) on the pipeline bar, so linear vs sub-linear scaling is visible at a glance alongside - the pipeline↔release gap; the right column carries self-scaling % of linear.""" + the pipeline↔release gap; the right column carries self-scaling % of linear. + `threads_key` selects the encode (`threads`) or decode (`decode_threads`) sweep; + a `null` pipeline series (decode while stubbed) draws baseline-only.""" ink, sink = INK, SERIES_INK - t = model["threads"] + t = model[threads_key] counts, pipe, base = t["counts"], t["pipeline_mbps"], t["baseline_mbps"] p1 = pipe[0] if pipe and pipe[0] else None # single-thread anchor for the linear reference ideal = [p1 * n for n in counts] if p1 else [] @@ -715,7 +875,7 @@ def x(v): sc = pipe[-1] / p1 scaling = f" · pipeline {sc:.1f}× on {counts[-1]} threads ({sc / counts[-1] * 100:.0f}% of linear)" subtitle = f"throughput at N threads vs {baseline_label}; tick = perfect linear scaling{scaling}" - return svg_doc(ink, height, "Thread scaling", + return svg_doc(ink, height, title, subtitle, grid + "".join(body) + legend, meta) @@ -766,8 +926,8 @@ def render_markdown(data, subtitle_base, meta, base, run_id, sizes, unsupported = [m for m in models if not m["results"]] mismatches = [f"{m['model']}/{r['fixture']}" for m in benched for r in m["results"] if r["ids_match"] is False] - base_mismatch = sorted({m["model"] for m in benched - for r in m["results"] if r.get("ids_match_baseline") is False}) + text_mismatches = [f"{m['model']}/{r['fixture']}" + for m in benched for r in m["results"] if r.get("text_match") is False] md = ["## PipelineTokenizer benchmark", "", f"**{len(benched)} / {len(models)} models supported** — PipelineTokenizer vs " @@ -782,13 +942,18 @@ def render_markdown(data, subtitle_base, meta, base, run_id, sizes, if sizes: md += [picture(base, run_id, "binsize", "Minimal encode binary size", 860), ""] + md += ["### Decode", "", + picture(base, run_id, "decode-overview", + "Per-model decode throughput vs latest release", 860), ""] + if any(has_decode_baseline(m) for m in benched): + md += [picture(base, run_id, "decode-memory", "Per-model decode memory footprint", 860), ""] + if mismatches: - md += [f"> ⚠️ **Pipeline token ids diverge from this tree's Tokenizer on: " + md += [f"> ⚠️ **Pipeline token ids diverge from `tokenizers` {baseline_label} on: " f"{', '.join(mismatches)}** — speedups there are meaningless until fixed.", ""] - if base_mismatch: - md += [f"> ℹ️ Token ids differ from {baseline_label} on: {', '.join(base_mismatch)} " - f"(† in the per-model charts) — expected when this branch fixes encode bugs, " - f"but worth a look.", ""] + if text_mismatches: + md += [f"> ⚠️ **Pipeline decode diverges from `tokenizers` {baseline_label} on: " + f"{', '.join(text_mismatches)}** — decode speedups there are meaningless until fixed.", ""] for m in benched: slug = slugify(m["model"]) @@ -800,7 +965,14 @@ def render_markdown(data, subtitle_base, meta, base, run_id, sizes, bvals = base_model_speedups(m, base_lookup) if bvals: summary += f" · ×{geomean(bvals):.2f} vs base" + dvals = decode_model_speedups(m) + if dvals: + summary += f" · decode ×{geomean(dvals):.2f}" + elif m.get("decode_reason"): + summary += " · decode pending" flag = " · ⚠ ids differ" if any(r["ids_match"] is False for r in m["results"]) else "" + if any(r.get("text_match") is False for r in m["results"]): + flag += " · ⚠ decode differs" md += [f"
{escape(m['model'])} — {escape(desc)} · " f"{summary}{flag}", ""] md += [picture(base, run_id, slug, f"{m['model']} speedup", 860), ""] @@ -810,6 +982,11 @@ def render_markdown(data, subtitle_base, meta, base, run_id, sizes, if has_threads(m): md += [picture(base, run_id, f"{slug}-threads", f"{m['model']} thread scaling", 860), ""] + md += [picture(base, run_id, f"{slug}-decode", + f"{m['model']} decode speedup", 860), ""] + if has_threads(m, "decode_threads"): + md += [picture(base, run_id, f"{slug}-decode-threads", + f"{m['model']} decode thread scaling", 860), ""] md += [mem_line(m, baseline_label), ""] # Per-stage columns carry each split's share of the pipeline's own encode # time with the ns/byte alongside — `share% (ns/B)` — readable as text @@ -823,12 +1000,7 @@ def render_markdown(data, subtitle_base, meta, base, run_id, sizes, f"|---|---|---:|---:|---:|{base_sep}{stage_sep}:--|"] for r in sorted(m["results"], key=lambda r: (r["group"], r["fixture"])): mb = r["mbps"] - flags = [] - if r["ids_match"] is False: - flags.append("⚠️ ≠ tree") - if r.get("ids_match_baseline") is False: - flags.append(f"≠ {baseline_label}") - ids = " · ".join(flags) if flags else "match" + ids = "⚠️ ≠ release" if r["ids_match"] is False else "match" s = r.get("stage_ns_per_byte") stages = " ".join(f"| {stage_cell(s, k)}" for k, _ in STAGES) base_cell = (f"| {fnum(base_speedup(r, base_lookup, m['model']), '×{:.2f}')} " @@ -917,6 +1089,18 @@ def main(): if sizes: (out / "pipeline_bench_binsize.svg").write_text( binsize_svg(sizes, meta, baseline_label)) + + # Decode: shares the id-correctness/perf story with encode but its own scale. + # The overview always renders (pending rows while the pipeline stub stands); + # decode memory only when a released decode number exists to draw against. + dlo, dhi = scale(benched, decode_model_speedups) + (out / "pipeline_bench_decode-overview.svg").write_text( + decode_overview_svg(models, args.subtitle, meta, dlo, dhi, baseline_label)) + if any(has_decode_baseline(m) for m in benched): + (out / "pipeline_bench_decode-memory.svg").write_text( + memory_svg(models, meta, baseline_label, pass_key="decode_bytes", + pass_label="decode", title="Memory footprint — decode")) + for m in models: slug = slugify(m["model"]) svg = (chart_svg(m, args.subtitle, meta, lo, hi, baseline_label) @@ -928,6 +1112,13 @@ def main(): if has_threads(m): (out / f"pipeline_bench_{slug}-threads.svg").write_text( threads_svg(m, meta, baseline_label)) + if m["results"]: + (out / f"pipeline_bench_{slug}-decode.svg").write_text( + decode_chart_svg(m, args.subtitle, meta, dlo, dhi, baseline_label)) + if has_threads(m, "decode_threads"): + (out / f"pipeline_bench_{slug}-decode-threads.svg").write_text( + threads_svg(m, meta, baseline_label, threads_key="decode_threads", + title="Thread scaling — decode")) (out / "pipeline_bench.md").write_text( render_markdown(data, args.subtitle, meta, args.img_base, args.run_id, sizes, diff --git a/.github/workflows/pipeline-bench.yml b/.github/workflows/pipeline-bench.yml index e0df04145..fa58112bd 100644 --- a/.github/workflows/pipeline-bench.yml +++ b/.github/workflows/pipeline-bench.yml @@ -1,10 +1,11 @@ name: Pipeline Benchmark # Comparative benchmark: the experimental `PipelineTokenizer` vs the latest -# *released* tokenizers crate (the baseline to beat — the in-tree legacy -# `Tokenizer` is being phased out, so it is only the id-correctness oracle, -# never a benched series), for every model in -# tk-encode/examples/bench_models.json across every corpus in data/fixtures/. +# *released* tokenizers crate — both the baseline to beat AND the correctness +# reference (`ids_match`/`text_match`). The in-tree legacy `Tokenizer` is being +# phased out, so it only *builds* the pipeline and is never benched or trusted as +# an oracle. Runs for every model in tk-encode/examples/bench_models.json across +# every corpus in data/fixtures/. # Measures single- and multi-thread throughput (1/2/4/8/device-max), per- # implementation memory footprint (RSS), and stripped minimal-binary size. The # release dep is behind tk-encode's `bench-baseline` feature, so production @@ -131,6 +132,46 @@ jobs: tokenizers/target/release/examples/binsize_pipeline retention-days: 3 + # Pipeline oracle tests: encode + decode parity of the pipeline against the + # released crate. They live behind `bench-baseline` (they link the released + # crate), so a plain `cargo test` skips them — this is where they run. Same + # fixtures + models as the shards; the decode oracle is #[ignore]d until + # PipelineTokenizer::decode lands, so only encode parity runs today. + oracle: + name: pipeline oracle tests + if: github.event_name != 'pull_request' || github.event.label.name == 'run-pipeline-bench' + runs-on: + group: aws-general-8-plus + defaults: + run: + working-directory: tokenizers + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + + - name: Install Rust stable + uses: dtolnay/rust-toolchain@stable + + # onig_sys (a bench-baseline dep) generates bindings via bindgen → needs libclang. + - name: Install libclang (onig_sys → bindgen) + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends libclang-dev + + - name: Install uv + uses: astral-sh/setup-uv@v6 + + - name: Cache fixtures + model tokenizers + uses: actions/cache@v4 + with: + path: tokenizers/data + key: bench-data-${{ hashFiles('tokenizers/Makefile', 'tokenizers/tk-encode/examples/bench_models.json') }} + + - name: Download fixtures and model tokenizers + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + run: make fixtures bench-models HF="uvx --from huggingface_hub hf" + + - name: Run pipeline oracle tests (parity vs the released crate) + run: cargo test -p tk-encode --features bench-baseline --test pipeline_oracle --test pipeline_decode_oracle + # Fan-out: each shard benches the i-th of SHARDS contiguous manifest slices on its # own 8-vCPU runner (isolated + parallel), running the prebuilt binary, and uploads # its partial JSON. No toolchain here — the shards only fetch data + the binary. @@ -405,16 +446,22 @@ jobs: gh api "repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/labels/run-pipeline-bench" \ -X DELETE || true - # Pipeline ids must match this tree's Tokenizer. Baseline id mismatches - # (ids_match_baseline) are report-only: a branch may fix encode bugs. - - name: Fail on id mismatches + # Pipeline encode ids (`ids_match`) AND decode text (`text_match`) must match + # the released crate. Both are null when the release can't load a model (no + # reference), and `text_match` is null while decode is a stub → inert until + # the pipeline can decode. + - name: Fail on id / decode mismatches run: | python3 -c " import json, sys models = json.load(open('pipeline_bench.json'))['models'] - bad = [f\"{m['model']}/{r['fixture']}\" for m in models + ids = [f\"{m['model']}/{r['fixture']}\" for m in models for r in m['results'] if r.get('ids_match') is False] - sys.exit(f'ids diverge on: {bad}' if bad else 0) + text = [f\"{m['model']}/{r['fixture']}\" for m in models + for r in m['results'] if r.get('text_match') is False] + errs = ([f'ids diverge on: {ids}'] if ids else []) + \ + ([f'decode diverges on: {text}'] if text else []) + sys.exit(' · '.join(errs) if errs else 0) " # Establish/refresh the base-branch baseline: on a push to diff --git a/tokenizers/tk-encode/README.md b/tokenizers/tk-encode/README.md index 5f3a9871b..2456a04ca 100644 --- a/tokenizers/tk-encode/README.md +++ b/tokenizers/tk-encode/README.md @@ -66,6 +66,42 @@ Training lives in the companion `tk-train` crate (re-exported by the environment variable. As an example setting `RAYON_RS_NUM_THREADS=4` will allocate a maximum of 4 threads. **_Please note this behavior may evolve in the future_** +## PipelineTokenizer: oracle tests & benchmark + +`PipelineTokenizer` is an experimental, allocation-light re-implementation of the +encode/decode pipeline. Its correctness is judged against the **latest released +`tokenizers` crate** (not the in-tree `Tokenizer`, which is being retired): the +pipeline must produce identical token ids and identical decoded text. + +That comparison lives behind the optional `bench-baseline` feature (it links the +released crate), so a plain `cargo test` skips it. To run it you need the fixture +corpora and model tokenizers, then the feature flag: + +```bash +# from tokenizers/ — fetches data/fixtures/ + model tokenizers (needs HF_TOKEN) +make fixtures bench-models + +# encode parity (pipeline ids == released `encode_fast` ids) +cargo test -p tk-encode --features bench-baseline --test pipeline_oracle + +# decode parity (pipeline decode == released `decode`) — ignored until +# PipelineTokenizer::decode is implemented, so pass --ignored to run it +cargo test -p tk-encode --features bench-baseline --test pipeline_decode_oracle -- --ignored +``` + +Both oracles sample seeded-random windows of every fixture corpus; a model whose +tokenizer file isn't present is skipped. CI runs them in the **Pipeline +Benchmark** workflow (`.github/workflows/pipeline-bench.yml`). + +The same workflow runs the comparative benchmark — throughput, thread scaling, +memory, and binary size vs the release — and renders it to charts. To reproduce +locally: + +```bash +cargo run --release -p tk-encode --features bench-baseline --example fixture_bench > bench.json +python3 ../.github/scripts/render_pipeline_bench.py bench.json # writes SVGs + pipeline_bench.md +``` + ## Features - **progressbar**: The progress bar visualization is enabled by default. It might be disabled if diff --git a/tokenizers/tk-encode/benches/pipeline_benchmark.rs b/tokenizers/tk-encode/benches/pipeline_benchmark.rs index 6c39d8b14..719c8c13a 100644 --- a/tokenizers/tk-encode/benches/pipeline_benchmark.rs +++ b/tokenizers/tk-encode/benches/pipeline_benchmark.rs @@ -3,7 +3,7 @@ //! ~128 B / 1 kB / 10 kB / 100 kB, sweeping from per-input-overhead-dominated //! to fully amortized. //! -//! Correctness (id equivalence with the reference `Tokenizer`) is asserted +//! Correctness (id equivalence with the released `tokenizers` crate) is asserted //! separately in `tests/pipeline_oracle.rs`. #[macro_use] diff --git a/tokenizers/tk-encode/examples/fixture_bench.rs b/tokenizers/tk-encode/examples/fixture_bench.rs index ffdba7c78..7cd7bad3b 100644 --- a/tokenizers/tk-encode/examples/fixture_bench.rs +++ b/tokenizers/tk-encode/examples/fixture_bench.rs @@ -6,7 +6,7 @@ //! because the pipeline's `encode` computes no offsets either; timing the //! baseline's offset-tracking `encode` would flatter the pipeline. //! -//! Three isolated phases per model: +//! Four isolated phases per model: //! //! 1. **Throughput** — single-thread warm MB/s per fixture on ~10 kB inputs //! (the regime where per-input overhead is amortized — see @@ -26,16 +26,25 @@ //! re-spawning this binary as `--memory ` children — one //! implementation per process, so allocator page reuse can't blur the //! attribution. +//! 4. **Decode** — the inverse direction, anchored on the RELEASED crate (never +//! the in-tree legacy `Tokenizer`, which is being removed — oracles must not +//! depend on it). The release's `encode_fast` produces the id stream; pipeline +//! and released `decode` then consume the SAME ids (single-thread throughput + +//! the 1/2/4/8/max sweep + a decode-pass RSS delta), gated by `text_match` +//! (pipeline decode == released decode). While `PipelineTokenizer::decode` is a +//! loud stub the pipeline decode series is `null` (rendered "pending"); the +//! released baseline is measured regardless. No released baseline → no decode +//! oracle → the phase is skipped for that model. //! -//! The in-tree `Tokenizer` is *not* benched: it only builds the pipeline and -//! serves as the id-correctness oracle (`ids_match`, which CI fails on). -//! `ids_match_baseline` — pipeline vs the released crate — is report-only, -//! since a branch may intentionally fix encode behavior. Models the pipeline -//! can't build (or encode) yet are reported with empty `results` (plus the -//! failure `reason`) and their pipeline shape rather than benched — the CI -//! grid renders those as roadmap cards. Each manifest entry carries a `desc`: -//! a one-line label of the workload archetype the model exercises, passed -//! through to the report. +//! Correctness is judged against the released crate, never the in-tree +//! `Tokenizer` (which is being removed and only *builds* the pipeline here): +//! `ids_match` (encode ids) and `text_match` (decode text) both compare against +//! the release and both fail CI. They are `null` when the release can't load the +//! model — no reference, so no gate. Models the pipeline can't build (or encode) +//! yet are reported with empty `results` (plus the failure `reason`) and their +//! pipeline shape rather than benched — the CI grid renders those as roadmap +//! cards. Each manifest entry carries a `desc`: a one-line label of the workload +//! archetype the model exercises, passed through to the report. //! //! Emits one JSON object (`{baseline, models}`) on stdout, consumed by //! `.github/scripts/render_pipeline_bench.py` in CI. @@ -237,19 +246,12 @@ fn bench_throughput( .map(|t| t.id) .collect() }; - // The correctness gate CI fails on: pipeline vs this tree's Tokenizer, both flags - // (add_special_tokens=true exercises the post-process prefix/suffix step). - let ids_match = [false, true].into_iter().all(|add_special_tokens| { - f.chunks.iter().take(3).all(|c| { - oracle - .encode(c.as_str(), add_special_tokens) - .unwrap() - .get_ids() - == pipe_ids(c, add_special_tokens) - }) - }); - // Report-only: pipeline vs the released crate (a branch may fix encode bugs). - let ids_match_baseline = base.as_ref().map(|b| { + // The correctness gate CI fails on: pipeline ids == the released crate's ids, + // for both `add_special_tokens` values (`true` exercises the post-process + // stage). The in-tree `Tokenizer` only *builds* the pipeline here — never the + // reference, since it is on its way out. `None` when the release can't load + // this model (no reference to compare against). + let ids_match = base.as_ref().map(|b| { [false, true].into_iter().all(|add_special_tokens| { f.chunks.iter().take(3).all(|c| { b.encode_fast(c.as_str(), add_special_tokens) @@ -286,7 +288,6 @@ fn bench_throughput( "group": f.group, "mbps": { "baseline": base_mbps, "pipeline": pipe_mbps }, "ids_match": ids_match, - "ids_match_baseline": ids_match_baseline, }) } @@ -660,6 +661,153 @@ fn bench_threads( json!({ "counts": counts, "pipeline_mbps": pipe, "baseline_mbps": base }) } +// ── decode: throughput + scaling ───────────────────────────────────────────── +// Mirror of the encode phases, over the inverse direction, anchored on the +// RELEASED crate — never the in-tree legacy `Tokenizer` (which is being removed; +// oracles must not depend on it). The id stream is produced by the release's +// `encode_fast`, and both decoders — pipeline and released baseline — consume +// those SAME ids, so decode is a clean apples-to-apples judged against the +// release. No released baseline → no decode oracle, so the whole phase is null. +// `pipeline_ok` is the once-probed "can the pipeline decode yet" flag: while +// `PipelineTokenizer::decode` is a loud stub it is false, so the pipeline series +// is `null` (rendered "pending") and only the baseline bar is drawn. + +/// Encode every chunk with the released crate into its id stream (untimed input). +fn ids_of(baseline: &BaselineTokenizer, chunks: &[String]) -> Vec> { + chunks + .iter() + .map(|c| { + baseline + .encode_fast(c.as_str(), false) + .unwrap() + .get_ids() + .to_vec() + }) + .collect() +} + +/// Warm single-thread decode throughput + the `text_match` gate for one fixture. +/// `text_match` = pipeline decode == released decode (the gate CI fails on). All +/// null when there is no released baseline to judge against. +fn bench_decode( + baseline: Option<&BaselineTokenizer>, + pipeline: &PipelineTokenizer, + f: &Fixture, + pipeline_ok: bool, +) -> Value { + let null = json!({ + "decode_mbps": { "baseline": Value::Null, "pipeline": Value::Null }, + "text_match": Value::Null, + }); + let Some(baseline) = baseline else { + return null; + }; + let ids = ids_of(baseline, &f.chunks); + // Throughput basis: bytes of text decode emits, measured once via the release. + let dec_bytes: usize = ids + .iter() + .map(|i| baseline.decode(i, false).unwrap().len()) + .sum(); + if dec_bytes == 0 { + return null; + } + + let one_pass = |dec: &dyn Fn(&[u32]) -> usize| -> f64 { + let start = Instant::now(); + let mut n = 0usize; + for i in &ids { + n += dec(i); + } + black_box(n); + start.elapsed().as_secs_f64() + }; + let mbps = |secs: f64| dec_bytes as f64 / secs / 1e6; + + // Correctness gate (first 3 chunks): pipeline decode == released decode. + let text_match = pipeline_ok.then(|| { + ids.iter() + .take(3) + .all(|i| pipeline.decode(i, false).unwrap() == baseline.decode(i, false).unwrap()) + }); + + // Interleaved warm-up + REPS so thermal drift hits both equally. + one_pass(&|i| baseline.decode(i, false).unwrap().len()); + if pipeline_ok { + one_pass(&|i| pipeline.decode(i, false).unwrap().len()); + } + let (mut base_s, mut pipe_s) = (Vec::new(), Vec::new()); + for _ in 0..REPS { + base_s.push(one_pass(&|i| baseline.decode(i, false).unwrap().len())); + if pipeline_ok { + pipe_s.push(one_pass(&|i| pipeline.decode(i, false).unwrap().len())); + } + } + let base_mbps = mbps(median_secs(base_s)); + let pipe_mbps = (!pipe_s.is_empty()).then(|| mbps(median_secs(pipe_s))); + + eprintln!( + " {} decode: baseline {base_mbps:.1} MB/s, pipeline {}", + f.name, + pipe_mbps.map_or("pending".into(), |v: f64| format!("{v:.1} MB/s")), + ); + + json!({ + "decode_mbps": { "baseline": base_mbps, "pipeline": pipe_mbps }, + "text_match": text_match, + }) +} + +/// Median MB/s of decoding `ids` across `n` threads in a private rayon pool. +fn par_decode_mbps( + decode: impl Fn(&[u32]) -> usize + Sync, + ids: &[Vec], + bytes: usize, + n: usize, +) -> f64 { + let pool = ThreadPoolBuilder::new().num_threads(n).build().unwrap(); + let run = || pool.install(|| ids.par_iter().map(|i| decode(i.as_slice())).sum::()); + black_box(run()); + let mut samples = Vec::with_capacity(REPS); + for _ in 0..REPS { + let t = Instant::now(); + black_box(run()); + samples.push(t.elapsed().as_secs_f64()); + } + bytes as f64 / median_secs(samples) / 1e6 +} + +/// Multi-thread decode throughput sweep — pipeline vs the released crate at +/// 1/2/4/8/max threads over the whole fixture corpus's id stream (release-produced). +fn bench_decode_threads( + baseline: Option<&BaselineTokenizer>, + pipeline: &PipelineTokenizer, + all_chunks: &[String], + pipeline_ok: bool, +) -> Value { + let Some(baseline) = baseline else { + return json!({ "counts": [], "pipeline_mbps": [], "baseline_mbps": [] }); + }; + let ids = ids_of(baseline, all_chunks); + let bytes: usize = ids + .iter() + .map(|i| baseline.decode(i, false).unwrap().len()) + .sum(); + let counts = thread_counts(); + let (mut pipe, mut base) = (Vec::new(), Vec::new()); + for &n in &counts { + let b = par_decode_mbps(|i| baseline.decode(i, false).unwrap().len(), &ids, bytes, n); + let p = pipeline_ok + .then(|| par_decode_mbps(|i| pipeline.decode(i, false).unwrap().len(), &ids, bytes, n)); + eprintln!( + " decode {n} thread(s): pipeline {}, baseline {b:.1} MB/s", + p.map_or("pending".into(), |v: f64| format!("{v:.1} MB/s")), + ); + pipe.push(p); + base.push(b); + } + json!({ "counts": counts, "pipeline_mbps": pipe, "baseline_mbps": base }) +} + /// Resident set size in bytes. Exact on Linux (`/proc`); on macOS a `ps` /// fallback good enough for local iteration — CI runs on Linux. fn rss_now() -> Option { @@ -718,22 +866,32 @@ fn memory_sample() -> Vec { } /// `--memory ` child entry: load one implementation, encode a -/// capped pass over the fixtures, print `{load_bytes, encode_bytes, peak_bytes}`. -/// One implementation per process so the deltas attribute cleanly. +/// capped pass over the fixtures, then decode that pass's ids, printing +/// `{load_bytes, encode_bytes, decode_bytes, peak_bytes}`. One implementation per +/// process so the deltas attribute cleanly. `decode_bytes` is `null` when the +/// implementation can't decode yet (the pipeline's loud stub). fn memory_child(which: &str, model: &Path) { let chunks = memory_sample(); let rss0 = rss_now().unwrap_or(0); let mut n = 0usize; - let (after_load, after_encode) = match which { + let mut ids: Vec> = Vec::new(); + let (after_load, after_encode, decode_bytes) = match which { "baseline" => { let mut tok = BaselineTokenizer::from_file(model).unwrap(); inject_added_tokens_baseline(&mut tok); let after_load = rss_now().unwrap_or(0); for c in &chunks { - n += tok.encode_fast(c.as_str(), true).unwrap().len(); + let enc = tok.encode_fast(c.as_str(), false).unwrap(); + n += enc.len(); + ids.push(enc.get_ids().to_vec()); } - (after_load, rss_now().unwrap_or(0)) + let after_encode = rss_now().unwrap_or(0); + for i in &ids { + n += tok.decode(i, false).map(|s| s.len()).unwrap_or(0); + } + let decode_bytes = rss_now().unwrap_or(0) - after_encode; + (after_load, after_encode, Some(decode_bytes)) } "pipeline" => { let mut tok = Tokenizer::from_file(model).unwrap(); @@ -745,9 +903,25 @@ fn memory_child(which: &str, model: &Path) { drop(tok); let after_load = rss_now().unwrap_or(0); for c in &chunks { - n += pipeline.encode(c, true).unwrap().len(); + let enc = pipeline.encode(c, false).unwrap(); + n += enc.len(); + ids.push(enc.iter().map(|t| t.id).collect()); } - (after_load, rss_now().unwrap_or(0)) + let after_encode = rss_now().unwrap_or(0); + // Decode is a loud stub today → skip the pass and report null, so the + // decode memory bar reads "pending" rather than a bogus 0. + let decode_bytes = if ids + .first() + .is_some_and(|i| pipeline.decode(i, false).is_ok()) + { + for i in &ids { + n += pipeline.decode(i, false).map(|s| s.len()).unwrap_or(0); + } + Some(rss_now().unwrap_or(0) - after_encode) + } else { + None + }; + (after_load, after_encode, decode_bytes) } other => panic!("unknown impl {:?}", other), }; @@ -758,6 +932,7 @@ fn memory_child(which: &str, model: &Path) { json!({ "load_bytes": after_load - rss0, "encode_bytes": after_encode - after_load, + "decode_bytes": decode_bytes, "peak_bytes": rss_peak().map(|p| p - rss0), }) ); @@ -987,12 +1162,33 @@ fn main() { row.insert("stage_ns_per_byte".into(), stages); row.insert("pretok_vs_regex".into(), pretok); } + // Decode: probe once whether the pipeline can decode yet (a loud stub + // today → the pipeline decode series is `null`, rendered "pending"). The + // probe uses a bare id slice so it never touches the legacy encode path. + let (decode_ok, decode_reason) = match pipeline.decode(&[0], false) { + Ok(_) => (true, None), + Err(e) => { + eprintln!(" pipeline decode pending ({shape}): {e}"); + (false, Some(format!("{e}"))) + } + }; + for (row, f) in rows.iter_mut().zip(&fixtures) { + let dec = bench_decode(baseline.as_ref(), &pipeline, f, decode_ok); + let row = row.as_object_mut().unwrap(); + for (k, v) in dec.as_object().unwrap() { + row.insert(k.clone(), v.clone()); + } + } + let decode_threads = + bench_decode_threads(baseline.as_ref(), &pipeline, &all_chunks, decode_ok); + let memory = measure_memory(&path, baseline.is_some()); let threads = bench_threads(baseline.as_ref(), &pipeline, &all_chunks); models.push(json!({ "model": name, "desc": desc, "shape": shape, "results": rows, "memory": memory, "threads": threads, + "decode_threads": decode_threads, "decode_reason": decode_reason, })); } diff --git a/tokenizers/tk-encode/src/tokenizer/pipeline.rs b/tokenizers/tk-encode/src/tokenizer/pipeline.rs index 10f6c6b9a..1526fc40f 100644 --- a/tokenizers/tk-encode/src/tokenizer/pipeline.rs +++ b/tokenizers/tk-encode/src/tokenizer/pipeline.rs @@ -533,6 +533,17 @@ impl PipelineTokenizer { Ok(output) } + /// Decode token ids back to a `String`. + /// + /// Not implemented yet — the pipeline decode path is being built. It fails + /// loud (rather than returning a plausible-but-wrong string) so the oracle + /// test and the comparative benchmark report decode as *pending* instead of + /// silently validating garbage. Implementing this flips the ignored + /// `pipeline_decode_oracle` test on and lights up the decode charts. + pub fn decode(&self, _ids: &[u32], _skip_special_tokens: bool) -> Result { + Err("PipelineTokenizer::decode is not implemented yet".into()) + } + /// Single source of truth for the encode pipeline, generic over how many stages /// run. `STAGE` is a **const generic**, so `if STAGE >= …` folds at compile time and /// the disabled stages are compiled out — the full specialization diff --git a/tokenizers/tk-encode/tests/common/mod.rs b/tokenizers/tk-encode/tests/common/mod.rs new file mode 100644 index 000000000..dece63fe1 --- /dev/null +++ b/tokenizers/tk-encode/tests/common/mod.rs @@ -0,0 +1,55 @@ +//! Shared setup for the pipeline oracle tests. Encode and decode parity are +//! judged over the same inputs: seeded-random windows of the real fixture corpora +//! (`data/fixtures/{lang,modalities}` — 14 languages + code/math/agentic). Random +//! for coverage across varied text, seeded so any failure reproduces exactly. + +use std::path::{Path, PathBuf}; + +/// Test-data root, shared with the benchmark harness (populated by `make fixtures`). +pub const DATA: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../data"); + +/// Window sizes sampled per fixture: one per-input-overhead-sized, one amortized. +pub const WINDOWS: &[usize] = &[1024, 8 * 1024]; + +/// splitmix64 — a tiny deterministic PRNG so the "random" offsets stay reproducible. +pub fn splitmix64(seed: u64) -> u64 { + let mut z = seed.wrapping_add(0x9E37_79B9_7F4A_7C15); + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) +} + +/// Existing `.txt` fixtures under `data/fixtures/{lang,modalities}`, sorted. Empty +/// when fixtures haven't been fetched — callers skip rather than fail. +pub fn fixture_files() -> Vec { + let mut out = Vec::new(); + for group in ["lang", "modalities"] { + let dir = Path::new(DATA).join("fixtures").join(group); + if let Ok(entries) = std::fs::read_dir(&dir) { + let mut paths: Vec = entries + .filter_map(|e| e.ok().map(|e| e.path())) + .filter(|p| p.extension().is_some_and(|x| x == "txt")) + .collect(); + paths.sort(); + out.extend(paths); + } + } + out +} + +/// A `window`-byte slice of `text` from a seeded offset, snapped to char boundaries. +pub fn random_chunk(text: &str, window: usize, seed: u64) -> &str { + let len = text.len(); + if len <= window { + return text; + } + let mut start = splitmix64(seed) as usize % (len - window); + while !text.is_char_boundary(start) { + start += 1; + } + let mut end = (start + window).min(len); + while end < len && !text.is_char_boundary(end) { + end += 1; + } + &text[start..end] +} diff --git a/tokenizers/tk-encode/tests/pipeline_decode_oracle.rs b/tokenizers/tk-encode/tests/pipeline_decode_oracle.rs new file mode 100644 index 000000000..321880b50 --- /dev/null +++ b/tokenizers/tk-encode/tests/pipeline_decode_oracle.rs @@ -0,0 +1,104 @@ +//! `PipelineTokenizer` must decode ids back to exactly the same string as the +//! latest *released* `tokenizers` crate — not the in-tree legacy `Tokenizer`, +//! which is being removed (oracles must not depend on it). +//! +//! Round-trip: encode a fixture window with the release's `encode_fast` +//! (`add_special_tokens = false`), then decode those ids with both the released +//! `decode` and `PipelineTokenizer::decode` and require byte-identical output. +//! Feeding the release's own ids keeps this honest even where the pipeline's +//! *encode* legitimately diverges — decode is judged on decode alone. (0.23.1 has +//! no `decode_fast`; switch to it if a future release adds one.) +//! +//! Inputs and covered tokenizers mirror the encode oracle (`pipeline_oracle.rs`): +//! bert-wiki (WordPiece `##`) and llama-3 (byte-level) over seeded-random windows +//! of the fixture corpora. +//! +//! Behind `bench-baseline`. IGNORED until `PipelineTokenizer::decode` is +//! implemented (a loud stub today, so these fail on `.unwrap()`); un-ignore with +//! the impl by dropping the `#[ignore]` lines: +//! cargo test -p tk-encode --features bench-baseline --test pipeline_decode_oracle -- --ignored + +#![cfg(feature = "bench-baseline")] + +mod common; + +use std::convert::TryFrom; +use std::path::Path; + +use common::{DATA, WINDOWS, fixture_files, random_chunk}; +use tk_encode::Tokenizer; +use tk_encode::pipeline::PipelineTokenizer; +use tokenizers_release::Tokenizer as Released; + +fn check(tok_file: &str) { + let path = Path::new(DATA).join(tok_file); + // The legacy `Tokenizer` only *builds* the pipeline (its sole constructor + // today); it is never a decode reference. Drops out once a direct loader exists. + let Ok(tree) = Tokenizer::from_file(&path) else { + eprintln!("skip {tok_file}: not present (fetch with `make fixtures bench-models`)"); + return; + }; + let Ok(pipeline) = PipelineTokenizer::try_from(&tree) else { + eprintln!("skip {tok_file}: not supported by PipelineTokenizer"); + return; + }; + let released = match Released::from_file(&path) { + Ok(r) => r, + Err(e) => { + eprintln!("skip {tok_file}: released crate can't load it: {e}"); + return; + } + }; + let files = fixture_files(); + if files.is_empty() { + eprintln!("skip {tok_file}: no fixtures under {DATA}/fixtures — run `make fixtures`"); + return; + } + + for (i, f) in files.iter().enumerate() { + let text = std::fs::read_to_string(f).unwrap(); + if text.is_empty() { + continue; + } + for (w, &window) in WINDOWS.iter().enumerate() { + let chunk = random_chunk(&text, window, ((i as u64) << 8) | w as u64); + if chunk.is_empty() { + continue; + } + let ids = released + .encode_fast(chunk, false) + .unwrap() + .get_ids() + .to_vec(); + let expected = released.decode(&ids, false).unwrap(); + let got = pipeline.decode(&ids, false).unwrap(); + assert_eq!( + expected, + got, + "decode mismatch on {} @ {:?}", + f.display(), + chunk.chars().take(60).collect::(), + ); + } + } +} + +macro_rules! decode_tests { + ($($name:ident => $tok:literal),* $(,)?) => { + $( + #[test] + #[ignore = "un-ignore once PipelineTokenizer::decode is implemented"] + fn $name() { + check($tok); + } + )* + }; +} + +// Mirrors the encode oracle's model set; missing files are skipped. +decode_tests! { + bert_wiki => "bert-wiki.json", + bert_base_uncased => "bert-base-uncased.json", + gpt2 => "gpt2.json", + llama3 => "llama-3-tokenizer.json", +} diff --git a/tokenizers/tk-encode/tests/pipeline_oracle.rs b/tokenizers/tk-encode/tests/pipeline_oracle.rs index fc8dc91a1..c2759b4f7 100644 --- a/tokenizers/tk-encode/tests/pipeline_oracle.rs +++ b/tokenizers/tk-encode/tests/pipeline_oracle.rs @@ -1,81 +1,107 @@ -//! The experimental `PipelineTokenizer` must produce exactly the same token -//! ids as the reference `Tokenizer` (the oracle). Exercised over the bert-wiki -//! tokenizer (`Whitespace` pre-tokenizer + `WordPiece`) on an English and a -//! Japanese corpus, with lines packed into ~1 kB and ~10 kB documents, for both -//! `add_special_tokens` values. bert-wiki's post-processor is -//! `Template("[CLS] A [SEP]")`, so `true` genuinely exercises `STAGE_POSTPROCESS` -//! on both corpora. +//! `PipelineTokenizer` must encode to exactly the same token ids as the latest +//! *released* `tokenizers` crate — not the in-tree legacy `Tokenizer`, which is +//! being removed (oracles must not depend on it). We compare against the release's +//! `encode_fast` (its offset-free path; the pipeline computes no offsets either), +//! over seeded-random windows of the fixture corpora. +//! +//! Covered: bert-wiki (Whitespace + WordPiece) and llama-3 (byte-level BPE) — a +//! model the pipeline can't build or encode yet is skipped, not failed. +//! +//! Behind the `bench-baseline` feature (the released crate is optional): +//! cargo test -p tk-encode --features bench-baseline --test pipeline_oracle + +#![cfg(feature = "bench-baseline")] + +mod common; use std::convert::TryFrom; +use std::path::Path; +use common::{DATA, WINDOWS, fixture_files, random_chunk}; use tk_encode::Tokenizer; use tk_encode::pipeline::PipelineTokenizer; +use tokenizers_release::Tokenizer as Released; -fn load(corpus: &str) -> (Tokenizer, PipelineTokenizer, String) { - let oracle = Tokenizer::from_file("../data/bert-wiki.json").unwrap(); - let pipeline = PipelineTokenizer::try_from(&oracle).unwrap(); - let text = std::fs::read_to_string(corpus).unwrap(); - (oracle, pipeline, text) -} +const PROBE: &str = "The quick brown fox jumps 123."; -fn make_chunks(text: &str, target_bytes: usize) -> Vec { - let lines = text.lines().filter(|l| !l.trim().is_empty()); - let mut chunks = Vec::new(); - let mut cur = String::new(); - for line in lines { - if !cur.is_empty() { - cur.push('\n'); - } - cur.push_str(line); - if cur.len() >= target_bytes { - chunks.push(std::mem::take(&mut cur)); - } +fn check(tok_file: &str) { + let path = Path::new(DATA).join(tok_file); + // The legacy `Tokenizer` only *builds* the pipeline (its sole constructor + // today); it is never an encode reference. Drops out once a direct loader exists. + let Ok(tree) = Tokenizer::from_file(&path) else { + eprintln!("skip {tok_file}: not present (fetch with `make fixtures bench-models`)"); + return; + }; + let Ok(pipeline) = PipelineTokenizer::try_from(&tree) else { + eprintln!("skip {tok_file}: not supported by PipelineTokenizer"); + return; + }; + // Build constraints can be met while encode is still unimplemented for this shape. + if pipeline.encode(PROBE, false).is_err() { + eprintln!("skip {tok_file}: pipeline can't encode this model yet"); + return; } - if !cur.is_empty() { - chunks.push(cur); + let released = match Released::from_file(&path) { + Ok(r) => r, + Err(e) => { + eprintln!("skip {tok_file}: released crate can't load it: {e}"); + return; + } + }; + let files = fixture_files(); + if files.is_empty() { + eprintln!("skip {tok_file}: no fixtures under {DATA}/fixtures — run `make fixtures`"); + return; } - chunks -} -fn check_chunks(corpus: &str, target_bytes: usize) { - let (oracle, pipeline, text) = load(corpus); - for chunk in make_chunks(&text, target_bytes) { - for add_special_tokens in [false, true] { - let expected = oracle.encode(chunk.as_str(), add_special_tokens).unwrap(); + for (i, f) in files.iter().enumerate() { + let text = std::fs::read_to_string(f).unwrap(); + if text.is_empty() { + continue; + } + for (w, &window) in WINDOWS.iter().enumerate() { + let chunk = random_chunk(&text, window, ((i as u64) << 8) | w as u64); + if chunk.is_empty() { + continue; + } + let expected = released + .encode_fast(chunk, false) + .unwrap() + .get_ids() + .to_vec(); let got: Vec = pipeline - .encode(&chunk, add_special_tokens) + .encode(chunk, false) .unwrap() .iter() .map(|t| t.id) .collect(); assert_eq!( - expected.get_ids(), - got.as_slice(), - "id mismatch (add_special_tokens={add_special_tokens}) on {:?}", - chunk.chars().take(80).collect::(), + expected, + got, + "id mismatch on {} @ {:?}", + f.display(), + chunk.chars().take(60).collect::(), ); } } } -macro_rules! corpus_tests { - ($($name:ident => $file:literal),* $(,)?) => { +macro_rules! oracle_tests { + ($($name:ident => $tok:literal),* $(,)?) => { $( - mod $name { - #[test] - fn chunks_1kb() { - super::check_chunks($file, 1024); - } - #[test] - fn chunks_10kb() { - super::check_chunks($file, 10 * 1024); - } + #[test] + fn $name() { + check($tok); } )* }; } -corpus_tests! { - big => "../data/big.txt", - wagahai => "../data/unigram_wagahaiwa_nekodearu.txt", +// Two decode/encode archetypes each; whichever files a given checkout has get run +// (bert-wiki + llama-3 ship with `make test`; the rest with `make bench-models`). +oracle_tests! { + bert_wiki => "bert-wiki.json", + bert_base_uncased => "bert-base-uncased.json", + gpt2 => "gpt2.json", + llama3 => "llama-3-tokenizer.json", } From 9ad0757278c4b872c7d45273ecb798b32027f712 Mon Sep 17 00:00:00 2001 From: SBrandeis <33657802+SBrandeis@users.noreply.github.com> Date: Wed, 22 Jul 2026 16:30:05 +0200 Subject: [PATCH 02/19] docs iter --- CONTRIBUTING.md | 32 ++++++++++++++++++++++++++++++ tokenizers/tk-encode/README.md | 36 ---------------------------------- 2 files changed, 32 insertions(+), 36 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fbf97d608..ee751c43d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -167,3 +167,35 @@ Profile from Python to see the full stack including PyO3 overhead: ```bash samply record python my_script.py ``` + +### PipelineTokenizer oracle tests & benchmark + +`PipelineTokenizer` (in `tk-encode`) is an experimental reimplementation of the +encode/decode pipeline. It is checked for parity against the latest *released* +`tokenizers` crate — same token ids on encode, same decoded string — rather than +the in-tree `Tokenizer`, which is being retired. + +Those checks link the released crate, so they live behind the `bench-baseline` +feature and a plain `cargo test` skips them. Fetch the corpora and model +tokenizers, then run with the feature: + +```bash +cd tokenizers +make fixtures bench-models # needs HF_TOKEN + +# encode parity: pipeline ids == released encode_fast ids +cargo test -p tk-encode --features bench-baseline --test pipeline_oracle + +# decode parity: #[ignore]d until PipelineTokenizer::decode is implemented +cargo test -p tk-encode --features bench-baseline --test pipeline_decode_oracle -- --ignored +``` + +The comparative benchmark (throughput, thread scaling, and memory vs the release) +runs off the same data and renders to charts: + +```bash +cargo run --release -p tk-encode --features bench-baseline --example fixture_bench > bench.json +python3 ../.github/scripts/render_pipeline_bench.py bench.json +``` + +CI runs both in the **Pipeline Benchmark** workflow (`.github/workflows/pipeline-bench.yml`). diff --git a/tokenizers/tk-encode/README.md b/tokenizers/tk-encode/README.md index 2456a04ca..5f3a9871b 100644 --- a/tokenizers/tk-encode/README.md +++ b/tokenizers/tk-encode/README.md @@ -66,42 +66,6 @@ Training lives in the companion `tk-train` crate (re-exported by the environment variable. As an example setting `RAYON_RS_NUM_THREADS=4` will allocate a maximum of 4 threads. **_Please note this behavior may evolve in the future_** -## PipelineTokenizer: oracle tests & benchmark - -`PipelineTokenizer` is an experimental, allocation-light re-implementation of the -encode/decode pipeline. Its correctness is judged against the **latest released -`tokenizers` crate** (not the in-tree `Tokenizer`, which is being retired): the -pipeline must produce identical token ids and identical decoded text. - -That comparison lives behind the optional `bench-baseline` feature (it links the -released crate), so a plain `cargo test` skips it. To run it you need the fixture -corpora and model tokenizers, then the feature flag: - -```bash -# from tokenizers/ — fetches data/fixtures/ + model tokenizers (needs HF_TOKEN) -make fixtures bench-models - -# encode parity (pipeline ids == released `encode_fast` ids) -cargo test -p tk-encode --features bench-baseline --test pipeline_oracle - -# decode parity (pipeline decode == released `decode`) — ignored until -# PipelineTokenizer::decode is implemented, so pass --ignored to run it -cargo test -p tk-encode --features bench-baseline --test pipeline_decode_oracle -- --ignored -``` - -Both oracles sample seeded-random windows of every fixture corpus; a model whose -tokenizer file isn't present is skipped. CI runs them in the **Pipeline -Benchmark** workflow (`.github/workflows/pipeline-bench.yml`). - -The same workflow runs the comparative benchmark — throughput, thread scaling, -memory, and binary size vs the release — and renders it to charts. To reproduce -locally: - -```bash -cargo run --release -p tk-encode --features bench-baseline --example fixture_bench > bench.json -python3 ../.github/scripts/render_pipeline_bench.py bench.json # writes SVGs + pipeline_bench.md -``` - ## Features - **progressbar**: The progress bar visualization is enabled by default. It might be disabled if From 8d2452d8505a6e42866b969b694b51e23bcdf4eb Mon Sep 17 00:00:00 2001 From: Simon Brandeis <33657802+SBrandeis@users.noreply.github.com> Date: Wed, 22 Jul 2026 17:38:21 +0200 Subject: [PATCH 03/19] Apply suggestions from code review Co-authored-by: Simon Brandeis <33657802+SBrandeis@users.noreply.github.com> --- CONTRIBUTING.md | 4 ++-- tokenizers/tk-encode/tests/pipeline_decode_oracle.rs | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ee751c43d..b5b568c72 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -186,8 +186,8 @@ make fixtures bench-models # needs HF_TOKEN # encode parity: pipeline ids == released encode_fast ids cargo test -p tk-encode --features bench-baseline --test pipeline_oracle -# decode parity: #[ignore]d until PipelineTokenizer::decode is implemented -cargo test -p tk-encode --features bench-baseline --test pipeline_decode_oracle -- --ignored +# decode parity +cargo test -p tk-encode --features bench-baseline --test pipeline_decode_oracle ``` The comparative benchmark (throughput, thread scaling, and memory vs the release) diff --git a/tokenizers/tk-encode/tests/pipeline_decode_oracle.rs b/tokenizers/tk-encode/tests/pipeline_decode_oracle.rs index 321880b50..0a0c294f1 100644 --- a/tokenizers/tk-encode/tests/pipeline_decode_oracle.rs +++ b/tokenizers/tk-encode/tests/pipeline_decode_oracle.rs @@ -87,7 +87,6 @@ macro_rules! decode_tests { ($($name:ident => $tok:literal),* $(,)?) => { $( #[test] - #[ignore = "un-ignore once PipelineTokenizer::decode is implemented"] fn $name() { check($tok); } From 5e8ca6f4ad8f3b91944e1de350a6c907428d0bb2 Mon Sep 17 00:00:00 2001 From: SBrandeis <33657802+SBrandeis@users.noreply.github.com> Date: Wed, 22 Jul 2026 10:45:47 +0200 Subject: [PATCH 04/19] tokenizers_pipeline --- bindings/python-pipeline/Cargo.lock | 1707 +++++++++++++++++ bindings/python-pipeline/Cargo.toml | 24 + bindings/python-pipeline/clippy.toml | 6 + .../examples/01_train_and_encode.py | 81 + .../python-pipeline/examples/02_pretrained.py | 53 + .../python-pipeline/examples/03_threading.py | 62 + .../py_src/tokenizers_pipeline/__init__.py | 18 + .../py_src/tokenizers_pipeline/__init__.pyi | 140 ++ .../tokenizers_pipeline/models/__init__.py | 9 + .../tokenizers_pipeline/models/__init__.pyi | 29 + .../normalizers/__init__.py | 29 + .../normalizers/__init__.pyi | 53 + .../pre_tokenizers/__init__.py | 29 + .../pre_tokenizers/__init__.pyi | 59 + .../py_src/tokenizers_pipeline/py.typed | 0 .../tokenizers_pipeline/trainers/__init__.py | 15 + .../tokenizers_pipeline/trainers/__init__.pyi | 26 + bindings/python-pipeline/pyproject.toml | 19 + bindings/python-pipeline/src/added_token.rs | 102 + bindings/python-pipeline/src/detached_lock.rs | 66 + bindings/python-pipeline/src/error.rs | 9 + bindings/python-pipeline/src/lib.rs | 68 + bindings/python-pipeline/src/models.rs | 145 ++ bindings/python-pipeline/src/normalizers.rs | 179 ++ .../python-pipeline/src/pre_tokenizers.rs | 244 +++ bindings/python-pipeline/src/tokenizer.rs | 526 +++++ bindings/python-pipeline/src/trainers.rs | 179 ++ .../python-pipeline/tools/stub-gen/Cargo.lock | 178 ++ .../python-pipeline/tools/stub-gen/Cargo.toml | 10 + .../tools/stub-gen/src/main.rs | 122 ++ 30 files changed, 4187 insertions(+) create mode 100644 bindings/python-pipeline/Cargo.lock create mode 100644 bindings/python-pipeline/Cargo.toml create mode 100644 bindings/python-pipeline/clippy.toml create mode 100644 bindings/python-pipeline/examples/01_train_and_encode.py create mode 100644 bindings/python-pipeline/examples/02_pretrained.py create mode 100644 bindings/python-pipeline/examples/03_threading.py create mode 100644 bindings/python-pipeline/py_src/tokenizers_pipeline/__init__.py create mode 100644 bindings/python-pipeline/py_src/tokenizers_pipeline/__init__.pyi create mode 100644 bindings/python-pipeline/py_src/tokenizers_pipeline/models/__init__.py create mode 100644 bindings/python-pipeline/py_src/tokenizers_pipeline/models/__init__.pyi create mode 100644 bindings/python-pipeline/py_src/tokenizers_pipeline/normalizers/__init__.py create mode 100644 bindings/python-pipeline/py_src/tokenizers_pipeline/normalizers/__init__.pyi create mode 100644 bindings/python-pipeline/py_src/tokenizers_pipeline/pre_tokenizers/__init__.py create mode 100644 bindings/python-pipeline/py_src/tokenizers_pipeline/pre_tokenizers/__init__.pyi create mode 100644 bindings/python-pipeline/py_src/tokenizers_pipeline/py.typed create mode 100644 bindings/python-pipeline/py_src/tokenizers_pipeline/trainers/__init__.py create mode 100644 bindings/python-pipeline/py_src/tokenizers_pipeline/trainers/__init__.pyi create mode 100644 bindings/python-pipeline/pyproject.toml create mode 100644 bindings/python-pipeline/src/added_token.rs create mode 100644 bindings/python-pipeline/src/detached_lock.rs create mode 100644 bindings/python-pipeline/src/error.rs create mode 100644 bindings/python-pipeline/src/lib.rs create mode 100644 bindings/python-pipeline/src/models.rs create mode 100644 bindings/python-pipeline/src/normalizers.rs create mode 100644 bindings/python-pipeline/src/pre_tokenizers.rs create mode 100644 bindings/python-pipeline/src/tokenizer.rs create mode 100644 bindings/python-pipeline/src/trainers.rs create mode 100644 bindings/python-pipeline/tools/stub-gen/Cargo.lock create mode 100644 bindings/python-pipeline/tools/stub-gen/Cargo.toml create mode 100644 bindings/python-pipeline/tools/stub-gen/src/main.rs diff --git a/bindings/python-pipeline/Cargo.lock b/bindings/python-pipeline/Cargo.lock new file mode 100644 index 000000000..8c20a0a4e --- /dev/null +++ b/bindings/python-pipeline/Cargo.lock @@ -0,0 +1,1707 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "serde", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "arbitrary-chunks" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ad8689a486416c401ea15715a4694de30054248ec627edbf31f49cb64ee4086" + +[[package]] +name = "atomsplit" +version = "0.1.0" +dependencies = [ + "memchr", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "bitvec" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "block-pseudorand" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2097358495d244a0643746f4d13eedba4608137008cf9dec54e53a3b700115a6" +dependencies = [ + "chiapos-chacha8", + "nanorand", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + +[[package]] +name = "cc" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core 0.10.1", +] + +[[package]] +name = "chiapos-chacha8" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33f8be573a85f6c2bc1b8e43834c07e32f95e489b914bf856c0549c3c269cd0a" +dependencies = [ + "rayon", +] + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colored" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "compact_str" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "serde", + "static_assertions", +] + +[[package]] +name = "console" +version = "0.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fe5f465a4f6fee88fad41b85d990f84c835335e85b5d9e6e63e0d06d28cba7c" +dependencies = [ + "encode_unicode", + "libc", + "unicode-width", + "windows-sys", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools 0.10.5", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "daachorse" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5614204febbc33cc07a2806aa6440b904ac012b68eecc37f4493ea4a76455a3d" + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "dary_heap" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" +dependencies = [ + "serde", +] + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.119", +] + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "esaxx-rs" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" +dependencies = [ + "cc", +] + +[[package]] +name = "fancy-regex" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "futures-core" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" + +[[package]] +name = "futures-task" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" + +[[package]] +name = "futures-util" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "indicatif" +version = "0.18.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9433806cd6b4ec1aba79c021c7e4c58fb4c3b9977c085062e611ac929998fb0c" +dependencies = [ + "console", + "portable-atomic", + "unicode-width", + "unit-prefix", + "web-time", +] + +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "macro_rules_attribute" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65049d7923698040cd0b1ddcced9b0eb14dd22c5f86ae59c3740eab64a676520" +dependencies = [ + "macro_rules_attribute-proc_macro", + "paste", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30" + +[[package]] +name = "matrixmultiply" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7" +dependencies = [ + "autocfg", + "rawpointer", +] + +[[package]] +name = "mem_dbg" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4ef2d80bfa14894b6d5a3ff537e7e9a908dbf4c95de8a5b8ad2a473301676e6" +dependencies = [ + "bitflags", + "hashbrown", + "mem_dbg-derive", +] + +[[package]] +name = "mem_dbg-derive" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73acd151c6ce84a41d8d6fb0958d9a3d5a18d649ad5a85ad5b719439af8ad257" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "monostate" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3341a273f6c9d5bef1908f17b7267bbab0e95c9bf69a0d4dcf8e9e1b2c76ef67" +dependencies = [ + "monostate-impl", + "serde", + "serde_core", +] + +[[package]] +name = "monostate-impl" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "nanorand" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "729eb334247daa1803e0a094d0a5c55711b85571179f5ec6e53eccfdf7008958" + +[[package]] +name = "ndarray" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "numpy" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a5b15d63a5ff39e378daed0e1340d3a5964703ea9712eb09a0dc66fade996f4" +dependencies = [ + "libc", + "ndarray", + "num-complex", + "num-integer", + "num-traits", + "pyo3", + "pyo3-build-config", + "rustc-hash", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "partition" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "947f833aaa585cf12b8ec7c0476c98784c49f33b861376ffc84ed92adebf2aba" + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "portable-atomic" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prefetch-index" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9057806a8d77d67bccdc0f542db43737a6f19ada3efab2adc63277feea27310f" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "ptr_hash" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f184d2c69ac0853853275df42e7160a7dc4f3248d93434002c28de27ed3f6d0" +dependencies = [ + "bitvec", + "colored", + "fastrand", + "fxhash", + "itertools 0.15.0", + "log", + "mem_dbg", + "prefetch-index", + "rand 0.10.2", + "rand_chacha 0.10.0", + "rayon", + "rdst", + "serde", + "tempfile", + "xxhash-rust", +] + +[[package]] +name = "pyo3" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd274650b21d4bfc26a0a47587962c1edb425f69287324355cd040c3ea66071c" +dependencies = [ + "libc", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", +] + +[[package]] +name = "pyo3-build-config" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5e2a7d2f0d013342f295c048ad19237add5154a55b1c5a254c0ec93d4109078" +dependencies = [ + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca85c467da1bbc8d866eea5deff9cf29ea5f7785054a17da36e65bda9c05845b" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ac53762fd065daa3194dd09337a38bd793a188100fd1a9304c4ab312d901771" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca3a1557399783172dc5bf39cfca835157732532cba56b71d2292161e53b362" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e6af7f3e25ded52c41df4e0b1af2d047e45896c2f3281792ed68a1c243daedb" +dependencies = [ + "ppv-lite86", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-cond" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f" +dependencies = [ + "either", + "itertools 0.14.0", + "rayon", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "rdst" +version = "0.20.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e7970b4e577b76a96d5e56b5f6662b66d1a4e1f5bb026ee118fc31b373c2752" +dependencies = [ + "arbitrary-chunks", + "block-pseudorand", + "criterion", + "partition", + "rayon", + "tikv-jemallocator", + "voracious_radix_sort", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64", + "nom", + "serde", + "unicode-segmentation", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "target-lexicon" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "tikv-jemalloc-sys" +version = "0.5.4+5.3.0-patched" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9402443cb8fd499b6f327e40565234ff34dbda27460c5b47db0db77443dd85d1" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "tikv-jemallocator" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "965fe0c26be5c56c94e38ba547249074803efd52adfb66de62107d95aab3eaca" +dependencies = [ + "libc", + "tikv-jemalloc-sys", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tk-encode" +version = "0.23.2-dev.0" +dependencies = [ + "ahash", + "atomsplit", + "compact_str", + "daachorse", + "dary_heap", + "derive_builder", + "fancy-regex", + "getrandom 0.3.4", + "indicatif", + "itertools 0.14.0", + "log", + "macro_rules_attribute", + "memchr", + "monostate", + "paste", + "ptr_hash", + "rand 0.9.5", + "rayon", + "rayon-cond", + "regex", + "serde", + "serde_json", + "spm_precompiled", + "thiserror", + "unicode-normalization", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", + "yada", +] + +[[package]] +name = "tk-train" +version = "0.23.2-dev.0" +dependencies = [ + "ahash", + "compact_str", + "dary_heap", + "derive_builder", + "esaxx-rs", + "indicatif", + "itertools 0.14.0", + "log", + "rayon", + "serde", + "serde_json", + "thiserror", + "tk-encode", +] + +[[package]] +name = "tokenizers-pipeline-python" +version = "0.1.0" +dependencies = [ + "libc", + "numpy", + "pyo3", + "rayon", + "serde", + "serde_json", + "tk-encode", + "tk-train", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "unit-prefix" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "voracious_radix_sort" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446e7ffcb6c27a71d05af7e51ef2ee5b71c48424b122a832f2439651e1914899" +dependencies = [ + "rayon", +] + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + +[[package]] +name = "xxhash-rust" +version = "0.8.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" + +[[package]] +name = "yada" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c3bb06259642a57b4ea1bf2a8260f7d94b7b78a096c46f193318918d925f61" + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/bindings/python-pipeline/Cargo.toml b/bindings/python-pipeline/Cargo.toml new file mode 100644 index 000000000..643f4e5f7 --- /dev/null +++ b/bindings/python-pipeline/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "tokenizers-pipeline-python" +version = "0.1.0" +edition = "2024" +description = "Experimental Python bindings for the tokenizers PipelineTokenizer" +license = "Apache-2.0" + +[lib] +name = "_native" +crate-type = ["cdylib", "rlib"] + +[dependencies] +pyo3 = { version = "=0.29", features = ["abi3-py310", "experimental-inspect"] } +numpy = "0.29" +rayon = "1.10" +libc = "0.2" +serde = "1.0" +serde_json = "1.0" +tk-encode = { path = "../../tokenizers/tk-encode", features = ["fancy-regex"] } +tk-train = { path = "../../tokenizers/tk-train" } + +[features] +default = [] +ext-module = ["pyo3/extension-module"] diff --git a/bindings/python-pipeline/clippy.toml b/bindings/python-pipeline/clippy.toml new file mode 100644 index 000000000..aec55c952 --- /dev/null +++ b/bindings/python-pipeline/clippy.toml @@ -0,0 +1,6 @@ +disallowed-types = [ + { path = "std::sync::RwLock", reason = "use DetachedRwLock: blocking on a lock while attached to the interpreter can deadlock against training (lock-before-GIL ordering); the only raw RwLock lives inside detached_lock.rs" }, +] +disallowed-methods = [ + { path = "pyo3::marker::Python::attach", reason = "re-attaching can invert the lock/GIL order; the one vetted use is BufferedPyIterator::refill (holds the lock, then attaches — the sanctioned direction)" }, +] diff --git a/bindings/python-pipeline/examples/01_train_and_encode.py b/bindings/python-pipeline/examples/01_train_and_encode.py new file mode 100644 index 000000000..c7463234e --- /dev/null +++ b/bindings/python-pipeline/examples/01_train_and_encode.py @@ -0,0 +1,81 @@ +"""End-to-end: build a tokenizer from scratch, train it, mutate its components +in place, encode, serialize, pickle, and hit the decode stub.""" + +import pickle +import tempfile +from pathlib import Path + +import numpy as np + +from tokenizers_pipeline import AddedToken, Tokenizer, models, normalizers, pre_tokenizers, trainers + +DATA = Path(__file__).resolve().parents[3] / "tokenizers" / "data" + + +def corpus(): + with open(DATA / "big.txt", encoding="utf-8") as f: + for i, line in enumerate(f): + if i >= 20_000: + break + yield line + + +# 1. Build: model + components, assigned as plain values +tok = Tokenizer(models.BPE()) +tok.normalizer = normalizers.Sequence([normalizers.NFKC(), normalizers.Lowercase()]) +tok.pre_tokenizer = pre_tokenizers.Whitespace() + +# 2. Train from a Python iterator (GIL only taken to refill 256-line buffers) +trainer = trainers.BpeTrainer( + vocab_size=1000, + special_tokens=["", "", AddedToken("", special=True)], + show_progress=False, +) +tok.train_from_iterator(corpus(), trainer=trainer) +print(f"trained: {tok!r}") +assert tok.get_vocab_size() == 1000, tok.get_vocab_size() +assert tok.token_to_id("") == 0 + +# 3. Encode -> numpy uint32 array; special tokens are matched in the text +ids = tok.encode("The quick brown fox jumps over the lazy dog") +print(f"ids: {ids.dtype} {ids}") +assert isinstance(ids, np.ndarray) and ids.dtype == np.uint32 +assert tok.token_to_id("") in ids +assert [tok.id_to_token(int(i)) for i in ids[:2]] is not None + +# 4. Mutate a component in place: dropping the lowercasing normalizer changes ids +tok_ids_lower = tok.encode("HELLO WORLD") +tok.normalizer = normalizers.NFKC() +tok_ids_upper = tok.encode("HELLO WORLD") +assert not np.array_equal(tok_ids_lower, tok_ids_upper), "normalizer change must affect ids" +tok.normalizer = normalizers.Sequence([normalizers.NFKC(), normalizers.Lowercase()]) +assert np.array_equal(tok.encode("HELLO WORLD"), tok_ids_lower) +print(f"component swap: {tok.normalizer!r}") + +# 5. Post-hoc vocabulary extension +added = tok.add_special_tokens([""]) +assert added == 1 and tok.token_to_id("") is not None +assert tok.token_to_id("") in tok.encode("a b") + +# 6. Serialize / reload round-trip +with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "tokenizer.json" + tok.save(path) + reloaded = Tokenizer.from_file(path) +text = "Round-trip: 42 tokens?" +assert np.array_equal(tok.encode(text), reloaded.encode(text)) +print("save/load round-trip: identical ids") + +# 7. Pickle round-trip (multiprocessing readiness) +unpickled = pickle.loads(pickle.dumps(tok)) +assert np.array_equal(tok.encode(text), unpickled.encode(text)) +print("pickle round-trip: identical ids") + +# 8. decode is a stub for now +try: + tok.decode(ids) + raise AssertionError("decode should not be implemented yet") +except NotImplementedError as e: + print(f"decode stub: NotImplementedError({e})") + +print("OK") diff --git a/bindings/python-pipeline/examples/02_pretrained.py b/bindings/python-pipeline/examples/02_pretrained.py new file mode 100644 index 000000000..3eccd15d6 --- /dev/null +++ b/bindings/python-pipeline/examples/02_pretrained.py @@ -0,0 +1,53 @@ +"""Load real tokenizer.json files and check id parity against the released +`tokenizers` package on a real corpus. Also demonstrates the two loud failure +modes: unsupported pre-tokenizers and unwired post-processing.""" + +from pathlib import Path + +import tokenizers as reference + +from tokenizers_pipeline import Tokenizer, TokenizersError + +DATA = Path(__file__).resolve().parents[3] / "tokenizers" / "data" + +with open(DATA / "big.txt", encoding="utf-8") as f: + LINES = [line for line in f.read(500_000).splitlines() if line.strip()] +print(f"corpus: {len(LINES)} lines") + +for name, file in [ + ("gpt2", "gpt2.json"), + ("llama-3", "llama-3-tokenizer.json"), + ("llama-2", "llama-2.json"), + ("bert-base-uncased", "bert-base-uncased.json"), +]: + tok = Tokenizer.from_file(DATA / file) + ref = reference.Tokenizer.from_file(str(DATA / file)) + + ours = tok.encode_batch(LINES, add_special_tokens=False) + theirs = ref.encode_batch_fast(LINES, add_special_tokens=False) + mismatches = sum( + 1 for a, b in zip(ours, theirs, strict=True) if a.tolist() != b.ids + ) + total = sum(len(a) for a in ours) + assert mismatches == 0, f"{name}: {mismatches} mismatching lines" + print(f"{name}: {total} tokens, ids identical to `tokenizers` {reference.__version__}") + +# Expected failure 1: post-processor would add special tokens -> loud error, +# not silently wrong ids +bert = Tokenizer.from_file(DATA / "bert-base-uncased.json") +try: + bert.encode("hello") + raise AssertionError("should have raised") +except NotImplementedError as e: + print(f"bert with add_special_tokens=True: NotImplementedError({e})") + +# Expected failure 2: pipeline-unsupported component (Metaspace) -> loud error +# at compile time, with the reason +t5 = Tokenizer.from_file(DATA / "t5-base.json") +try: + t5.encode("hello", add_special_tokens=False) + raise AssertionError("should have raised") +except TokenizersError as e: + print(f"t5-base (Metaspace): TokenizersError({e})") + +print("OK") diff --git a/bindings/python-pipeline/examples/03_threading.py b/bindings/python-pipeline/examples/03_threading.py new file mode 100644 index 000000000..cc12166f2 --- /dev/null +++ b/bindings/python-pipeline/examples/03_threading.py @@ -0,0 +1,62 @@ +"""Demonstrates that encode runs without the GIL: Python threads calling +encode() scale, and encode_batch parallelizes in Rust via rayon.""" + +import os +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +from tokenizers_pipeline import Tokenizer + +DATA = Path(__file__).resolve().parents[3] / "tokenizers" / "data" +N_THREADS = 4 + +tok = Tokenizer.from_file(DATA / "llama-3-tokenizer.json") +with open(DATA / "big.txt", encoding="utf-8") as f: + text = f.read(2_000_000) +lines = [line for line in text.splitlines() if line.strip()] + +tok.encode(text, add_special_tokens=False) # warmup + compile + + +def encode_once(): + return tok.encode(text, add_special_tokens=False) + + +# 1. Python threads: with the GIL held during encode this could not scale +start = time.perf_counter() +for _ in range(N_THREADS): + encode_once() +sequential = time.perf_counter() - start + +with ThreadPoolExecutor(N_THREADS) as pool: # warmup thread pool + list(pool.map(lambda _: None, range(N_THREADS))) + start = time.perf_counter() + results = list(pool.map(lambda _: encode_once(), range(N_THREADS))) + threaded = time.perf_counter() - start + +speedup = sequential / threaded +print(f"{N_THREADS} encodes of {len(text) / 1e6:.1f}MB: " + f"sequential {sequential:.2f}s, {N_THREADS} threads {threaded:.2f}s " + f"({speedup:.1f}x)") +assert speedup > 1.5, f"threads did not scale ({speedup:.2f}x): is the GIL held?" + +# 2. encode_batch: rayon parallelism inside one call, toggled by env var +os.environ["TOKENIZERS_PARALLELISM"] = "false" +start = time.perf_counter() +serial_ids = tok.encode_batch(lines, add_special_tokens=False) +serial = time.perf_counter() - start + +os.environ["TOKENIZERS_PARALLELISM"] = "true" +tok.encode_batch(lines[:100], add_special_tokens=False) # spin up the pool +start = time.perf_counter() +parallel_ids = tok.encode_batch(lines, add_special_tokens=False) +parallel = time.perf_counter() - start + +assert all(a.tolist() == b.tolist() for a, b in zip(serial_ids, parallel_ids, strict=True)) +mbps = len(text) / parallel / 1e6 +print(f"encode_batch {len(lines)} lines: serial {serial:.2f}s, " + f"rayon {parallel:.2f}s ({serial / parallel:.1f}x, {mbps:.0f} MB/s)") +assert serial / parallel > 1.5, "rayon batch did not scale" + +print("OK") diff --git a/bindings/python-pipeline/py_src/tokenizers_pipeline/__init__.py b/bindings/python-pipeline/py_src/tokenizers_pipeline/__init__.py new file mode 100644 index 000000000..f63f89f55 --- /dev/null +++ b/bindings/python-pipeline/py_src/tokenizers_pipeline/__init__.py @@ -0,0 +1,18 @@ +from ._native import ( + AddedToken, + Tokenizer, + TokenizersError, + __version__, +) +from . import models, normalizers, pre_tokenizers, trainers + +__all__ = [ + "AddedToken", + "Tokenizer", + "TokenizersError", + "__version__", + "models", + "normalizers", + "pre_tokenizers", + "trainers", +] diff --git a/bindings/python-pipeline/py_src/tokenizers_pipeline/__init__.pyi b/bindings/python-pipeline/py_src/tokenizers_pipeline/__init__.pyi new file mode 100644 index 000000000..e421d42b6 --- /dev/null +++ b/bindings/python-pipeline/py_src/tokenizers_pipeline/__init__.pyi @@ -0,0 +1,140 @@ +import numpy as np +import numpy.typing as npt + +from tokenizers_pipeline.models import Model +from tokenizers_pipeline.normalizers import Normalizer +from tokenizers_pipeline.pre_tokenizers import PreTokenizer +from tokenizers_pipeline.trainers import Trainer +from _typeshed import Incomplete +from collections.abc import Sequence +from os import PathLike +from typing import Any, Final, final +__version__: Final[str] + +@final +class AddedToken: + """ + A token added on top of the model's vocabulary, with its matching options. + """ + def __new__(cls, /, content: str, *, single_word: bool = False, lstrip: bool = False, rstrip: bool = False, normalized: bool |None = None, special: bool = False) -> AddedToken: ... + def __repr__(self, /) -> str: ... + @property + def content(self, /) -> str: ... + @property + def lstrip(self, /) -> bool: ... + @property + def normalized(self, /) -> bool: ... + @property + def rstrip(self, /) -> bool: ... + @property + def single_word(self, /) -> bool: ... + @property + def special(self, /) -> bool: ... + +@final +class Tokenizer: + """ + A tokenizer: a model plus its optional normalizer and pre-tokenizer. + Mutations apply to the serializable definition; encode runs a compiled + pipeline that is rebuilt automatically after any change. + """ + def __new__(cls, /, model: Model) -> Tokenizer: ... + def __reduce__(self, /) -> tuple[Any, tuple[bytes]]: ... + def __repr__(self, /) -> str: ... + def add_special_tokens(self, /, tokens: Sequence[str |AddedToken]) -> int: + """ + Add special tokens (never split, skipped on decode) to the vocabulary. + """ + def add_tokens(self, /, tokens: Sequence[str |AddedToken]) -> int: + """ + Add tokens to the vocabulary, matched literally in the input text. + """ + def decode(self, /, ids: Sequence[int], *, skip_special_tokens: bool = True) -> str: + """ + Not implemented yet: decoding is not part of the encode pipeline. + """ + def encode(self, /, text: str, *, add_special_tokens: bool = True) -> "npt.NDArray[np.uint32]": + """ + Encode `text` into token ids. + + Runs entirely outside the interpreter lock and returns a `numpy.uint32` + array backed by the Rust output buffer (no copy). + """ + def encode_batch(self, /, texts: Sequence[str], *, add_special_tokens: bool = True) -> "list[npt.NDArray[np.uint32]]": + """ + Encode a batch of texts, in parallel across Rust threads (respects + `TOKENIZERS_PARALLELISM`), without holding the interpreter lock. + Input strings are borrowed, not copied; each output is a `numpy.uint32` + array backed by its Rust buffer. + """ + @staticmethod + def from_buffer(buffer: Sequence[int]) -> "Tokenizer": + """ + Load a tokenizer from the bytes of a `tokenizer.json` file. + """ + @staticmethod + def from_file(path: str |PathLike[str]) -> "Tokenizer": + """ + Load a tokenizer from a `tokenizer.json` file. + """ + @staticmethod + def from_pretrained(identifier: str, *, revision: str = ..., token: str |None = None) -> "Tokenizer": + """ + Download `tokenizer.json` from a model on the Hugging Face Hub (requires + the `huggingface_hub` package) and load it. + """ + def get_vocab(self, /, *, with_added_tokens: bool = True) -> dict[str, int]: + """ + The whole vocabulary as a dict. This copies every entry; prefer + `token_to_id` for lookups. + """ + def get_vocab_size(self, /, *, with_added_tokens: bool = True) -> int: ... + def id_to_token(self, /, id: int) -> str |None: ... + @property + def model(self, /) -> Model: + """ + The model in use by this tokenizer (a copy: reassign to change it). + """ + @model.setter + def model(self, /, model: Model) -> None: ... + @property + def normalizer(self, /) -> Normalizer |None: + """ + The optional normalizer in use by this tokenizer (a copy: reassign to + change it). + """ + @normalizer.setter + def normalizer(self, /, normalizer: Normalizer |None) -> None: ... + @property + def pre_tokenizer(self, /) -> PreTokenizer |None: + """ + The optional pre-tokenizer in use by this tokenizer (a copy: reassign + to change it). + """ + @pre_tokenizer.setter + def pre_tokenizer(self, /, pre_tokenizer: PreTokenizer |None) -> None: ... + def save(self, /, path: str |PathLike[str], *, pretty: bool = True) -> None: + """ + Save the tokenizer definition to a `tokenizer.json` file. + """ + def to_str(self, /, *, pretty: bool = False) -> str: + """ + Serialize the tokenizer definition as a `tokenizer.json` string. + """ + def token_to_id(self, /, token: str) -> int |None: ... + def train(self, /, files: Sequence[str], *, trainer: Trainer |None = None) -> None: + """ + Train the model on text files (one sequence per line). + """ + def train_from_iterator(self, /, iterator: Any, *, trainer: Trainer |None = None) -> None: + """ + Train the model from any iterator of `str`. + + The interpreter lock is only re-acquired to refill an internal buffer + (256 sequences at a time); the training itself runs multi-threaded in + Rust with the lock released. + """ + +def __getattr__(name: str) -> Incomplete: ... + +class TokenizersError(Exception): ... diff --git a/bindings/python-pipeline/py_src/tokenizers_pipeline/models/__init__.py b/bindings/python-pipeline/py_src/tokenizers_pipeline/models/__init__.py new file mode 100644 index 000000000..05d3828ae --- /dev/null +++ b/bindings/python-pipeline/py_src/tokenizers_pipeline/models/__init__.py @@ -0,0 +1,9 @@ +from .._native import models as _models + +Model = _models.Model +BPE = _models.BPE +WordPiece = _models.WordPiece +WordLevel = _models.WordLevel +Unigram = _models.Unigram + +__all__ = ["Model", "BPE", "WordPiece", "WordLevel", "Unigram"] diff --git a/bindings/python-pipeline/py_src/tokenizers_pipeline/models/__init__.pyi b/bindings/python-pipeline/py_src/tokenizers_pipeline/models/__init__.pyi new file mode 100644 index 000000000..c7d80c954 --- /dev/null +++ b/bindings/python-pipeline/py_src/tokenizers_pipeline/models/__init__.pyi @@ -0,0 +1,29 @@ +from typing import final + +@final +class BPE(Model): + def __new__(cls, /, *, unk_token: str |None = None, dropout: float |None = None, fuse_unk: bool = False, byte_fallback: bool = False, ignore_merges: bool = False) -> BPE: ... + @staticmethod + def from_file(vocab: str, merges: str, *, unk_token: str |None = None) -> "BPE": + """ + Load a BPE from the legacy vocab.json + merges.txt format. + """ + +class Model: + """ + Base class for all models. Not constructible from Python; holds the actual + Rust model by value (no sharing with the Tokenizer — assignment copies). + """ + def __repr__(self, /) -> str: ... + +@final +class Unigram(Model): + def __new__(cls, /) -> Unigram: ... + +@final +class WordLevel(Model): + def __new__(cls, /, *, unk_token: str = ...) -> WordLevel: ... + +@final +class WordPiece(Model): + def __new__(cls, /, *, unk_token: str = ..., continuing_subword_prefix: str = ..., max_input_chars_per_word: int = 100) -> WordPiece: ... diff --git a/bindings/python-pipeline/py_src/tokenizers_pipeline/normalizers/__init__.py b/bindings/python-pipeline/py_src/tokenizers_pipeline/normalizers/__init__.py new file mode 100644 index 000000000..03d8469aa --- /dev/null +++ b/bindings/python-pipeline/py_src/tokenizers_pipeline/normalizers/__init__.py @@ -0,0 +1,29 @@ +from .._native import normalizers as _normalizers + +Normalizer = _normalizers.Normalizer +BertNormalizer = _normalizers.BertNormalizer +Lowercase = _normalizers.Lowercase +NFC = _normalizers.NFC +NFD = _normalizers.NFD +NFKC = _normalizers.NFKC +NFKD = _normalizers.NFKD +Prepend = _normalizers.Prepend +Replace = _normalizers.Replace +Sequence = _normalizers.Sequence +Strip = _normalizers.Strip +StripAccents = _normalizers.StripAccents + +__all__ = [ + "Normalizer", + "BertNormalizer", + "Lowercase", + "NFC", + "NFD", + "NFKC", + "NFKD", + "Prepend", + "Replace", + "Sequence", + "Strip", + "StripAccents", +] diff --git a/bindings/python-pipeline/py_src/tokenizers_pipeline/normalizers/__init__.pyi b/bindings/python-pipeline/py_src/tokenizers_pipeline/normalizers/__init__.pyi new file mode 100644 index 000000000..54a493c63 --- /dev/null +++ b/bindings/python-pipeline/py_src/tokenizers_pipeline/normalizers/__init__.pyi @@ -0,0 +1,53 @@ +from collections.abc import Sequence as Sequence2 +from typing import final + +@final +class BertNormalizer(Normalizer): + def __new__(cls, /, *, clean_text: bool = True, handle_chinese_chars: bool = True, strip_accents: bool |None = None, lowercase: bool = True) -> BertNormalizer: ... + +@final +class Lowercase(Normalizer): + def __new__(cls, /) -> Lowercase: ... + +@final +class NFC(Normalizer): + def __new__(cls, /) -> NFC: ... + +@final +class NFD(Normalizer): + def __new__(cls, /) -> NFD: ... + +@final +class NFKC(Normalizer): + def __new__(cls, /) -> NFKC: ... + +@final +class NFKD(Normalizer): + def __new__(cls, /) -> NFKD: ... + +class Normalizer: + """ + Base class for all normalizers. Immutable value: assigning it to a + Tokenizer copies the configuration, there is no shared state. + """ + def __repr__(self, /) -> str: ... + +@final +class Prepend(Normalizer): + def __new__(cls, /, prepend: str) -> Prepend: ... + +@final +class Replace(Normalizer): + def __new__(cls, /, pattern: str, content: str, *, regex: bool = False) -> Replace: ... + +@final +class Sequence(Normalizer): + def __new__(cls, /, normalizers: Sequence2[Normalizer]) -> Sequence: ... + +@final +class Strip(Normalizer): + def __new__(cls, /, *, left: bool = True, right: bool = True) -> Strip: ... + +@final +class StripAccents(Normalizer): + def __new__(cls, /) -> StripAccents: ... diff --git a/bindings/python-pipeline/py_src/tokenizers_pipeline/pre_tokenizers/__init__.py b/bindings/python-pipeline/py_src/tokenizers_pipeline/pre_tokenizers/__init__.py new file mode 100644 index 000000000..1256f852b --- /dev/null +++ b/bindings/python-pipeline/py_src/tokenizers_pipeline/pre_tokenizers/__init__.py @@ -0,0 +1,29 @@ +from .._native import pre_tokenizers as _pre_tokenizers + +PreTokenizer = _pre_tokenizers.PreTokenizer +BertPreTokenizer = _pre_tokenizers.BertPreTokenizer +ByteLevel = _pre_tokenizers.ByteLevel +CharDelimiterSplit = _pre_tokenizers.CharDelimiterSplit +Digits = _pre_tokenizers.Digits +FixedLength = _pre_tokenizers.FixedLength +Punctuation = _pre_tokenizers.Punctuation +Sequence = _pre_tokenizers.Sequence +Split = _pre_tokenizers.Split +UnicodeScripts = _pre_tokenizers.UnicodeScripts +Whitespace = _pre_tokenizers.Whitespace +WhitespaceSplit = _pre_tokenizers.WhitespaceSplit + +__all__ = [ + "PreTokenizer", + "BertPreTokenizer", + "ByteLevel", + "CharDelimiterSplit", + "Digits", + "FixedLength", + "Punctuation", + "Sequence", + "Split", + "UnicodeScripts", + "Whitespace", + "WhitespaceSplit", +] diff --git a/bindings/python-pipeline/py_src/tokenizers_pipeline/pre_tokenizers/__init__.pyi b/bindings/python-pipeline/py_src/tokenizers_pipeline/pre_tokenizers/__init__.pyi new file mode 100644 index 000000000..1e1c31738 --- /dev/null +++ b/bindings/python-pipeline/py_src/tokenizers_pipeline/pre_tokenizers/__init__.pyi @@ -0,0 +1,59 @@ +from collections.abc import Sequence as Sequence2 +from typing import final + +@final +class BertPreTokenizer(PreTokenizer): + def __new__(cls, /) -> BertPreTokenizer: ... + +@final +class ByteLevel(PreTokenizer): + def __new__(cls, /, *, use_regex: bool = True) -> ByteLevel: + """ + `add_prefix_space` is not supported by the pipeline and is always false. + """ + +@final +class CharDelimiterSplit(PreTokenizer): + def __new__(cls, /, delimiter: str) -> CharDelimiterSplit: ... + +@final +class Digits(PreTokenizer): + def __new__(cls, /, *, individual_digits: bool = False) -> Digits: ... + +@final +class FixedLength(PreTokenizer): + def __new__(cls, /, *, length: int = 5) -> FixedLength: ... + +class PreTokenizer: + """ + Base class for all pre-tokenizers. Immutable value: assigning it to a + Tokenizer copies the configuration, there is no shared state. + + Only pre-tokenizers supported by the encode pipeline are constructible here; + notably `Metaspace` is not available yet. + """ + def __repr__(self, /) -> str: ... + +@final +class Punctuation(PreTokenizer): + def __new__(cls, /, behavior: str = ...) -> Punctuation: ... + +@final +class Sequence(PreTokenizer): + def __new__(cls, /, pre_tokenizers: Sequence2[PreTokenizer]) -> Sequence: ... + +@final +class Split(PreTokenizer): + def __new__(cls, /, pattern: str, behavior: str = ..., *, invert: bool = False, regex: bool = False) -> Split: ... + +@final +class UnicodeScripts(PreTokenizer): + def __new__(cls, /) -> UnicodeScripts: ... + +@final +class Whitespace(PreTokenizer): + def __new__(cls, /) -> Whitespace: ... + +@final +class WhitespaceSplit(PreTokenizer): + def __new__(cls, /) -> WhitespaceSplit: ... diff --git a/bindings/python-pipeline/py_src/tokenizers_pipeline/py.typed b/bindings/python-pipeline/py_src/tokenizers_pipeline/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/bindings/python-pipeline/py_src/tokenizers_pipeline/trainers/__init__.py b/bindings/python-pipeline/py_src/tokenizers_pipeline/trainers/__init__.py new file mode 100644 index 000000000..c90414b9b --- /dev/null +++ b/bindings/python-pipeline/py_src/tokenizers_pipeline/trainers/__init__.py @@ -0,0 +1,15 @@ +from .._native import trainers as _trainers + +Trainer = _trainers.Trainer +BpeTrainer = _trainers.BpeTrainer +UnigramTrainer = _trainers.UnigramTrainer +WordLevelTrainer = _trainers.WordLevelTrainer +WordPieceTrainer = _trainers.WordPieceTrainer + +__all__ = [ + "Trainer", + "BpeTrainer", + "UnigramTrainer", + "WordLevelTrainer", + "WordPieceTrainer", +] diff --git a/bindings/python-pipeline/py_src/tokenizers_pipeline/trainers/__init__.pyi b/bindings/python-pipeline/py_src/tokenizers_pipeline/trainers/__init__.pyi new file mode 100644 index 000000000..a93e342b9 --- /dev/null +++ b/bindings/python-pipeline/py_src/tokenizers_pipeline/trainers/__init__.pyi @@ -0,0 +1,26 @@ +from tokenizers_pipeline import AddedToken +from collections.abc import Sequence +from typing import final + +@final +class BpeTrainer(Trainer): + def __new__(cls, /, *, vocab_size: int = 30000, min_frequency: int = 0, special_tokens: Sequence[str |AddedToken] = ..., limit_alphabet: int |None = None, initial_alphabet: Sequence[str] = ..., continuing_subword_prefix: str |None = None, end_of_word_suffix: str |None = None, max_token_length: int |None = None, show_progress: bool = True) -> BpeTrainer: ... + +class Trainer: + """ + Base class for all trainers. A trainer is a plain configuration value: + `Tokenizer.train*` copies it, no state is shared or written back. + """ + def __repr__(self, /) -> str: ... + +@final +class UnigramTrainer(Trainer): + def __new__(cls, /, *, vocab_size: int = 8000, special_tokens: Sequence[str |AddedToken] = ..., initial_alphabet: Sequence[str] = ..., unk_token: str |None = None, shrinking_factor: float = 0.75, max_piece_length: int = 16, n_sub_iterations: int = 2, show_progress: bool = True) -> UnigramTrainer: ... + +@final +class WordLevelTrainer(Trainer): + def __new__(cls, /, *, vocab_size: int = 30000, min_frequency: int = 0, special_tokens: Sequence[str |AddedToken] = ..., show_progress: bool = True) -> WordLevelTrainer: ... + +@final +class WordPieceTrainer(Trainer): + def __new__(cls, /, *, vocab_size: int = 30000, min_frequency: int = 0, special_tokens: Sequence[str |AddedToken] = ..., limit_alphabet: int |None = None, initial_alphabet: Sequence[str] = ..., continuing_subword_prefix: str = ..., end_of_word_suffix: str |None = None, show_progress: bool = True) -> WordPieceTrainer: ... diff --git a/bindings/python-pipeline/pyproject.toml b/bindings/python-pipeline/pyproject.toml new file mode 100644 index 000000000..9bce55d1b --- /dev/null +++ b/bindings/python-pipeline/pyproject.toml @@ -0,0 +1,19 @@ +[build-system] +requires = ["maturin>=1.5,<2.0"] +build-backend = "maturin" + +[project] +name = "tokenizers-pipeline" +description = "Experimental Python bindings for the tokenizers PipelineTokenizer" +requires-python = ">=3.10" +dependencies = ["numpy>=1.24"] +dynamic = ["version"] + +[project.optional-dependencies] +hub = ["huggingface_hub>=0.16.4"] + +[tool.maturin] +python-source = "py_src" +module-name = "tokenizers_pipeline._native" +bindings = "pyo3" +features = ["ext-module"] diff --git a/bindings/python-pipeline/src/added_token.rs b/bindings/python-pipeline/src/added_token.rs new file mode 100644 index 000000000..be60c109d --- /dev/null +++ b/bindings/python-pipeline/src/added_token.rs @@ -0,0 +1,102 @@ +use pyo3::prelude::*; +use tk_encode::tokenizer::AddedToken; + +/// A token added on top of the model's vocabulary, with its matching options. +#[pyclass( + frozen, + from_py_object, + name = "AddedToken", + module = "tokenizers_pipeline" +)] +#[derive(Clone)] +pub struct PyAddedToken { + pub inner: AddedToken, +} + +#[pymethods] +impl PyAddedToken { + #[new] + #[pyo3(signature = (content, *, single_word = false, lstrip = false, rstrip = false, normalized = None, special = false))] + fn new( + content: String, + single_word: bool, + lstrip: bool, + rstrip: bool, + normalized: Option, + special: bool, + ) -> Self { + let inner = AddedToken::from(content, special) + .single_word(single_word) + .lstrip(lstrip) + .rstrip(rstrip) + .normalized(normalized.unwrap_or(!special)); + Self { inner } + } + + #[getter] + fn content(&self) -> &str { + &self.inner.content + } + + #[getter] + fn single_word(&self) -> bool { + self.inner.single_word + } + + #[getter] + fn lstrip(&self) -> bool { + self.inner.lstrip + } + + #[getter] + fn rstrip(&self) -> bool { + self.inner.rstrip + } + + #[getter] + fn normalized(&self) -> bool { + self.inner.normalized + } + + #[getter] + fn special(&self) -> bool { + self.inner.special + } + + fn __repr__(&self) -> String { + format!( + "AddedToken({:?}, single_word={}, lstrip={}, rstrip={}, normalized={}, special={})", + self.inner.content, + self.inner.single_word, + self.inner.lstrip, + self.inner.rstrip, + self.inner.normalized, + self.inner.special + ) + } +} + +/// A `str | AddedToken` argument. +#[derive(FromPyObject)] +pub enum TokenInput { + Str(String), + Token(PyAddedToken), +} + +/// Plain strings become tokens with `special=special_default` (and +/// `normalized=!special_default`, matching v1). +pub fn parse_tokens(items: Vec, special_default: bool) -> Vec { + items + .into_iter() + .map(|item| match item { + TokenInput::Str(content) => AddedToken::from(content, special_default), + TokenInput::Token(token) => { + let mut inner = token.inner; + if special_default { + inner.special = true; + } + inner + } + }) + .collect() +} diff --git a/bindings/python-pipeline/src/detached_lock.rs b/bindings/python-pipeline/src/detached_lock.rs new file mode 100644 index 000000000..e16883aa3 --- /dev/null +++ b/bindings/python-pipeline/src/detached_lock.rs @@ -0,0 +1,66 @@ +//! Compile-time enforcement of the crate's lock/GIL ordering. +//! +//! The rule: **never block on the tokenizer lock while attached to the +//! interpreter**. Training holds the write lock for its whole run and +//! re-attaches to refill from the Python iterator (lock → GIL); an attached +//! thread blocking on the lock would take GIL → lock, closing a deadlock +//! cycle. +//! +//! [`DetachedRwLock`] turns that rule from a review checklist into a type +//! property: the raw `RwLock` is private to this module, and the only way to +//! reach a lock guard is [`DetachedRwLock::with`], which detaches first. Code +//! that forgets to detach has no method to call. +//! +//! Residual hole (not expressible on stable Rust): re-attaching *inside* +//! `with` and locking from there. `clippy.toml` bans `Python::attach` +//! crate-wide as a backstop; the single vetted use is the training iterator +//! refill, which holds the lock and *then* attaches — the sanctioned +//! direction. + +#![allow( + clippy::disallowed_types, + reason = "the one raw RwLock the wrapper encapsulates" +)] + +use std::sync::{PoisonError, RwLock, RwLockReadGuard, RwLockWriteGuard}; + +use pyo3::Python; +use pyo3::marker::Ungil; + +pub struct DetachedRwLock { + inner: RwLock, +} + +/// Proof of detachment: only constructed by [`DetachedRwLock::with`], never +/// clonable or storable beyond the closure (its lifetime is bound to the +/// borrow of the lock inside `with`). +pub struct Detached<'a, T> { + lock: &'a RwLock, +} + +impl DetachedRwLock { + pub fn new(value: T) -> Self { + Self { + inner: RwLock::new(value), + } + } + + /// Detach from the interpreter, then run `f` with lock access. + pub fn with(&self, py: Python<'_>, f: F) -> R + where + F: for<'a> FnOnce(Detached<'a, T>) -> R + Ungil + Send, + R: Ungil + Send, + { + py.detach(|| f(Detached { lock: &self.inner })) + } +} + +impl<'a, T> Detached<'a, T> { + pub fn read(&self) -> Result, PoisonError>> { + self.lock.read() + } + + pub fn write(&self) -> Result, PoisonError>> { + self.lock.write() + } +} diff --git a/bindings/python-pipeline/src/error.rs b/bindings/python-pipeline/src/error.rs new file mode 100644 index 000000000..ccd5f7db8 --- /dev/null +++ b/bindings/python-pipeline/src/error.rs @@ -0,0 +1,9 @@ +use pyo3::PyErr; +use pyo3::create_exception; +use pyo3::exceptions::PyException; + +create_exception!(tokenizers_pipeline, TokenizersError, PyException); + +pub fn to_pyerr(e: tk_encode::Error) -> PyErr { + TokenizersError::new_err(e.to_string()) +} diff --git a/bindings/python-pipeline/src/lib.rs b/bindings/python-pipeline/src/lib.rs new file mode 100644 index 000000000..eb230591b --- /dev/null +++ b/bindings/python-pipeline/src/lib.rs @@ -0,0 +1,68 @@ +#![warn(clippy::all)] + +pub mod added_token; +pub mod detached_lock; +pub mod error; +pub mod models; +pub mod normalizers; +pub mod pre_tokenizers; +pub mod tokenizer; +pub mod trainers; + +use pyo3::prelude::*; + +/// Components repr as their tokenizer.json serialization: compact, and always +/// in sync with what `Tokenizer.save` writes. +pub fn component_repr(component: &T) -> String { + serde_json::to_string(component).unwrap_or_else(|_| "".to_owned()) +} + +// Forked children of a process that used our rayon threads would inherit a +// poisoned thread pool; disable parallelism there unless the user configured +// it explicitly (same behavior as the v1 bindings). +#[cfg(target_family = "unix")] +extern "C" fn child_after_fork() { + use std::sync::atomic::Ordering; + use tk_encode::utils::parallelism::{is_parallelism_configured, set_parallelism}; + if crate::tokenizer::USED_PARALLELISM.load(Ordering::SeqCst) && !is_parallelism_configured() { + set_parallelism(false); + } +} + +#[pymodule(gil_used = false)] +pub mod _native { + use super::*; + + #[pymodule_export] + pub use super::added_token::PyAddedToken; + #[pymodule_export] + pub use super::error::TokenizersError; + #[pymodule_export] + pub use super::tokenizer::PyTokenizer; + + #[pymodule_export] + pub use super::models::models; + #[pymodule_export] + pub use super::normalizers::normalizers; + #[pymodule_export] + pub use super::pre_tokenizers::pre_tokenizers; + #[pymodule_export] + pub use super::trainers::trainers; + + #[allow(non_upper_case_globals)] + #[pymodule_export] + pub const __version__: &str = env!("CARGO_PKG_VERSION"); + + #[pymodule_init] + fn init(_m: &Bound<'_, PyModule>) -> PyResult<()> { + #[cfg(target_family = "unix")] + { + use std::sync::Once; + static REGISTER_FORK_CALLBACK: Once = Once::new(); + REGISTER_FORK_CALLBACK.call_once(|| unsafe { + libc::pthread_atfork(None, None, Some(child_after_fork)); + }); + } + Ok(()) + } +} diff --git a/bindings/python-pipeline/src/models.rs b/bindings/python-pipeline/src/models.rs new file mode 100644 index 000000000..911419037 --- /dev/null +++ b/bindings/python-pipeline/src/models.rs @@ -0,0 +1,145 @@ +use pyo3::prelude::*; +use tk_encode::models::ModelWrapper; +use tk_encode::models::bpe::BPE; +use tk_encode::models::unigram::Unigram; +use tk_encode::models::wordlevel::WordLevel; +use tk_encode::models::wordpiece::WordPiece; + +use crate::error::to_pyerr; + +/// Base class for all models. Not constructible from Python; holds the actual +/// Rust model by value (no sharing with the Tokenizer — assignment copies). +#[pyclass( + frozen, + subclass, + name = "Model", + module = "tokenizers_pipeline.models" +)] +pub struct PyModel { + pub inner: ModelWrapper, +} + +#[pymethods] +impl PyModel { + fn __repr__(&self) -> String { + crate::component_repr(&self.inner) + } +} + +pub fn wrap_model(py: Python<'_>, inner: ModelWrapper) -> PyResult> { + let base = PyModel { + inner: inner.clone(), + }; + let init = PyClassInitializer::from(base); + let obj = match inner { + ModelWrapper::BPE(_) => Bound::new(py, init.add_subclass(PyBPE))?.into_super(), + ModelWrapper::WordPiece(_) => Bound::new(py, init.add_subclass(PyWordPiece))?.into_super(), + ModelWrapper::WordLevel(_) => Bound::new(py, init.add_subclass(PyWordLevel))?.into_super(), + ModelWrapper::Unigram(_) => Bound::new(py, init.add_subclass(PyUnigram))?.into_super(), + }; + Ok(obj.unbind()) +} + +#[pyclass(frozen, extends = PyModel, name = "BPE", module = "tokenizers_pipeline.models")] +pub struct PyBPE; + +#[pymethods] +impl PyBPE { + #[new] + #[pyo3(signature = (*, unk_token = None, dropout = None, fuse_unk = false, byte_fallback = false, ignore_merges = false))] + fn new( + unk_token: Option, + dropout: Option, + fuse_unk: bool, + byte_fallback: bool, + ignore_merges: bool, + ) -> PyResult> { + let mut builder = BPE::builder() + .fuse_unk(fuse_unk) + .byte_fallback(byte_fallback) + .ignore_merges(ignore_merges); + if let Some(unk) = unk_token { + builder = builder.unk_token(unk); + } + if let Some(d) = dropout { + builder = builder.dropout(d); + } + let bpe = builder.build().map_err(to_pyerr)?; + Ok(PyClassInitializer::from(PyModel { inner: bpe.into() }).add_subclass(PyBPE)) + } + + /// Load a BPE from the legacy vocab.json + merges.txt format. + #[staticmethod] + #[pyo3(signature = (vocab, merges, *, unk_token = None) -> "BPE")] + fn from_file( + py: Python<'_>, + vocab: &str, + merges: &str, + unk_token: Option, + ) -> PyResult> { + let mut builder = BPE::from_file(vocab, merges); + if let Some(unk) = unk_token { + builder = builder.unk_token(unk); + } + let bpe = py.detach(|| builder.build()).map_err(to_pyerr)?; + wrap_model(py, bpe.into()) + } +} + +#[pyclass(frozen, extends = PyModel, name = "WordPiece", module = "tokenizers_pipeline.models")] +pub struct PyWordPiece; + +#[pymethods] +impl PyWordPiece { + #[new] + #[pyo3(signature = (*, unk_token = String::from("[UNK]"), continuing_subword_prefix = String::from("##"), max_input_chars_per_word = 100))] + fn new( + unk_token: String, + continuing_subword_prefix: String, + max_input_chars_per_word: usize, + ) -> PyResult> { + let wp = WordPiece::builder() + .unk_token(unk_token) + .continuing_subword_prefix(continuing_subword_prefix) + .max_input_chars_per_word(max_input_chars_per_word) + .build() + .map_err(to_pyerr)?; + Ok(PyClassInitializer::from(PyModel { inner: wp.into() }).add_subclass(PyWordPiece)) + } +} + +#[pyclass(frozen, extends = PyModel, name = "WordLevel", module = "tokenizers_pipeline.models")] +pub struct PyWordLevel; + +#[pymethods] +impl PyWordLevel { + #[new] + #[pyo3(signature = (*, unk_token = String::from("[UNK]")))] + fn new(unk_token: String) -> PyResult> { + let wl = WordLevel::builder() + .unk_token(unk_token) + .build() + .map_err(to_pyerr)?; + Ok(PyClassInitializer::from(PyModel { inner: wl.into() }).add_subclass(PyWordLevel)) + } +} + +#[pyclass(frozen, extends = PyModel, name = "Unigram", module = "tokenizers_pipeline.models")] +pub struct PyUnigram; + +#[pymethods] +impl PyUnigram { + #[new] + fn new() -> PyClassInitializer { + PyClassInitializer::from(PyModel { + inner: Unigram::default().into(), + }) + .add_subclass(PyUnigram) + } +} + +#[pymodule(gil_used = false)] +pub mod models { + #[pymodule_export] + pub use super::{PyBPE, PyModel, PyUnigram, PyWordLevel, PyWordPiece}; +} diff --git a/bindings/python-pipeline/src/normalizers.rs b/bindings/python-pipeline/src/normalizers.rs new file mode 100644 index 000000000..c0a6e3243 --- /dev/null +++ b/bindings/python-pipeline/src/normalizers.rs @@ -0,0 +1,179 @@ +use pyo3::prelude::*; +use tk_encode::normalizers::{ + BertNormalizer, Lowercase, NFC, NFD, NFKC, NFKD, NormalizerWrapper, Prepend, Replace, Sequence, + Strip, StripAccents, +}; + +use crate::error::to_pyerr; + +/// Base class for all normalizers. Immutable value: assigning it to a +/// Tokenizer copies the configuration, there is no shared state. +#[pyclass( + frozen, + subclass, + name = "Normalizer", + module = "tokenizers_pipeline.normalizers" +)] +pub struct PyNormalizer { + pub inner: NormalizerWrapper, +} + +#[pymethods] +impl PyNormalizer { + fn __repr__(&self) -> String { + crate::component_repr(&self.inner) + } +} + +pub fn wrap_normalizer(py: Python<'_>, inner: NormalizerWrapper) -> PyResult> { + let base = PyNormalizer { + inner: inner.clone(), + }; + let init = PyClassInitializer::from(base); + let obj = match inner { + NormalizerWrapper::BertNormalizer(_) => { + Bound::new(py, init.add_subclass(PyBertNormalizer))?.into_super() + } + NormalizerWrapper::StripNormalizer(_) => { + Bound::new(py, init.add_subclass(PyStrip))?.into_super() + } + NormalizerWrapper::StripAccents(_) => { + Bound::new(py, init.add_subclass(PyStripAccents))?.into_super() + } + NormalizerWrapper::NFC(_) => Bound::new(py, init.add_subclass(PyNFC))?.into_super(), + NormalizerWrapper::NFD(_) => Bound::new(py, init.add_subclass(PyNFD))?.into_super(), + NormalizerWrapper::NFKC(_) => Bound::new(py, init.add_subclass(PyNFKC))?.into_super(), + NormalizerWrapper::NFKD(_) => Bound::new(py, init.add_subclass(PyNFKD))?.into_super(), + NormalizerWrapper::Sequence(_) => { + Bound::new(py, init.add_subclass(PySequence))?.into_super() + } + NormalizerWrapper::Lowercase(_) => { + Bound::new(py, init.add_subclass(PyLowercase))?.into_super() + } + NormalizerWrapper::Replace(_) => Bound::new(py, init.add_subclass(PyReplace))?.into_super(), + NormalizerWrapper::Prepend(_) => Bound::new(py, init.add_subclass(PyPrepend))?.into_super(), + // Loadable from tokenizer.json but not constructible from Python: exposed as the base class. + NormalizerWrapper::Nmt(_) + | NormalizerWrapper::Precompiled(_) + | NormalizerWrapper::ByteLevel(_) => Bound::new(py, init)?, + }; + Ok(obj.unbind()) +} + +macro_rules! unit_normalizer { + ($pyname:ident, $name:literal, $inner:expr) => { + #[pyclass(frozen, extends = PyNormalizer, name = $name, module = "tokenizers_pipeline.normalizers")] + pub struct $pyname; + + #[pymethods] + impl $pyname { + #[new] + fn new() -> PyClassInitializer { + PyClassInitializer::from(PyNormalizer { inner: $inner.into() }).add_subclass($pyname) + } + } + }; +} + +unit_normalizer!(PyNFC, "NFC", NFC); +unit_normalizer!(PyNFD, "NFD", NFD); +unit_normalizer!(PyNFKC, "NFKC", NFKC); +unit_normalizer!(PyNFKD, "NFKD", NFKD); +unit_normalizer!(PyLowercase, "Lowercase", Lowercase); +unit_normalizer!(PyStripAccents, "StripAccents", StripAccents); + +#[pyclass(frozen, extends = PyNormalizer, name = "Strip", module = "tokenizers_pipeline.normalizers")] +pub struct PyStrip; + +#[pymethods] +impl PyStrip { + #[new] + #[pyo3(signature = (*, left = true, right = true))] + fn new(left: bool, right: bool) -> PyClassInitializer { + PyClassInitializer::from(PyNormalizer { + inner: Strip::new(left, right).into(), + }) + .add_subclass(PyStrip) + } +} + +#[pyclass(frozen, extends = PyNormalizer, name = "Replace", module = "tokenizers_pipeline.normalizers")] +pub struct PyReplace; + +#[pymethods] +impl PyReplace { + #[new] + #[pyo3(signature = (pattern, content, *, regex = false))] + fn new(pattern: &str, content: &str, regex: bool) -> PyResult> { + use tk_encode::normalizers::replace::ReplacePattern; + let pattern = if regex { + ReplacePattern::Regex(pattern.to_owned()) + } else { + ReplacePattern::String(pattern.to_owned()) + }; + let replace = Replace::new(pattern, content).map_err(to_pyerr)?; + Ok(PyClassInitializer::from(PyNormalizer { + inner: replace.into(), + }) + .add_subclass(PyReplace)) + } +} + +#[pyclass(frozen, extends = PyNormalizer, name = "Prepend", module = "tokenizers_pipeline.normalizers")] +pub struct PyPrepend; + +#[pymethods] +impl PyPrepend { + #[new] + fn new(prepend: String) -> PyClassInitializer { + PyClassInitializer::from(PyNormalizer { + inner: Prepend::new(prepend).into(), + }) + .add_subclass(PyPrepend) + } +} + +#[pyclass(frozen, extends = PyNormalizer, name = "BertNormalizer", module = "tokenizers_pipeline.normalizers")] +pub struct PyBertNormalizer; + +#[pymethods] +impl PyBertNormalizer { + #[new] + #[pyo3(signature = (*, clean_text = true, handle_chinese_chars = true, strip_accents = None, lowercase = true))] + fn new( + clean_text: bool, + handle_chinese_chars: bool, + strip_accents: Option, + lowercase: bool, + ) -> PyClassInitializer { + let inner = BertNormalizer::new(clean_text, handle_chinese_chars, strip_accents, lowercase); + PyClassInitializer::from(PyNormalizer { + inner: inner.into(), + }) + .add_subclass(PyBertNormalizer) + } +} + +#[pyclass(frozen, extends = PyNormalizer, name = "Sequence", module = "tokenizers_pipeline.normalizers")] +pub struct PySequence; + +#[pymethods] +impl PySequence { + #[new] + fn new(normalizers: Vec>) -> PyClassInitializer { + let inner: Vec = normalizers.iter().map(|n| n.inner.clone()).collect(); + PyClassInitializer::from(PyNormalizer { + inner: Sequence::new(inner).into(), + }) + .add_subclass(PySequence) + } +} + +#[pymodule(gil_used = false)] +pub mod normalizers { + #[pymodule_export] + pub use super::{ + PyBertNormalizer, PyLowercase, PyNFC, PyNFD, PyNFKC, PyNFKD, PyNormalizer, PyPrepend, + PyReplace, PySequence, PyStrip, PyStripAccents, + }; +} diff --git a/bindings/python-pipeline/src/pre_tokenizers.rs b/bindings/python-pipeline/src/pre_tokenizers.rs new file mode 100644 index 000000000..010d4bda9 --- /dev/null +++ b/bindings/python-pipeline/src/pre_tokenizers.rs @@ -0,0 +1,244 @@ +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use tk_encode::pre_tokenizers::PreTokenizerWrapper; +use tk_encode::pre_tokenizers::bert::BertPreTokenizer; +use tk_encode::pre_tokenizers::byte_level::ByteLevel; +use tk_encode::pre_tokenizers::delimiter::CharDelimiterSplit; +use tk_encode::pre_tokenizers::digits::Digits; +use tk_encode::pre_tokenizers::fixed_length::FixedLength; +use tk_encode::pre_tokenizers::punctuation::Punctuation; +use tk_encode::pre_tokenizers::sequence::Sequence; +use tk_encode::pre_tokenizers::split::{Split, SplitPattern}; +use tk_encode::pre_tokenizers::unicode_scripts::UnicodeScripts; +use tk_encode::pre_tokenizers::whitespace::{Whitespace, WhitespaceSplit}; +use tk_encode::tokenizer::SplitDelimiterBehavior; + +use crate::error::to_pyerr; + +pub fn parse_behavior(s: &str) -> PyResult { + match s { + "removed" => Ok(SplitDelimiterBehavior::Removed), + "isolated" => Ok(SplitDelimiterBehavior::Isolated), + "merged_with_previous" => Ok(SplitDelimiterBehavior::MergedWithPrevious), + "merged_with_next" => Ok(SplitDelimiterBehavior::MergedWithNext), + "contiguous" => Ok(SplitDelimiterBehavior::Contiguous), + other => Err(PyValueError::new_err(format!( + "unknown behavior {other:?}; expected one of: removed, isolated, \ + merged_with_previous, merged_with_next, contiguous" + ))), + } +} + +/// Base class for all pre-tokenizers. Immutable value: assigning it to a +/// Tokenizer copies the configuration, there is no shared state. +/// +/// Only pre-tokenizers supported by the encode pipeline are constructible here; +/// notably `Metaspace` is not available yet. +#[pyclass( + frozen, + subclass, + name = "PreTokenizer", + module = "tokenizers_pipeline.pre_tokenizers" +)] +pub struct PyPreTokenizer { + pub inner: PreTokenizerWrapper, +} + +#[pymethods] +impl PyPreTokenizer { + fn __repr__(&self) -> String { + crate::component_repr(&self.inner) + } +} + +pub fn wrap_pre_tokenizer( + py: Python<'_>, + inner: PreTokenizerWrapper, +) -> PyResult> { + let base = PyPreTokenizer { + inner: inner.clone(), + }; + let init = PyClassInitializer::from(base); + let obj = match inner { + PreTokenizerWrapper::BertPreTokenizer(_) => { + Bound::new(py, init.add_subclass(PyBertPreTokenizer))?.into_super() + } + PreTokenizerWrapper::ByteLevel(_) => { + Bound::new(py, init.add_subclass(PyByteLevel))?.into_super() + } + PreTokenizerWrapper::Delimiter(_) => { + Bound::new(py, init.add_subclass(PyCharDelimiterSplit))?.into_super() + } + PreTokenizerWrapper::Whitespace(_) => { + Bound::new(py, init.add_subclass(PyWhitespace))?.into_super() + } + PreTokenizerWrapper::WhitespaceSplit(_) => { + Bound::new(py, init.add_subclass(PyWhitespaceSplit))?.into_super() + } + PreTokenizerWrapper::Sequence(_) => { + Bound::new(py, init.add_subclass(PySequence))?.into_super() + } + PreTokenizerWrapper::Split(_) => Bound::new(py, init.add_subclass(PySplit))?.into_super(), + PreTokenizerWrapper::Punctuation(_) => { + Bound::new(py, init.add_subclass(PyPunctuation))?.into_super() + } + PreTokenizerWrapper::Digits(_) => Bound::new(py, init.add_subclass(PyDigits))?.into_super(), + PreTokenizerWrapper::UnicodeScripts(_) => { + Bound::new(py, init.add_subclass(PyUnicodeScripts))?.into_super() + } + PreTokenizerWrapper::FixedLength(_) => { + Bound::new(py, init.add_subclass(PyFixedLength))?.into_super() + } + // Loadable from tokenizer.json but not constructible from Python (and + // rejected by the pipeline at compile time): exposed as the base class. + PreTokenizerWrapper::Metaspace(_) => Bound::new(py, init)?, + }; + Ok(obj.unbind()) +} + +macro_rules! unit_pre_tokenizer { + ($pyname:ident, $name:literal, $inner:expr) => { + #[pyclass(frozen, extends = PyPreTokenizer, name = $name, module = "tokenizers_pipeline.pre_tokenizers")] + pub struct $pyname; + + #[pymethods] + impl $pyname { + #[new] + fn new() -> PyClassInitializer { + PyClassInitializer::from(PyPreTokenizer { inner: $inner.into() }).add_subclass($pyname) + } + } + }; +} + +unit_pre_tokenizer!(PyWhitespace, "Whitespace", Whitespace); +unit_pre_tokenizer!(PyWhitespaceSplit, "WhitespaceSplit", WhitespaceSplit); +unit_pre_tokenizer!(PyBertPreTokenizer, "BertPreTokenizer", BertPreTokenizer); +unit_pre_tokenizer!(PyUnicodeScripts, "UnicodeScripts", UnicodeScripts); + +#[pyclass(frozen, extends = PyPreTokenizer, name = "ByteLevel", module = "tokenizers_pipeline.pre_tokenizers")] +pub struct PyByteLevel; + +#[pymethods] +impl PyByteLevel { + /// `add_prefix_space` is not supported by the pipeline and is always false. + #[new] + #[pyo3(signature = (*, use_regex = true))] + fn new(use_regex: bool) -> PyClassInitializer { + PyClassInitializer::from(PyPreTokenizer { + inner: ByteLevel::new(false, true, use_regex).into(), + }) + .add_subclass(PyByteLevel) + } +} + +#[pyclass(frozen, extends = PyPreTokenizer, name = "CharDelimiterSplit", module = "tokenizers_pipeline.pre_tokenizers")] +pub struct PyCharDelimiterSplit; + +#[pymethods] +impl PyCharDelimiterSplit { + #[new] + fn new(delimiter: char) -> PyClassInitializer { + PyClassInitializer::from(PyPreTokenizer { + inner: CharDelimiterSplit::new(delimiter).into(), + }) + .add_subclass(PyCharDelimiterSplit) + } +} + +#[pyclass(frozen, extends = PyPreTokenizer, name = "Digits", module = "tokenizers_pipeline.pre_tokenizers")] +pub struct PyDigits; + +#[pymethods] +impl PyDigits { + #[new] + #[pyo3(signature = (*, individual_digits = false))] + fn new(individual_digits: bool) -> PyClassInitializer { + PyClassInitializer::from(PyPreTokenizer { + inner: Digits::new(individual_digits).into(), + }) + .add_subclass(PyDigits) + } +} + +#[pyclass(frozen, extends = PyPreTokenizer, name = "FixedLength", module = "tokenizers_pipeline.pre_tokenizers")] +pub struct PyFixedLength; + +#[pymethods] +impl PyFixedLength { + #[new] + #[pyo3(signature = (*, length = 5))] + fn new(length: usize) -> PyClassInitializer { + PyClassInitializer::from(PyPreTokenizer { + inner: FixedLength::new(length).into(), + }) + .add_subclass(PyFixedLength) + } +} + +#[pyclass(frozen, extends = PyPreTokenizer, name = "Punctuation", module = "tokenizers_pipeline.pre_tokenizers")] +pub struct PyPunctuation; + +#[pymethods] +impl PyPunctuation { + #[new] + #[pyo3(signature = (behavior = String::from("isolated")))] + fn new(behavior: String) -> PyResult> { + Ok(PyClassInitializer::from(PyPreTokenizer { + inner: Punctuation::new(parse_behavior(&behavior)?).into(), + }) + .add_subclass(PyPunctuation)) + } +} + +#[pyclass(frozen, extends = PyPreTokenizer, name = "Split", module = "tokenizers_pipeline.pre_tokenizers")] +pub struct PySplit; + +#[pymethods] +impl PySplit { + #[new] + #[pyo3(signature = (pattern, behavior = String::from("isolated"), *, invert = false, regex = false))] + fn new( + pattern: &str, + behavior: String, + invert: bool, + regex: bool, + ) -> PyResult> { + let pattern = if regex { + SplitPattern::Regex(pattern.to_owned()) + } else { + SplitPattern::String(pattern.to_owned()) + }; + let split = Split::new(pattern, parse_behavior(&behavior)?, invert).map_err(to_pyerr)?; + Ok(PyClassInitializer::from(PyPreTokenizer { + inner: split.into(), + }) + .add_subclass(PySplit)) + } +} + +#[pyclass(frozen, extends = PyPreTokenizer, name = "Sequence", module = "tokenizers_pipeline.pre_tokenizers")] +pub struct PySequence; + +#[pymethods] +impl PySequence { + #[new] + fn new(pre_tokenizers: Vec>) -> PyClassInitializer { + let inner: Vec = + pre_tokenizers.iter().map(|p| p.inner.clone()).collect(); + PyClassInitializer::from(PyPreTokenizer { + inner: Sequence::new(inner).into(), + }) + .add_subclass(PySequence) + } +} + +#[pymodule(gil_used = false)] +pub mod pre_tokenizers { + #[pymodule_export] + pub use super::{ + PyBertPreTokenizer, PyByteLevel, PyCharDelimiterSplit, PyDigits, PyFixedLength, + PyPreTokenizer, PyPunctuation, PySequence, PySplit, PyUnicodeScripts, PyWhitespace, + PyWhitespaceSplit, + }; +} diff --git a/bindings/python-pipeline/src/tokenizer.rs b/bindings/python-pipeline/src/tokenizer.rs new file mode 100644 index 000000000..437452166 --- /dev/null +++ b/bindings/python-pipeline/src/tokenizer.rs @@ -0,0 +1,526 @@ +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +use numpy::{IntoPyArray, PyArray1}; +use pyo3::exceptions::{PyNotImplementedError, PyRuntimeError, PyStopIteration, PyTypeError}; +use pyo3::marker::Ungil; +use pyo3::prelude::*; +use pyo3::pybacked::PyBackedStr; +use pyo3::types::{PyBytes, PyList}; +use rayon::prelude::*; +use tk_encode::Tokenizer as SpecTokenizer; +use tk_encode::pipeline::{ + Model as _, PipelineModelScratch, PipelineToken, PipelineTokenizer, Span, +}; +use tk_encode::tokenizer::PostProcessor as _; +use tk_encode::utils::parallelism::get_parallelism; +use tk_train::{TokenizerTrainExt, Trainable}; + +use crate::added_token::{TokenInput, parse_tokens}; +use crate::detached_lock::{Detached, DetachedRwLock}; +use crate::error::{TokenizersError, to_pyerr}; +use crate::models::{PyModel, wrap_model}; +use crate::normalizers::{PyNormalizer, wrap_normalizer}; +use crate::pre_tokenizers::{PyPreTokenizer, wrap_pre_tokenizer}; +use crate::trainers::PyTrainer; + +/// Set when the bindings actually run a rayon-parallel section, so the +/// pthread_atfork handler only disables parallelism in children of processes +/// that really used it (mirrors the v1 bindings' semantics). +pub static USED_PARALLELISM: AtomicBool = AtomicBool::new(false); + +/// The compiled encode path plus the facts about the spec the encode calls +/// need without re-locking it. +#[derive(Clone)] +struct Compiled { + pipe: Arc, + /// Whether the spec's post-processor would add special tokens. Post-processing + /// is not wired into the pipeline yet, so encode(add_special_tokens=True) + /// must fail loudly instead of silently dropping them. + post_adds_special_tokens: bool, +} + +struct Inner { + /// Source of truth: the mutable, serializable tokenizer definition. + spec: SpecTokenizer, + /// Memoized compilation of `spec`; invalidated by every mutation. + compiled: Option, +} + +fn poisoned(_: std::sync::PoisonError) -> PyErr { + PyRuntimeError::new_err("tokenizer lock poisoned") +} + +/// A tokenizer: a model plus its optional normalizer and pre-tokenizer. +/// Mutations apply to the serializable definition; encode runs a compiled +/// pipeline that is rebuilt automatically after any change. +// The lock/GIL ordering rule (never block on the lock while attached) is +// enforced by DetachedRwLock: guards are only reachable inside its +// detach-first `with` closure. See detached_lock.rs for the rationale and +// the residual hole. +#[pyclass(frozen, name = "Tokenizer", module = "tokenizers_pipeline")] +pub struct PyTokenizer { + inner: DetachedRwLock, +} + +impl PyTokenizer { + fn from_spec(spec: SpecTokenizer) -> Self { + Self { + inner: DetachedRwLock::new(Inner { + spec, + compiled: None, + }), + } + } + + fn read_spec( + &self, + py: Python<'_>, + f: impl FnOnce(&SpecTokenizer) -> T + Ungil + Send, + ) -> PyResult { + self.inner.with(py, |lock| { + let guard = lock.read().map_err(poisoned)?; + Ok(f(&guard.spec)) + }) + } + + /// Write access to the spec; invalidates the compiled pipeline. + fn mutate_spec( + &self, + py: Python<'_>, + f: impl FnOnce(&mut SpecTokenizer) -> PyResult + Ungil + Send, + ) -> PyResult { + self.inner.with(py, |lock| { + let mut guard = lock.write().map_err(poisoned)?; + let result = f(&mut guard.spec)?; + guard.compiled = None; + Ok(result) + }) + } +} + +/// Get the compiled pipeline, building it from the spec on first use after a +/// mutation. The `Detached` parameter is the proof this runs off the GIL. +fn get_or_compile(lock: &Detached<'_, Inner>) -> PyResult { + { + let guard = lock.read().map_err(poisoned)?; + if let Some(compiled) = &guard.compiled { + return Ok(compiled.clone()); + } + } + let mut guard = lock.write().map_err(poisoned)?; + if guard.compiled.is_none() { + let pipe = PipelineTokenizer::try_from(&guard.spec).map_err(|e| { + TokenizersError::new_err(format!( + "this tokenizer cannot be compiled to an encode pipeline: {e}" + )) + })?; + let post_adds_special_tokens = guard + .spec + .get_post_processor() + .is_some_and(|p| p.added_tokens(false) > 0); + guard.compiled = Some(Compiled { + pipe: Arc::new(pipe), + post_adds_special_tokens, + }); + } + Ok(guard.compiled.clone().expect("just set")) +} + +fn check_special_tokens_flag(compiled: &Compiled, add_special_tokens: bool) -> PyResult<()> { + if add_special_tokens && compiled.post_adds_special_tokens { + return Err(PyNotImplementedError::new_err( + "this tokenizer's post-processor adds special tokens, but post-processing is not \ + implemented in the encode pipeline yet; pass add_special_tokens=False to encode \ + without them", + )); + } + Ok(()) +} + +fn encode_one( + pipe: &PipelineTokenizer, + text: &str, + pre_tokens: &mut Vec, + scratch: &mut PipelineModelScratch, +) -> PyResult> { + let mut output: Vec = Vec::new(); + pipe.encode_generic::<{ PipelineTokenizer::STAGE_MODEL }>( + text, + pre_tokens, + scratch, + &mut output, + ) + .map_err(to_pyerr)?; + Ok(output.iter().map(|t| t.id).collect()) +} + +#[pymethods] +impl PyTokenizer { + #[new] + fn new(model: PyRef<'_, PyModel>) -> Self { + Self::from_spec(SpecTokenizer::new(model.inner.clone())) + } + + /// Load a tokenizer from a `tokenizer.json` file. + #[staticmethod] + #[pyo3(signature = (path) -> "Tokenizer")] + fn from_file(py: Python<'_>, path: PathBuf) -> PyResult { + let spec = py + .detach(|| SpecTokenizer::from_file(path)) + .map_err(to_pyerr)?; + Ok(Self::from_spec(spec)) + } + + /// Load a tokenizer from the bytes of a `tokenizer.json` file. + #[staticmethod] + #[pyo3(signature = (buffer) -> "Tokenizer")] + fn from_buffer(py: Python<'_>, buffer: Vec) -> PyResult { + let spec = py + .detach(|| SpecTokenizer::from_bytes(&buffer)) + .map_err(to_pyerr)?; + Ok(Self::from_spec(spec)) + } + + /// Download `tokenizer.json` from a model on the Hugging Face Hub (requires + /// the `huggingface_hub` package) and load it. + #[staticmethod] + #[pyo3(signature = (identifier, *, revision = String::from("main"), token = None) -> "Tokenizer")] + fn from_pretrained( + py: Python<'_>, + identifier: &str, + revision: String, + token: Option, + ) -> PyResult { + let hub = py.import("huggingface_hub")?; + let kwargs = pyo3::types::PyDict::new(py); + kwargs.set_item("repo_id", identifier)?; + kwargs.set_item("filename", "tokenizer.json")?; + kwargs.set_item("revision", revision)?; + kwargs.set_item("token", token)?; + let path: PathBuf = hub + .getattr("hf_hub_download")? + .call((), Some(&kwargs))? + .extract()?; + Self::from_file(py, path) + } + + /// Serialize the tokenizer definition as a `tokenizer.json` string. + #[pyo3(signature = (*, pretty = false))] + fn to_str(&self, py: Python<'_>, pretty: bool) -> PyResult { + self.read_spec(py, move |spec| spec.to_string(pretty).map_err(to_pyerr))? + } + + /// Save the tokenizer definition to a `tokenizer.json` file. + #[pyo3(signature = (path, *, pretty = true))] + fn save(&self, py: Python<'_>, path: PathBuf, pretty: bool) -> PyResult<()> { + self.read_spec(py, move |spec| spec.save(path, pretty).map_err(to_pyerr))? + } + + /// Encode `text` into token ids. + /// + /// Runs entirely outside the interpreter lock and returns a `numpy.uint32` + /// array backed by the Rust output buffer (no copy). + #[pyo3(signature = (text, *, add_special_tokens = true) -> "npt.NDArray[np.uint32]")] + fn encode<'py>( + &self, + py: Python<'py>, + text: &str, + add_special_tokens: bool, + ) -> PyResult>> { + let ids = self.inner.with(py, |lock| -> PyResult> { + let compiled = get_or_compile(&lock)?; + check_special_tokens_flag(&compiled, add_special_tokens)?; + let mut pre_tokens = Vec::new(); + let mut scratch = compiled.pipe.get_model().init_scratch(); + encode_one(&compiled.pipe, text, &mut pre_tokens, &mut scratch) + })?; + Ok(ids.into_pyarray(py)) + } + + /// Encode a batch of texts, in parallel across Rust threads (respects + /// `TOKENIZERS_PARALLELISM`), without holding the interpreter lock. + /// Input strings are borrowed, not copied; each output is a `numpy.uint32` + /// array backed by its Rust buffer. + #[pyo3(signature = (texts, *, add_special_tokens = true) -> "list[npt.NDArray[np.uint32]]")] + fn encode_batch<'py>( + &self, + py: Python<'py>, + texts: Vec, + add_special_tokens: bool, + ) -> PyResult> { + let batches = self.inner.with(py, |lock| -> PyResult>> { + let compiled = get_or_compile(&lock)?; + check_special_tokens_flag(&compiled, add_special_tokens)?; + if get_parallelism() && texts.len() > 1 { + USED_PARALLELISM.store(true, Ordering::SeqCst); + texts + .par_iter() + .map_init( + || (Vec::new(), compiled.pipe.get_model().init_scratch()), + |(pre_tokens, scratch), text| { + encode_one(&compiled.pipe, text, pre_tokens, scratch) + }, + ) + .collect() + } else { + let mut pre_tokens = Vec::new(); + let mut scratch = compiled.pipe.get_model().init_scratch(); + texts + .iter() + .map(|text| encode_one(&compiled.pipe, text, &mut pre_tokens, &mut scratch)) + .collect() + } + })?; + let list = PyList::empty(py); + for ids in batches { + list.append(ids.into_pyarray(py))?; + } + Ok(list) + } + + /// Not implemented yet: decoding is not part of the encode pipeline. + #[pyo3(signature = (ids, *, skip_special_tokens = true))] + #[allow(unused_variables)] + fn decode(&self, ids: Vec, skip_special_tokens: bool) -> PyResult { + Err(PyNotImplementedError::new_err( + "decode is not implemented in the encode pipeline yet", + )) + } + + /// Train the model on text files (one sequence per line). + #[pyo3(signature = (files, *, trainer = None))] + fn train( + &self, + py: Python<'_>, + files: Vec, + trainer: Option>, + ) -> PyResult<()> { + let explicit = trainer.map(|t| t.inner.clone()); + self.inner.with(py, |lock| { + let mut guard = lock.write().map_err(poisoned)?; + let mut trainer = explicit.unwrap_or_else(|| guard.spec.get_model().get_trainer()); + guard + .spec + .train_from_files(&mut trainer, files) + .map_err(to_pyerr)?; + guard.compiled = None; + Ok(()) + }) + } + + /// Train the model from any iterator of `str`. + /// + /// The interpreter lock is only re-acquired to refill an internal buffer + /// (256 sequences at a time); the training itself runs multi-threaded in + /// Rust with the lock released. + #[pyo3(signature = (iterator, *, trainer = None))] + fn train_from_iterator( + &self, + py: Python<'_>, + iterator: &Bound<'_, PyAny>, + trainer: Option>, + ) -> PyResult<()> { + let explicit = trainer.map(|t| t.inner.clone()); + let sequences = BufferedPyIterator::new(iterator)?; + let error = sequences.error.clone(); + self.inner.with(py, |lock| { + USED_PARALLELISM.store(true, Ordering::SeqCst); + let mut guard = lock.write().map_err(poisoned)?; + let mut trainer = explicit.unwrap_or_else(|| guard.spec.get_model().get_trainer()); + guard + .spec + .train(&mut trainer, sequences) + .map_err(to_pyerr)?; + guard.compiled = None; + Ok::<_, PyErr>(()) + })?; + if let Some(err) = error.lock().expect("error slot poisoned").take() { + return Err(err); + } + Ok(()) + } + + /// Add tokens to the vocabulary, matched literally in the input text. + fn add_tokens(&self, py: Python<'_>, tokens: Vec) -> PyResult { + let tokens = parse_tokens(tokens, false); + self.mutate_spec(py, move |spec| spec.add_tokens(tokens).map_err(to_pyerr)) + } + + /// Add special tokens (never split, skipped on decode) to the vocabulary. + fn add_special_tokens(&self, py: Python<'_>, tokens: Vec) -> PyResult { + let tokens = parse_tokens(tokens, true); + self.mutate_spec(py, move |spec| { + spec.add_special_tokens(tokens).map_err(to_pyerr) + }) + } + + fn token_to_id(&self, py: Python<'_>, token: &str) -> PyResult> { + self.read_spec(py, |spec| spec.token_to_id(token)) + } + + fn id_to_token(&self, py: Python<'_>, id: u32) -> PyResult> { + self.read_spec(py, move |spec| spec.id_to_token(id)) + } + + /// The whole vocabulary as a dict. This copies every entry; prefer + /// `token_to_id` for lookups. + #[pyo3(signature = (*, with_added_tokens = true))] + fn get_vocab(&self, py: Python<'_>, with_added_tokens: bool) -> PyResult> { + self.read_spec(py, move |spec| spec.get_vocab(with_added_tokens)) + } + + #[pyo3(signature = (*, with_added_tokens = true))] + fn get_vocab_size(&self, py: Python<'_>, with_added_tokens: bool) -> PyResult { + self.read_spec(py, move |spec| spec.get_vocab_size(with_added_tokens)) + } + + /// The model in use by this tokenizer (a copy: reassign to change it). + #[getter] + fn model(&self, py: Python<'_>) -> PyResult> { + let model = self.read_spec(py, |spec| spec.get_model().clone())?; + wrap_model(py, model) + } + + #[setter] + fn set_model(&self, py: Python<'_>, model: PyRef<'_, PyModel>) -> PyResult<()> { + let model = model.inner.clone(); + self.mutate_spec(py, move |spec| { + spec.with_model(model); + Ok(()) + }) + } + + /// The optional normalizer in use by this tokenizer (a copy: reassign to + /// change it). + #[getter] + fn normalizer(&self, py: Python<'_>) -> PyResult>> { + let normalizer = self.read_spec(py, |spec| spec.get_normalizer().cloned())?; + normalizer.map(|n| wrap_normalizer(py, n)).transpose() + } + + #[setter] + fn set_normalizer( + &self, + py: Python<'_>, + normalizer: Option>, + ) -> PyResult<()> { + let normalizer = normalizer.map(|n| n.inner.clone()); + self.mutate_spec(py, move |spec| { + spec.with_normalizer(normalizer).map_err(to_pyerr)?; + Ok(()) + }) + } + + /// The optional pre-tokenizer in use by this tokenizer (a copy: reassign + /// to change it). + #[getter] + fn pre_tokenizer(&self, py: Python<'_>) -> PyResult>> { + let pre_tokenizer = self.read_spec(py, |spec| spec.get_pre_tokenizer().cloned())?; + pre_tokenizer.map(|p| wrap_pre_tokenizer(py, p)).transpose() + } + + #[setter] + fn set_pre_tokenizer( + &self, + py: Python<'_>, + pre_tokenizer: Option>, + ) -> PyResult<()> { + let pre_tokenizer = pre_tokenizer.map(|p| p.inner.clone()); + self.mutate_spec(py, move |spec| { + spec.with_pre_tokenizer(pre_tokenizer); + Ok(()) + }) + } + + fn __repr__(&self, py: Python<'_>) -> PyResult { + self.read_spec(py, |spec| { + format!( + "Tokenizer(model={}, vocab_size={})", + match spec.get_model() { + tk_encode::ModelWrapper::BPE(_) => "BPE", + tk_encode::ModelWrapper::WordPiece(_) => "WordPiece", + tk_encode::ModelWrapper::WordLevel(_) => "WordLevel", + tk_encode::ModelWrapper::Unigram(_) => "Unigram", + }, + spec.get_vocab_size(true) + ) + }) + } + + fn __reduce__<'py>( + &self, + py: Python<'py>, + ) -> PyResult<(Bound<'py, PyAny>, (Bound<'py, PyBytes>,))> { + let data = self.to_str(py, false)?; + let from_buffer = py.get_type::().getattr("from_buffer")?; + Ok((from_buffer, (PyBytes::new(py, data.as_bytes()),))) + } +} + +/// Pulls a Python iterator of `str` from Rust threads: re-attaches to the +/// interpreter only to refill an internal buffer, `CHUNK` items at a time. +/// A conversion error stops the stream and is stashed in `error` for the +/// caller to surface once training finishes. +struct BufferedPyIterator { + iterator: Py, + buffer: std::collections::VecDeque, + finished: bool, + error: Arc>>, +} + +impl BufferedPyIterator { + const CHUNK: usize = 256; + + fn new(iterable: &Bound<'_, PyAny>) -> PyResult { + Ok(Self { + iterator: iterable.try_iter()?.unbind().into(), + buffer: std::collections::VecDeque::with_capacity(Self::CHUNK), + finished: false, + error: Arc::new(Mutex::new(None)), + }) + } + + // The vetted lock-then-GIL direction: the caller (train) holds the write + // lock and re-attaches here. Safe because no attached thread can be + // blocking on the lock — DetachedRwLock makes that unrepresentable. + #[allow(clippy::disallowed_methods)] + fn refill(&mut self) { + let result = Python::attach(|py| -> PyResult { + let iterator = self.iterator.bind(py); + for _ in 0..Self::CHUNK { + match iterator.call_method0("__next__") { + Ok(item) => { + let sequence = item.extract::().map_err(|_| { + PyTypeError::new_err("train_from_iterator expects an iterator of str") + })?; + self.buffer.push_back(sequence); + } + Err(e) if e.is_instance_of::(py) => return Ok(true), + Err(e) => return Err(e), + } + } + Ok(false) + }); + match result { + Ok(done) => self.finished = done, + Err(e) => { + *self.error.lock().expect("error slot poisoned") = Some(e); + self.finished = true; + } + } + } +} + +impl Iterator for BufferedPyIterator { + type Item = String; + + fn next(&mut self) -> Option { + if self.buffer.is_empty() && !self.finished { + self.refill(); + } + self.buffer.pop_front() + } +} diff --git a/bindings/python-pipeline/src/trainers.rs b/bindings/python-pipeline/src/trainers.rs new file mode 100644 index 000000000..f523eb51f --- /dev/null +++ b/bindings/python-pipeline/src/trainers.rs @@ -0,0 +1,179 @@ +use pyo3::prelude::*; +use tk_train::trainers::{ + BpeTrainer, TrainerWrapper, UnigramTrainer, WordLevelTrainer, WordPieceTrainer, +}; + +use crate::added_token::{TokenInput, parse_tokens}; +use crate::error::to_pyerr; + +/// Base class for all trainers. A trainer is a plain configuration value: +/// `Tokenizer.train*` copies it, no state is shared or written back. +#[pyclass( + frozen, + subclass, + name = "Trainer", + module = "tokenizers_pipeline.trainers" +)] +pub struct PyTrainer { + pub inner: TrainerWrapper, +} + +#[pymethods] +impl PyTrainer { + fn __repr__(&self) -> String { + crate::component_repr(&self.inner) + } +} + +#[pyclass(frozen, extends = PyTrainer, name = "BpeTrainer", module = "tokenizers_pipeline.trainers")] +pub struct PyBpeTrainer; + +#[pymethods] +impl PyBpeTrainer { + #[new] + #[pyo3(signature = (*, vocab_size = 30000, min_frequency = 0, special_tokens = vec![], limit_alphabet = None, initial_alphabet = vec![], continuing_subword_prefix = None, end_of_word_suffix = None, max_token_length = None, show_progress = true))] + #[allow(clippy::too_many_arguments)] + fn new( + vocab_size: usize, + min_frequency: u64, + special_tokens: Vec, + limit_alphabet: Option, + initial_alphabet: Vec, + continuing_subword_prefix: Option, + end_of_word_suffix: Option, + max_token_length: Option, + show_progress: bool, + ) -> PyResult> { + let mut builder = BpeTrainer::builder() + .vocab_size(vocab_size) + .min_frequency(min_frequency) + .special_tokens(parse_tokens(special_tokens, true)) + .initial_alphabet(initial_alphabet.into_iter().collect()) + .show_progress(show_progress); + if let Some(limit) = limit_alphabet { + builder = builder.limit_alphabet(limit); + } + if let Some(prefix) = continuing_subword_prefix { + builder = builder.continuing_subword_prefix(prefix); + } + if let Some(suffix) = end_of_word_suffix { + builder = builder.end_of_word_suffix(suffix); + } + if let Some(max) = max_token_length { + builder = builder.max_token_length(Some(max)); + } + Ok(PyClassInitializer::from(PyTrainer { + inner: builder.build().into(), + }) + .add_subclass(PyBpeTrainer)) + } +} + +#[pyclass(frozen, extends = PyTrainer, name = "WordPieceTrainer", module = "tokenizers_pipeline.trainers")] +pub struct PyWordPieceTrainer; + +#[pymethods] +impl PyWordPieceTrainer { + #[new] + #[pyo3(signature = (*, vocab_size = 30000, min_frequency = 0, special_tokens = vec![], limit_alphabet = None, initial_alphabet = vec![], continuing_subword_prefix = String::from("##"), end_of_word_suffix = None, show_progress = true))] + #[allow(clippy::too_many_arguments)] + fn new( + vocab_size: usize, + min_frequency: u64, + special_tokens: Vec, + limit_alphabet: Option, + initial_alphabet: Vec, + continuing_subword_prefix: String, + end_of_word_suffix: Option, + show_progress: bool, + ) -> PyResult> { + let mut builder = WordPieceTrainer::builder() + .vocab_size(vocab_size) + .min_frequency(min_frequency) + .special_tokens(parse_tokens(special_tokens, true)) + .initial_alphabet(initial_alphabet.into_iter().collect()) + .continuing_subword_prefix(continuing_subword_prefix) + .show_progress(show_progress); + if let Some(limit) = limit_alphabet { + builder = builder.limit_alphabet(limit); + } + if let Some(suffix) = end_of_word_suffix { + builder = builder.end_of_word_suffix(suffix); + } + Ok(PyClassInitializer::from(PyTrainer { + inner: builder.build().into(), + }) + .add_subclass(PyWordPieceTrainer)) + } +} + +#[pyclass(frozen, extends = PyTrainer, name = "UnigramTrainer", module = "tokenizers_pipeline.trainers")] +pub struct PyUnigramTrainer; + +#[pymethods] +impl PyUnigramTrainer { + #[new] + #[pyo3(signature = (*, vocab_size = 8000, special_tokens = vec![], initial_alphabet = vec![], unk_token = None, shrinking_factor = 0.75, max_piece_length = 16, n_sub_iterations = 2, show_progress = true))] + #[allow(clippy::too_many_arguments)] + fn new( + vocab_size: u32, + special_tokens: Vec, + initial_alphabet: Vec, + unk_token: Option, + shrinking_factor: f64, + max_piece_length: usize, + n_sub_iterations: u32, + show_progress: bool, + ) -> PyResult> { + let trainer = UnigramTrainer::builder() + .vocab_size(vocab_size) + .special_tokens(parse_tokens(special_tokens, true)) + .initial_alphabet(initial_alphabet.into_iter().collect()) + .unk_token(unk_token) + .shrinking_factor(shrinking_factor) + .max_piece_length(max_piece_length) + .n_sub_iterations(n_sub_iterations) + .show_progress(show_progress) + .build() + .map_err(|e| to_pyerr(e.to_string().into()))?; + Ok(PyClassInitializer::from(PyTrainer { + inner: trainer.into(), + }) + .add_subclass(PyUnigramTrainer)) + } +} + +#[pyclass(frozen, extends = PyTrainer, name = "WordLevelTrainer", module = "tokenizers_pipeline.trainers")] +pub struct PyWordLevelTrainer; + +#[pymethods] +impl PyWordLevelTrainer { + #[new] + #[pyo3(signature = (*, vocab_size = 30000, min_frequency = 0, special_tokens = vec![], show_progress = true))] + fn new( + vocab_size: usize, + min_frequency: u64, + special_tokens: Vec, + show_progress: bool, + ) -> PyResult> { + let trainer = WordLevelTrainer::builder() + .vocab_size(vocab_size) + .min_frequency(min_frequency) + .special_tokens(parse_tokens(special_tokens, true)) + .show_progress(show_progress) + .build() + .map_err(|e| to_pyerr(e.to_string().into()))?; + Ok(PyClassInitializer::from(PyTrainer { + inner: trainer.into(), + }) + .add_subclass(PyWordLevelTrainer)) + } +} + +#[pymodule(gil_used = false)] +pub mod trainers { + #[pymodule_export] + pub use super::{ + PyBpeTrainer, PyTrainer, PyUnigramTrainer, PyWordLevelTrainer, PyWordPieceTrainer, + }; +} diff --git a/bindings/python-pipeline/tools/stub-gen/Cargo.lock b/bindings/python-pipeline/tools/stub-gen/Cargo.lock new file mode 100644 index 000000000..c00f814df --- /dev/null +++ b/bindings/python-pipeline/tools/stub-gen/Cargo.lock @@ -0,0 +1,178 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "goblin" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17582616a7718cca54cec18e534a76c7c4aec11a8b9a85695712f262fd15a4c8" +dependencies = [ + "log", + "plain", + "scroll", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3-introspection" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7775fcc875acdce3872dcb91a4b7bd155ffba6e0ea8be88b8caab7d0b34539a6" +dependencies = [ + "anyhow", + "goblin", + "serde", + "serde_json", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "scroll" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1257cd4248b4132760d6524d6dda4e053bc648c9070b960929bf50cfb1e7add" +dependencies = [ + "scroll_derive", +] + +[[package]] +name = "scroll_derive" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed76efe62313ab6610570951494bdaa81568026e0318eaa55f167de70eeea67d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "stub-gen" +version = "0.1.0" +dependencies = [ + "pyo3-introspection", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/bindings/python-pipeline/tools/stub-gen/Cargo.toml b/bindings/python-pipeline/tools/stub-gen/Cargo.toml new file mode 100644 index 000000000..afb83b1bc --- /dev/null +++ b/bindings/python-pipeline/tools/stub-gen/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "stub-gen" +version = "0.1.0" +edition = "2024" +description = "Generates py_src .pyi stubs by introspecting the built cdylib" + +[dependencies] +pyo3-introspection = "0.29" + +[workspace] diff --git a/bindings/python-pipeline/tools/stub-gen/src/main.rs b/bindings/python-pipeline/tools/stub-gen/src/main.rs new file mode 100644 index 000000000..8acfd4c44 --- /dev/null +++ b/bindings/python-pipeline/tools/stub-gen/src/main.rs @@ -0,0 +1,122 @@ +//! Generates the `.pyi` stubs under `py_src/tokenizers_pipeline/` from the +//! introspection metadata pyo3 embeds in the built extension (the +//! `experimental-inspect` feature). Run after `maturin develop --release`: +//! +//! ```sh +//! cargo run --manifest-path tools/stub-gen/Cargo.toml +//! ``` +//! +//! Return types beyond introspection's reach come from the +//! `#[pyo3(signature = (...) -> "Type")]` annotations in the sources; numpy +//! imports for those annotations are injected here. + +use std::path::{Path, PathBuf}; + +const MODULE: &str = "tokenizers_pipeline"; +/// The `#[pymodule]` name inside the cdylib. +const NATIVE_MODULE: &str = "_native"; + +fn main() -> Result<(), Box> { + let crate_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .ancestors() + .nth(2) + .expect("tools/stub-gen sits two levels under the crate root") + .to_path_buf(); + let cdylib = crate_dir.join(format!( + "target/release/{}{NATIVE_MODULE}.{}", + std::env::consts::DLL_PREFIX, + std::env::consts::DLL_EXTENSION + )); + let out_dir = crate_dir.join("py_src").join(MODULE); + + if !cdylib.is_file() { + return Err(format!( + "no cdylib at {} — run `maturin develop --release` first", + cdylib.display() + ) + .into()); + } + + let module = pyo3_introspection::introspect_cdylib(&cdylib, NATIVE_MODULE)?; + assert_has_docstrings(&module); + + for (rel_path, contents) in pyo3_introspection::module_stub_files(&module) { + let out_path = out_dir.join(place(&rel_path)); + if let Some(parent) = out_path.parent() { + std::fs::create_dir_all(parent)?; + } + let mut contents = postprocess(&contents); + if rel_path == Path::new("__init__.pyi") { + // `create_exception!` types carry no introspection metadata. + contents.push_str("\nclass TokenizersError(Exception): ...\n"); + } + std::fs::write(&out_path, &contents)?; + println!("generated {}", out_path.display()); + } + Ok(()) +} + +/// Map the introspected layout onto the package layout: the root module stub +/// becomes `__init__.pyi`, and each submodule stub lands inside its runtime +/// shim package (`models.pyi` -> `models/__init__.pyi`) so it shadows the +/// `.py` re-exports for type checkers. +fn place(rel_path: &Path) -> PathBuf { + let name = rel_path + .file_name() + .and_then(|n| n.to_str()) + .expect("stub paths are utf-8 files"); + match name.strip_suffix(".pyi") { + Some("__init__") | None => rel_path.to_path_buf(), + Some(module) => rel_path.with_file_name(module).join("__init__.pyi"), + } +} + +fn postprocess(contents: &str) -> String { + // Cross-submodule references come out relative to the extension root; + // absolutize them to the package. + let mut contents = contents + .replace("from . import", &format!("from {MODULE} import")) + .replace("from .", &format!("from {MODULE}.")); + // Annotated numpy return types need their imports. + if contents.contains("npt.") || contents.contains("np.") { + contents = format!( + "import numpy as np\nimport numpy.typing as npt\n\n{contents}" + ); + } + contents +} + +/// Fail loudly if introspection came back without docstrings — that means the +/// cdylib was built without `experimental-inspect` (or the feature broke) and +/// the stubs would silently lose all documentation. +fn assert_has_docstrings(module: &pyo3_introspection::model::Module) { + fn count(module: &pyo3_introspection::model::Module) -> (usize, usize) { + let mut with_doc = 0; + let mut total = 0; + for f in &module.functions { + total += 1; + with_doc += f.docstring.is_some() as usize; + } + for c in &module.classes { + total += 1; + with_doc += c.docstring.is_some() as usize; + for m in &c.methods { + total += 1; + with_doc += m.docstring.is_some() as usize; + } + } + for sub in &module.modules { + let (w, t) = count(sub); + with_doc += w; + total += t; + } + (with_doc, total) + } + let (with_doc, total) = count(module); + println!("docstring coverage: {with_doc}/{total}"); + assert!( + with_doc > 0, + "introspection returned 0/{total} docstrings — was the cdylib built \ + with the `experimental-inspect` pyo3 feature?" + ); +} From 8f8631ab6e1f3afe122eed846127d0e35f483f18 Mon Sep 17 00:00:00 2001 From: SBrandeis <33657802+SBrandeis@users.noreply.github.com> Date: Wed, 22 Jul 2026 11:04:33 +0200 Subject: [PATCH 05/19] iteration --- .github/workflows/pipeline-bench.yml | 55 +++++ bindings/python-pipeline/Makefile | 36 +++ bindings/python-pipeline/README.md | 94 ++++++++ .../benches/bench_vs_release.py | 218 ++++++++++++++++++ .../py_src/tokenizers_pipeline/__init__.py | 2 + .../py_src/tokenizers_pipeline/__init__.pyi | 55 ++++- .../tokenizers_pipeline/models/__init__.py | 2 + .../tokenizers_pipeline/models/__init__.pyi | 32 ++- .../normalizers/__init__.py | 2 + .../normalizers/__init__.pyi | 48 +++- .../pre_tokenizers/__init__.py | 2 + .../pre_tokenizers/__init__.pyi | 61 ++++- .../tokenizers_pipeline/trainers/__init__.py | 2 + .../tokenizers_pipeline/trainers/__init__.pyi | 31 ++- bindings/python-pipeline/src/added_token.rs | 7 +- bindings/python-pipeline/src/lib.rs | 1 + bindings/python-pipeline/src/models.rs | 21 +- bindings/python-pipeline/src/normalizers.rs | 64 ++++- .../python-pipeline/src/pre_tokenizers.rs | 59 ++++- bindings/python-pipeline/src/tokenizer.rs | 28 ++- bindings/python-pipeline/src/trainers.rs | 20 +- 21 files changed, 784 insertions(+), 56 deletions(-) create mode 100644 bindings/python-pipeline/Makefile create mode 100644 bindings/python-pipeline/README.md create mode 100644 bindings/python-pipeline/benches/bench_vs_release.py diff --git a/.github/workflows/pipeline-bench.yml b/.github/workflows/pipeline-bench.yml index fa58112bd..9257000a4 100644 --- a/.github/workflows/pipeline-bench.yml +++ b/.github/workflows/pipeline-bench.yml @@ -47,6 +47,7 @@ on: paths: - "tokenizers/tk-encode/**" - "tokenizers/src/**" + - "bindings/python-pipeline/**" - ".github/workflows/pipeline-bench.yml" - ".github/scripts/render_pipeline_bench.py" - "tokenizers/tk-encode/examples/bench_models.json" @@ -229,6 +230,60 @@ jobs: path: tokenizers/pipeline_bench_${{ matrix.shard }}.json retention-days: 3 + # Python bindings: the tokenizers_pipeline wheel vs the latest *released* + # tokenizers wheel from PyPI, timed end-to-end through Python — input + # conversion, encode, and output objects all count, because that is what a + # user pays. Same models (bench_models.json) and fixture corpora as the Rust + # bench; unsupported models are reported as skipped, an id mismatch fails + # the job. Results land in the run's step summary + an artifact. + python-bindings-bench: + name: Python bindings vs released wheel + if: github.event_name != 'pull_request' || github.event.label.name == 'run-pipeline-bench' + runs-on: + group: aws-general-8-plus + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + + - name: Install Rust stable + uses: dtolnay/rust-toolchain@stable + + - name: Setup sccache + uses: mozilla-actions/sccache-action@v0.0.9 + + - name: Install uv + uses: astral-sh/setup-uv@v6 + + - name: Download fixtures and model tokenizers + working-directory: tokenizers + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + run: make fixtures bench-models HF="uvx --from huggingface_hub hf" + + - name: Build and install the tokenizers_pipeline wheel + working-directory: bindings/python-pipeline + run: | + uv venv .venv + uv pip install --python .venv/bin/python maturin numpy tokenizers + source .venv/bin/activate && maturin develop --release + + - name: Bench against the released wheel + working-directory: bindings/python-pipeline + run: | + .venv/bin/python benches/bench_vs_release.py \ + --manifest ../../tokenizers/tk-encode/examples/bench_models.json \ + --json python_bench.json --markdown python_bench.md + cat python_bench.md >> "$GITHUB_STEP_SUMMARY" + + - name: Upload results + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: python-bindings-bench + path: | + bindings/python-pipeline/python_bench.json + bindings/python-pipeline/python_bench.md + retention-days: 30 + # Fan-in: concatenate the shard partials (in shard order = manifest order), # then render + upload charts + update the PR, once. report: diff --git a/bindings/python-pipeline/Makefile b/bindings/python-pipeline/Makefile new file mode 100644 index 000000000..af4f2d6cf --- /dev/null +++ b/bindings/python-pipeline/Makefile @@ -0,0 +1,36 @@ +PYTHON := .venv/bin/python + +# Everything needed to hack on the bindings: venv, deps, release build. +.PHONY: dev +dev: .venv + . .venv/bin/activate && maturin develop --release + +.venv: + uv venv .venv + uv pip install --python $(PYTHON) maturin numpy tokenizers + +# Regenerate the .pyi stubs from the built extension. Run after `make dev`. +.PHONY: stubs +stubs: + cargo run --manifest-path tools/stub-gen/Cargo.toml + +# Run the end-to-end examples (train, pretrained parity, threading). +.PHONY: examples +examples: dev + $(PYTHON) examples/01_train_and_encode.py + $(PYTHON) examples/02_pretrained.py + $(PYTHON) examples/03_threading.py + +# Benchmark against the released tokenizers wheel (same script CI runs). +.PHONY: bench +bench: dev + $(PYTHON) benches/bench_vs_release.py + +.PHONY: lint +lint: + cargo fmt --check + cargo clippy --all-targets -- -D warnings + +.PHONY: clean +clean: + rm -rf .venv target tools/stub-gen/target diff --git a/bindings/python-pipeline/README.md b/bindings/python-pipeline/README.md new file mode 100644 index 000000000..c280c6ce7 --- /dev/null +++ b/bindings/python-pipeline/README.md @@ -0,0 +1,94 @@ +# tokenizers-pipeline + +Experimental Python bindings for 🤗 tokenizers, built on the `PipelineTokenizer` +encode path. Same `tokenizer.json` files, same ids, and much faster through +Python: encode never holds the GIL, batches run multi-threaded in Rust, inputs +are borrowed instead of copied, and ids come back as `numpy.uint32` arrays +without a copy. + +```python +import tokenizers_pipeline as tp + +tok = tp.Tokenizer.from_file("tokenizer.json") +ids = tok.encode("Hello world", add_special_tokens=False) # np.ndarray[uint32] +batch = tok.encode_batch(lines, add_special_tokens=False) # list of arrays +``` + +Training and in-place modification work too: + +```python +tok = tp.Tokenizer(tp.models.BPE()) +tok.normalizer = tp.normalizers.Lowercase() +tok.pre_tokenizer = tp.pre_tokenizers.Whitespace() +tok.train_from_iterator(lines, trainer=tp.trainers.BpeTrainer(vocab_size=30000)) +tok.save("tokenizer.json") +``` + +Not there yet (loud errors, never wrong ids): `decode`, post-processor +templates (`[CLS]`/`` insertion — pass `add_special_tokens=False`), and the +`Metaspace` pre-tokenizer (t5-style files). + +## Build and use locally + +Requirements: Rust (stable), [uv](https://docs.astral.sh/uv/), Python ≥ 3.10. + +```sh +cd bindings/python-pipeline +make dev # venv + deps + release build, installed editable +source .venv/bin/activate +python -c "import tokenizers_pipeline; print(tokenizers_pipeline.__version__)" +``` + +Rebuild after changing Rust code with `make dev` again (or `maturin develop +--release` inside the venv). Always use `--release`: a debug build encodes +10-100× slower and any timing you take from it is meaningless. + +To build a distributable wheel instead: `maturin build --release` (find it in +`target/wheels/`). + +Other targets: + +```sh +make examples # run the three end-to-end examples (needs ../../tokenizers/data) +make bench # benchmark against the released tokenizers wheel +make stubs # regenerate the .pyi type stubs from the built extension +make lint # cargo fmt --check + clippy -D warnings +``` + +The examples and the benchmark read test data from `../../tokenizers/data`. +Fetch it once with `make -C ../../tokenizers fixtures bench-models data/big.txt` +(needs `HF_TOKEN` for the mirror repo). + +## Type stubs are generated + +Do not edit the `.pyi` files under `py_src/` by hand. They are produced by +`tools/stub-gen`, which reads the introspection metadata pyo3 embeds in the +built extension — so run `make dev` first, then `make stubs`. Docstrings and +signatures come from the Rust sources; return types that introspection cannot +see (numpy arrays, `Self`) are declared with +`#[pyo3(signature = (...) -> "Type")]` annotations in the Rust code. + +## How it works + +A `Tokenizer` holds two things behind one lock: + +- the **spec** — the plain Rust `Tokenizer`, the serializable source of truth. + Setters, `train*`, and `add_*` write here. +- the **compiled pipeline** — an immutable `Arc` the encode + methods share with worker threads. Any mutation drops it; the next encode + rebuilds it once. Configurations the pipeline cannot run fail at that point + with the reason, never with different ids. + +Every method releases the GIL before touching the lock — enforced at compile +time by `DetachedRwLock` (see `src/detached_lock.rs`), with a clippy ban on +`Python::attach` as the backstop. + +## Benchmark + +`benches/bench_vs_release.py` times `encode_batch` end-to-end through Python +against the latest released `tokenizers` wheel, on the same corpora and ~10 KiB +chunking as the Rust benchmark (`tk-encode/examples/fixture_bench.rs`): every +fixture under `data/fixtures/{lang,modalities}`, warmed up, median of N runs, +single-thread per fixture plus one multi-thread sweep, ids verified equal +before timing. CI runs it in the `python-bindings-bench` job of the Pipeline +Benchmark workflow and posts the table to the run's step summary. diff --git a/bindings/python-pipeline/benches/bench_vs_release.py b/bindings/python-pipeline/benches/bench_vs_release.py new file mode 100644 index 000000000..005d7d5ad --- /dev/null +++ b/bindings/python-pipeline/benches/bench_vs_release.py @@ -0,0 +1,218 @@ +"""Benchmark tokenizers_pipeline against the released `tokenizers` wheel. + +Mirrors tk-encode/examples/fixture_bench.rs: every `.txt` corpus under +data/fixtures/{lang,modalities}, cut into ~10 KiB multi-line chunks (at most +100 per fixture), single-thread throughput per fixture plus one multi-thread +sweep over all fixtures flattened. Timing is end-to-end through Python — +input conversion, encode, and output objects all count, because that is what +a user pays. Ids are checked to match on every fixture before anything is +timed; a mismatch fails the run. + +Usage: + python benches/bench_vs_release.py [--manifest bench_models.json] + [--data-dir ../../tokenizers/data] [--iters 3] + [--json out.json] [--markdown out.md] +""" + +import argparse +import json +import os +import statistics +import sys +import time +from pathlib import Path + +import tokenizers as release +import tokenizers_pipeline as pipeline + +# Keep in sync with fixture_bench.rs (CHUNK_BYTES, MAX_CHUNKS). +CHUNK_BYTES = 10 * 1024 +MAX_CHUNKS = 100 + +DEFAULT_MODELS = [ + {"name": "gpt2", "file": "gpt2.json"}, + {"name": "llama-3", "file": "llama-3-tokenizer.json"}, + {"name": "llama-2", "file": "llama-2.json"}, + {"name": "bert-base-uncased", "file": "bert-base-uncased.json"}, +] + + +def make_chunks(text: str) -> list[str]: + """~10 KiB multi-line chunks, same construction as fixture_bench.rs.""" + chunks: list[str] = [] + cur: list[str] = [] + cur_bytes = 0 + for line in text.splitlines(): + if not line.strip(): + continue + cur_bytes += len(line.encode()) + bool(cur) + cur.append(line) + if cur_bytes >= CHUNK_BYTES: + chunks.append("\n".join(cur)) + if len(chunks) == MAX_CHUNKS: + return chunks + cur, cur_bytes = [], 0 + if cur: + chunks.append("\n".join(cur)) + return chunks + + +def load_fixtures(data_dir: Path) -> list[dict]: + fixtures = [] + for group in ["lang", "modalities"]: + directory = data_dir / "fixtures" / group + if not directory.is_dir(): + sys.exit(f"{directory} not found — run `make fixtures` in tokenizers/ first") + for path in sorted(directory.glob("*.txt")): + chunks = make_chunks(path.read_text(encoding="utf-8")) + fixtures.append( + { + "group": group, + "name": path.stem, + "chunks": chunks, + "bytes": sum(len(c.encode()) for c in chunks), + } + ) + return fixtures + + +def timed(fn, iters: int) -> float: + fn() # warmup: compile the pipeline, fill caches, spin up thread pools + samples = [] + for _ in range(iters): + start = time.perf_counter() + fn() + samples.append(time.perf_counter() - start) + return statistics.median(samples) + + +def bench_model(model: dict, fixtures: list[dict], iters: int) -> dict: + row = {"model": model["name"], "fixtures": []} + try: + ours = pipeline.Tokenizer.from_file(model["path"]) + ours.encode("warmup", add_special_tokens=False) + except (pipeline.TokenizersError, NotImplementedError) as e: + row["skipped"] = str(e) + return row + theirs = release.Tokenizer.from_file(str(model["path"])) + + os.environ["TOKENIZERS_PARALLELISM"] = "false" + for fixture in fixtures: + chunks = fixture["chunks"] + parity = ours.encode_batch(chunks, add_special_tokens=False) + reference = theirs.encode_batch_fast(chunks, add_special_tokens=False) + t_ours = timed(lambda: ours.encode_batch(chunks, add_special_tokens=False), iters) + t_theirs = timed( + lambda: theirs.encode_batch_fast(chunks, add_special_tokens=False), iters + ) + row["fixtures"].append( + { + "fixture": fixture["name"], + "group": fixture["group"], + "bytes": fixture["bytes"], + "ids_match": all( + a.tolist() == b.ids for a, b in zip(parity, reference, strict=True) + ), + "pipeline_mbps": fixture["bytes"] / t_ours / 1e6, + "release_mbps": fixture["bytes"] / t_theirs / 1e6, + "speedup": t_theirs / t_ours, + } + ) + + # Multi-thread: one sweep over all fixtures flattened, like fixture_bench.rs. + all_chunks = [c for f in fixtures for c in f["chunks"]] + nbytes = sum(f["bytes"] for f in fixtures) + os.environ["TOKENIZERS_PARALLELISM"] = "true" + t_ours = timed(lambda: ours.encode_batch(all_chunks, add_special_tokens=False), iters) + t_theirs = timed( + lambda: theirs.encode_batch_fast(all_chunks, add_special_tokens=False), iters + ) + row["multi_thread"] = { + "bytes": nbytes, + "pipeline_mbps": nbytes / t_ours / 1e6, + "release_mbps": nbytes / t_theirs / 1e6, + "speedup": t_theirs / t_ours, + } + return row + + +def render_markdown(report: dict) -> str: + lines = [ + "## Python bindings: `tokenizers_pipeline` vs released `tokenizers` " + f"{report['release_version']}", + "", + f"{report['fixture_count']} fixtures (~10 KiB chunks, ≤100/fixture), median of " + f"{report['iters']} runs, {report['cpus']} CPUs. Single-thread numbers aggregate " + "all fixtures (speedup range = slowest…fastest fixture); multi-thread runs the " + "flattened corpus. Speedup >1 means the pipeline bindings are faster.", + "", + "| model | ids | pipeline 1t (MB/s) | release 1t (MB/s) | speedup 1t (range) " + "| pipeline mt (MB/s) | release mt (MB/s) | speedup mt |", + "|---|---|---|---|---|---|---|---|", + ] + for row in report["models"]: + if "skipped" in row: + lines.append(f"| {row['model']} | — | skipped: {row['skipped']} | | | | | |") + continue + fixtures = row["fixtures"] + nbytes = sum(f["bytes"] for f in fixtures) + t_ours = sum(f["bytes"] / (f["pipeline_mbps"] * 1e6) for f in fixtures) + t_theirs = sum(f["bytes"] / (f["release_mbps"] * 1e6) for f in fixtures) + speedups = [f["speedup"] for f in fixtures] + ids = "✓" if all(f["ids_match"] for f in fixtures) else "✗ MISMATCH" + mt = row["multi_thread"] + lines.append( + f"| {row['model']} | {ids} " + f"| {nbytes / t_ours / 1e6:.0f} | {nbytes / t_theirs / 1e6:.0f} " + f"| {t_theirs / t_ours:.2f}× ({min(speedups):.2f}…{max(speedups):.2f}) " + f"| {mt['pipeline_mbps']:.0f} | {mt['release_mbps']:.0f} | {mt['speedup']:.2f}× |" + ) + lines += ["", "Per-fixture numbers are in the `python-bindings-bench` artifact."] + return "\n".join(lines) + "\n" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", type=Path, help="bench_models.json to take the model list from") + parser.add_argument("--data-dir", type=Path, default=Path(__file__).parents[3] / "tokenizers" / "data") + parser.add_argument("--iters", type=int, default=3) + parser.add_argument("--json", type=Path, help="write the full report here") + parser.add_argument("--markdown", type=Path, help="write the summary table here") + args = parser.parse_args() + + models = json.load(open(args.manifest)) if args.manifest else DEFAULT_MODELS + for model in models: + model["path"] = args.data_dir / model.get("file", model["name"] + ".json") + models = [m for m in models if m["path"].is_file()] or sys.exit("no model files found") + + fixtures = load_fixtures(args.data_dir) + report = { + "release_version": release.__version__, + "pipeline_version": pipeline.__version__, + "iters": args.iters, + "fixture_count": len(fixtures), + "cpus": os.cpu_count(), + "models": [bench_model(m, fixtures, args.iters) for m in models], + } + + markdown = render_markdown(report) + print(markdown) + if args.json: + args.json.write_text(json.dumps(report, indent=2)) + if args.markdown: + args.markdown.write_text(markdown) + + mismatches = [ + f"{row['model']}/{f['fixture']}" + for row in report["models"] + for f in row.get("fixtures", []) + if not f["ids_match"] + ] + if mismatches: + print(f"ids diverge on: {mismatches}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bindings/python-pipeline/py_src/tokenizers_pipeline/__init__.py b/bindings/python-pipeline/py_src/tokenizers_pipeline/__init__.py index f63f89f55..003333d64 100644 --- a/bindings/python-pipeline/py_src/tokenizers_pipeline/__init__.py +++ b/bindings/python-pipeline/py_src/tokenizers_pipeline/__init__.py @@ -1,3 +1,5 @@ +"""Fast tokenizers built on the pipeline encode path. Start with `Tokenizer`.""" + from ._native import ( AddedToken, Tokenizer, diff --git a/bindings/python-pipeline/py_src/tokenizers_pipeline/__init__.pyi b/bindings/python-pipeline/py_src/tokenizers_pipeline/__init__.pyi index e421d42b6..75859f3f1 100644 --- a/bindings/python-pipeline/py_src/tokenizers_pipeline/__init__.pyi +++ b/bindings/python-pipeline/py_src/tokenizers_pipeline/__init__.pyi @@ -1,6 +1,10 @@ import numpy as np import numpy.typing as npt +""" +Fast tokenizers built on the pipeline encode path. Start with `Tokenizer`. +""" + from tokenizers_pipeline.models import Model from tokenizers_pipeline.normalizers import Normalizer from tokenizers_pipeline.pre_tokenizers import PreTokenizer @@ -14,7 +18,12 @@ __version__: Final[str] @final class AddedToken: """ - A token added on top of the model's vocabulary, with its matching options. + A token added to the vocabulary after training, with options for how it + is matched in text: `single_word` only matches when it stands alone (not + inside a word); `lstrip`/`rstrip` also swallow the whitespace before/after + it; `normalized` matches against normalized instead of raw text (defaults + to the opposite of `special`); `special` marks template tokens like "" + that decoding should be able to skip. """ def __new__(cls, /, content: str, *, single_word: bool = False, lstrip: bool = False, rstrip: bool = False, normalized: bool |None = None, special: bool = False) -> AddedToken: ... def __repr__(self, /) -> str: ... @@ -35,19 +44,31 @@ class AddedToken: class Tokenizer: """ A tokenizer: a model plus its optional normalizer and pre-tokenizer. - Mutations apply to the serializable definition; encode runs a compiled - pipeline that is rebuilt automatically after any change. + + Create one from a model (`Tokenizer(models.BPE())`), a file + (`Tokenizer.from_file`), or the Hub (`Tokenizer.from_pretrained`). + Changes — assigning components, training, adding tokens — apply to the + serializable definition; encoding runs a compiled pipeline that is rebuilt + automatically after any change. A definition the pipeline cannot run + raises `TokenizersError` at that point, with the reason. """ - def __new__(cls, /, model: Model) -> Tokenizer: ... + def __new__(cls, /, model: Model) -> Tokenizer: + """ + Create an untrained tokenizer from a model. + """ def __reduce__(self, /) -> tuple[Any, tuple[bytes]]: ... def __repr__(self, /) -> str: ... def add_special_tokens(self, /, tokens: Sequence[str |AddedToken]) -> int: """ - Add special tokens (never split, skipped on decode) to the vocabulary. + Add special tokens ("", "[CLS]", …) to the vocabulary. Same as + `add_tokens`, but every token is marked `special`. Returns how many + were actually new. """ def add_tokens(self, /, tokens: Sequence[str |AddedToken]) -> int: """ - Add tokens to the vocabulary, matched literally in the input text. + Add tokens to the vocabulary and match them in the input text from now + on. Plain strings match with default options; pass `AddedToken` to + control matching. Returns how many were actually new. """ def decode(self, /, ids: Sequence[int], *, skip_special_tokens: bool = True) -> str: """ @@ -88,8 +109,15 @@ class Tokenizer: The whole vocabulary as a dict. This copies every entry; prefer `token_to_id` for lookups. """ - def get_vocab_size(self, /, *, with_added_tokens: bool = True) -> int: ... - def id_to_token(self, /, id: int) -> str |None: ... + def get_vocab_size(self, /, *, with_added_tokens: bool = True) -> int: + """ + Number of entries in the vocabulary. `with_added_tokens=False` counts + only what the model was trained with. + """ + def id_to_token(self, /, id: int) -> str |None: + """ + The token behind `id`, or None if the id is out of range. + """ @property def model(self, /) -> Model: """ @@ -121,14 +149,19 @@ class Tokenizer: """ Serialize the tokenizer definition as a `tokenizer.json` string. """ - def token_to_id(self, /, token: str) -> int |None: ... + def token_to_id(self, /, token: str) -> int |None: + """ + The id of `token`, or None if it is not in the vocabulary. + """ def train(self, /, files: Sequence[str], *, trainer: Trainer |None = None) -> None: """ - Train the model on text files (one sequence per line). + Train the model's vocabulary on text files (one sequence per line). + Without a `trainer`, the model's default trainer is used. """ def train_from_iterator(self, /, iterator: Any, *, trainer: Trainer |None = None) -> None: """ - Train the model from any iterator of `str`. + Train the model's vocabulary from any iterator of `str`. Without a + `trainer`, the model's default trainer is used. The interpreter lock is only re-acquired to refill an internal buffer (256 sequences at a time); the training itself runs multi-threaded in diff --git a/bindings/python-pipeline/py_src/tokenizers_pipeline/models/__init__.py b/bindings/python-pipeline/py_src/tokenizers_pipeline/models/__init__.py index 05d3828ae..6ae6ab347 100644 --- a/bindings/python-pipeline/py_src/tokenizers_pipeline/models/__init__.py +++ b/bindings/python-pipeline/py_src/tokenizers_pipeline/models/__init__.py @@ -1,3 +1,5 @@ +"""The algorithms that turn pre-tokenized pieces into token ids.""" + from .._native import models as _models Model = _models.Model diff --git a/bindings/python-pipeline/py_src/tokenizers_pipeline/models/__init__.pyi b/bindings/python-pipeline/py_src/tokenizers_pipeline/models/__init__.pyi index c7d80c954..ef01439ae 100644 --- a/bindings/python-pipeline/py_src/tokenizers_pipeline/models/__init__.pyi +++ b/bindings/python-pipeline/py_src/tokenizers_pipeline/models/__init__.pyi @@ -1,7 +1,18 @@ +""" +The algorithms that turn pre-tokenized pieces into token ids. +""" + from typing import final @final class BPE(Model): + """ + Byte-Pair Encoding: builds tokens by applying the merges learned during + training. `unk_token` stands in for characters the vocabulary cannot + represent; `byte_fallback` encodes them as raw bytes instead. `dropout` + randomly skips merges (a training-time regularization). `ignore_merges` + looks whole pieces up in the vocabulary before merging. + """ def __new__(cls, /, *, unk_token: str |None = None, dropout: float |None = None, fuse_unk: bool = False, byte_fallback: bool = False, ignore_merges: bool = False) -> BPE: ... @staticmethod def from_file(vocab: str, merges: str, *, unk_token: str |None = None) -> "BPE": @@ -11,19 +22,36 @@ class BPE(Model): class Model: """ - Base class for all models. Not constructible from Python; holds the actual - Rust model by value (no sharing with the Tokenizer — assignment copies). + Base class for all models. + + The model is the trained part of a tokenizer: it turns each pre-tokenized + piece into token ids using its vocabulary. Models are immutable values — + assigning one to a tokenizer copies it. """ def __repr__(self, /) -> str: ... @final class Unigram(Model): + """ + The SentencePiece Unigram model: picks the most probable segmentation + under a learned piece vocabulary. Starts empty — train it, or load a + tokenizer.json. + """ def __new__(cls, /) -> Unigram: ... @final class WordLevel(Model): + """ + The simplest model: one whole word, one id. Words outside the vocabulary + become `unk_token`. + """ def __new__(cls, /, *, unk_token: str = ...) -> WordLevel: ... @final class WordPiece(Model): + """ + The BERT model: greedily matches the longest vocabulary entry, marking + word continuations with a prefix ("##" by default). A piece longer than + `max_input_chars_per_word` becomes `unk_token` outright. + """ def __new__(cls, /, *, unk_token: str = ..., continuing_subword_prefix: str = ..., max_input_chars_per_word: int = 100) -> WordPiece: ... diff --git a/bindings/python-pipeline/py_src/tokenizers_pipeline/normalizers/__init__.py b/bindings/python-pipeline/py_src/tokenizers_pipeline/normalizers/__init__.py index 03d8469aa..c66373610 100644 --- a/bindings/python-pipeline/py_src/tokenizers_pipeline/normalizers/__init__.py +++ b/bindings/python-pipeline/py_src/tokenizers_pipeline/normalizers/__init__.py @@ -1,3 +1,5 @@ +"""Text cleanup that runs before the text is split.""" + from .._native import normalizers as _normalizers Normalizer = _normalizers.Normalizer diff --git a/bindings/python-pipeline/py_src/tokenizers_pipeline/normalizers/__init__.pyi b/bindings/python-pipeline/py_src/tokenizers_pipeline/normalizers/__init__.pyi index 54a493c63..04ebfd371 100644 --- a/bindings/python-pipeline/py_src/tokenizers_pipeline/normalizers/__init__.pyi +++ b/bindings/python-pipeline/py_src/tokenizers_pipeline/normalizers/__init__.pyi @@ -1,53 +1,97 @@ +""" +Text cleanup that runs before the text is split. +""" + from collections.abc import Sequence as Sequence2 from typing import final @final class BertNormalizer(Normalizer): + """ + The BERT cleanup: removes control characters, puts spaces around CJK + characters, and optionally strips accents and lowercases. + `strip_accents=None` means "follow the lowercase setting", like the + original BERT. + """ def __new__(cls, /, *, clean_text: bool = True, handle_chinese_chars: bool = True, strip_accents: bool |None = None, lowercase: bool = True) -> BertNormalizer: ... @final class Lowercase(Normalizer): + """ + Lowercases everything. + """ def __new__(cls, /) -> Lowercase: ... @final class NFC(Normalizer): + """ + Unicode NFC: recombines split characters (e + ´ becomes é). + """ def __new__(cls, /) -> NFC: ... @final class NFD(Normalizer): + """ + Unicode NFD: splits characters into base + accents (é becomes e + ´). + """ def __new__(cls, /) -> NFD: ... @final class NFKC(Normalizer): + """ + Unicode NFKC: NFC, plus compatibility replacements (fi becomes fi). + """ def __new__(cls, /) -> NFKC: ... @final class NFKD(Normalizer): + """ + Unicode NFKD: NFD, plus compatibility replacements (fi becomes fi). + """ def __new__(cls, /) -> NFKD: ... class Normalizer: """ - Base class for all normalizers. Immutable value: assigning it to a - Tokenizer copies the configuration, there is no shared state. + Base class for all normalizers. + + A normalizer rewrites text before it is split: cleanup, case-folding, + Unicode normalization. Normalizers are immutable values — assigning one to + a tokenizer copies it. """ def __repr__(self, /) -> str: ... @final class Prepend(Normalizer): + """ + Puts a fixed string in front of the text (SentencePiece prepends "▁"). + """ def __new__(cls, /, prepend: str) -> Prepend: ... @final class Replace(Normalizer): + """ + Replaces every occurrence of `pattern` with `content`. With `regex=True` + the pattern is a regular expression. + """ def __new__(cls, /, pattern: str, content: str, *, regex: bool = False) -> Replace: ... @final class Sequence(Normalizer): + """ + Runs several normalizers in order. + """ def __new__(cls, /, normalizers: Sequence2[Normalizer]) -> Sequence: ... @final class Strip(Normalizer): + """ + Removes whitespace at the start and/or end of the text. + """ def __new__(cls, /, *, left: bool = True, right: bool = True) -> Strip: ... @final class StripAccents(Normalizer): + """ + Removes accents (é becomes e). Only works on decomposed text: put NFD before it. + """ def __new__(cls, /) -> StripAccents: ... diff --git a/bindings/python-pipeline/py_src/tokenizers_pipeline/pre_tokenizers/__init__.py b/bindings/python-pipeline/py_src/tokenizers_pipeline/pre_tokenizers/__init__.py index 1256f852b..49c12046d 100644 --- a/bindings/python-pipeline/py_src/tokenizers_pipeline/pre_tokenizers/__init__.py +++ b/bindings/python-pipeline/py_src/tokenizers_pipeline/pre_tokenizers/__init__.py @@ -1,3 +1,5 @@ +"""How text is cut into pieces before the model runs.""" + from .._native import pre_tokenizers as _pre_tokenizers PreTokenizer = _pre_tokenizers.PreTokenizer diff --git a/bindings/python-pipeline/py_src/tokenizers_pipeline/pre_tokenizers/__init__.pyi b/bindings/python-pipeline/py_src/tokenizers_pipeline/pre_tokenizers/__init__.pyi index 1e1c31738..63ecde851 100644 --- a/bindings/python-pipeline/py_src/tokenizers_pipeline/pre_tokenizers/__init__.pyi +++ b/bindings/python-pipeline/py_src/tokenizers_pipeline/pre_tokenizers/__init__.pyi @@ -1,59 +1,104 @@ +""" +How text is cut into pieces before the model runs. +""" + from collections.abc import Sequence as Sequence2 from typing import final @final class BertPreTokenizer(PreTokenizer): + """ + The BERT split: on whitespace, and each punctuation character becomes its own piece. + """ def __new__(cls, /) -> BertPreTokenizer: ... @final class ByteLevel(PreTokenizer): - def __new__(cls, /, *, use_regex: bool = True) -> ByteLevel: - """ - `add_prefix_space` is not supported by the pipeline and is always false. - """ + """ + GPT-2 style byte-level splitting: cuts with the GPT-2 regex unless + `use_regex=False`. The pipeline does not support `add_prefix_space`, so it + is always off. + """ + def __new__(cls, /, *, use_regex: bool = True) -> ByteLevel: ... @final class CharDelimiterSplit(PreTokenizer): + """ + Splits on one fixed character, dropping it. + """ def __new__(cls, /, delimiter: str) -> CharDelimiterSplit: ... @final class Digits(PreTokenizer): + """ + Separates digits from everything else. With `individual_digits=True`, + every digit becomes its own piece. + """ def __new__(cls, /, *, individual_digits: bool = False) -> Digits: ... @final class FixedLength(PreTokenizer): + """ + Cuts the text into pieces of exactly `length` characters (the last one may + be shorter). + """ def __new__(cls, /, *, length: int = 5) -> FixedLength: ... class PreTokenizer: """ - Base class for all pre-tokenizers. Immutable value: assigning it to a - Tokenizer copies the configuration, there is no shared state. + Base class for all pre-tokenizers. - Only pre-tokenizers supported by the encode pipeline are constructible here; - notably `Metaspace` is not available yet. + A pre-tokenizer cuts text into pieces (usually words); the model then turns + each piece into token ids. Pre-tokenizers are immutable values — assigning + one to a tokenizer copies it. Only pre-tokenizers the encode pipeline can + run are constructible here; `Metaspace` is not available yet. """ def __repr__(self, /) -> str: ... @final class Punctuation(PreTokenizer): + """ + Splits on punctuation. `behavior` says what happens to the punctuation + itself — see `Split` for the options. + """ def __new__(cls, /, behavior: str = ...) -> Punctuation: ... @final class Sequence(PreTokenizer): + """ + Runs several pre-tokenizers in order, each one further splitting the + pieces left by the previous. + """ def __new__(cls, /, pre_tokenizers: Sequence2[PreTokenizer]) -> Sequence: ... @final class Split(PreTokenizer): + """ + Splits on a pattern: a literal string, or a regular expression with + `regex=True`. `behavior` says what to do with each match — "removed" drops + it, "isolated" keeps it as its own piece, "merged_with_previous" / + "merged_with_next" glue it to a neighbor, "contiguous" merges runs of + matches. `invert=True` keeps the matches and splits everything else. + """ def __new__(cls, /, pattern: str, behavior: str = ..., *, invert: bool = False, regex: bool = False) -> Split: ... @final class UnicodeScripts(PreTokenizer): + """ + Splits where the script changes (Latin to Han, for example), so a piece never mixes alphabets. + """ def __new__(cls, /) -> UnicodeScripts: ... @final class Whitespace(PreTokenizer): + """ + Splits into runs of letters/digits/underscore or runs of other symbols (the pattern `\w+|[^\w\s]+`). + """ def __new__(cls, /) -> Whitespace: ... @final class WhitespaceSplit(PreTokenizer): + """ + Splits on whitespace only. + """ def __new__(cls, /) -> WhitespaceSplit: ... diff --git a/bindings/python-pipeline/py_src/tokenizers_pipeline/trainers/__init__.py b/bindings/python-pipeline/py_src/tokenizers_pipeline/trainers/__init__.py index c90414b9b..99cc2a3eb 100644 --- a/bindings/python-pipeline/py_src/tokenizers_pipeline/trainers/__init__.py +++ b/bindings/python-pipeline/py_src/tokenizers_pipeline/trainers/__init__.py @@ -1,3 +1,5 @@ +"""Recipes for learning a vocabulary from text.""" + from .._native import trainers as _trainers Trainer = _trainers.Trainer diff --git a/bindings/python-pipeline/py_src/tokenizers_pipeline/trainers/__init__.pyi b/bindings/python-pipeline/py_src/tokenizers_pipeline/trainers/__init__.pyi index a93e342b9..42dfd5207 100644 --- a/bindings/python-pipeline/py_src/tokenizers_pipeline/trainers/__init__.pyi +++ b/bindings/python-pipeline/py_src/tokenizers_pipeline/trainers/__init__.pyi @@ -1,26 +1,53 @@ +""" +Recipes for learning a vocabulary from text. +""" + from tokenizers_pipeline import AddedToken from collections.abc import Sequence from typing import final @final class BpeTrainer(Trainer): + """ + Learns a BPE vocabulary: keeps merging the most frequent pair until + `vocab_size` is reached, ignoring pairs seen fewer than `min_frequency` + times. `special_tokens` get the first ids. `limit_alphabet` caps how many + distinct characters are kept; `initial_alphabet` forces characters in even + if the data never shows them; `max_token_length` caps merged token length. + """ def __new__(cls, /, *, vocab_size: int = 30000, min_frequency: int = 0, special_tokens: Sequence[str |AddedToken] = ..., limit_alphabet: int |None = None, initial_alphabet: Sequence[str] = ..., continuing_subword_prefix: str |None = None, end_of_word_suffix: str |None = None, max_token_length: int |None = None, show_progress: bool = True) -> BpeTrainer: ... class Trainer: """ - Base class for all trainers. A trainer is a plain configuration value: - `Tokenizer.train*` copies it, no state is shared or written back. + Base class for all trainers. + + A trainer is the recipe for learning a model's vocabulary from text; pass + one to `Tokenizer.train` or `train_from_iterator`. Trainers are plain + configuration values — training copies them and writes nothing back. """ def __repr__(self, /) -> str: ... @final class UnigramTrainer(Trainer): + """ + Learns a Unigram vocabulary: starts from a large candidate set and prunes + it by `shrinking_factor` each round until `vocab_size` pieces remain. + `unk_token` names the fallback piece for unknown characters. + """ def __new__(cls, /, *, vocab_size: int = 8000, special_tokens: Sequence[str |AddedToken] = ..., initial_alphabet: Sequence[str] = ..., unk_token: str |None = None, shrinking_factor: float = 0.75, max_piece_length: int = 16, n_sub_iterations: int = 2, show_progress: bool = True) -> UnigramTrainer: ... @final class WordLevelTrainer(Trainer): + """ + Learns a WordLevel vocabulary: the `vocab_size` most frequent words, + keeping only those seen at least `min_frequency` times. + """ def __new__(cls, /, *, vocab_size: int = 30000, min_frequency: int = 0, special_tokens: Sequence[str |AddedToken] = ..., show_progress: bool = True) -> WordLevelTrainer: ... @final class WordPieceTrainer(Trainer): + """ + Learns a WordPiece vocabulary. Same knobs as `BpeTrainer`, plus the + continuation prefix ("##" by default). + """ def __new__(cls, /, *, vocab_size: int = 30000, min_frequency: int = 0, special_tokens: Sequence[str |AddedToken] = ..., limit_alphabet: int |None = None, initial_alphabet: Sequence[str] = ..., continuing_subword_prefix: str = ..., end_of_word_suffix: str |None = None, show_progress: bool = True) -> WordPieceTrainer: ... diff --git a/bindings/python-pipeline/src/added_token.rs b/bindings/python-pipeline/src/added_token.rs index be60c109d..e1136b949 100644 --- a/bindings/python-pipeline/src/added_token.rs +++ b/bindings/python-pipeline/src/added_token.rs @@ -1,7 +1,12 @@ use pyo3::prelude::*; use tk_encode::tokenizer::AddedToken; -/// A token added on top of the model's vocabulary, with its matching options. +/// A token added to the vocabulary after training, with options for how it +/// is matched in text: `single_word` only matches when it stands alone (not +/// inside a word); `lstrip`/`rstrip` also swallow the whitespace before/after +/// it; `normalized` matches against normalized instead of raw text (defaults +/// to the opposite of `special`); `special` marks template tokens like "" +/// that decoding should be able to skip. #[pyclass( frozen, from_py_object, diff --git a/bindings/python-pipeline/src/lib.rs b/bindings/python-pipeline/src/lib.rs index eb230591b..6bcc17ebe 100644 --- a/bindings/python-pipeline/src/lib.rs +++ b/bindings/python-pipeline/src/lib.rs @@ -29,6 +29,7 @@ extern "C" fn child_after_fork() { } } +/// Fast tokenizers built on the pipeline encode path. Start with `Tokenizer`. #[pymodule(gil_used = false)] pub mod _native { use super::*; diff --git a/bindings/python-pipeline/src/models.rs b/bindings/python-pipeline/src/models.rs index 911419037..0e82449d1 100644 --- a/bindings/python-pipeline/src/models.rs +++ b/bindings/python-pipeline/src/models.rs @@ -7,8 +7,11 @@ use tk_encode::models::wordpiece::WordPiece; use crate::error::to_pyerr; -/// Base class for all models. Not constructible from Python; holds the actual -/// Rust model by value (no sharing with the Tokenizer — assignment copies). +/// Base class for all models. +/// +/// The model is the trained part of a tokenizer: it turns each pre-tokenized +/// piece into token ids using its vocabulary. Models are immutable values — +/// assigning one to a tokenizer copies it. #[pyclass( frozen, subclass, @@ -40,6 +43,11 @@ pub fn wrap_model(py: Python<'_>, inner: ModelWrapper) -> PyResult> Ok(obj.unbind()) } +/// Byte-Pair Encoding: builds tokens by applying the merges learned during +/// training. `unk_token` stands in for characters the vocabulary cannot +/// represent; `byte_fallback` encodes them as raw bytes instead. `dropout` +/// randomly skips merges (a training-time regularization). `ignore_merges` +/// looks whole pieces up in the vocabulary before merging. #[pyclass(frozen, extends = PyModel, name = "BPE", module = "tokenizers_pipeline.models")] pub struct PyBPE; @@ -86,6 +94,9 @@ impl PyBPE { } } +/// The BERT model: greedily matches the longest vocabulary entry, marking +/// word continuations with a prefix ("##" by default). A piece longer than +/// `max_input_chars_per_word` becomes `unk_token` outright. #[pyclass(frozen, extends = PyModel, name = "WordPiece", module = "tokenizers_pipeline.models")] pub struct PyWordPiece; @@ -108,6 +119,8 @@ impl PyWordPiece { } } +/// The simplest model: one whole word, one id. Words outside the vocabulary +/// become `unk_token`. #[pyclass(frozen, extends = PyModel, name = "WordLevel", module = "tokenizers_pipeline.models")] pub struct PyWordLevel; @@ -124,6 +137,9 @@ impl PyWordLevel { } } +/// The SentencePiece Unigram model: picks the most probable segmentation +/// under a learned piece vocabulary. Starts empty — train it, or load a +/// tokenizer.json. #[pyclass(frozen, extends = PyModel, name = "Unigram", module = "tokenizers_pipeline.models")] pub struct PyUnigram; @@ -138,6 +154,7 @@ impl PyUnigram { } } +/// The algorithms that turn pre-tokenized pieces into token ids. #[pymodule(gil_used = false)] pub mod models { #[pymodule_export] diff --git a/bindings/python-pipeline/src/normalizers.rs b/bindings/python-pipeline/src/normalizers.rs index c0a6e3243..f7a1d0d7b 100644 --- a/bindings/python-pipeline/src/normalizers.rs +++ b/bindings/python-pipeline/src/normalizers.rs @@ -6,8 +6,11 @@ use tk_encode::normalizers::{ use crate::error::to_pyerr; -/// Base class for all normalizers. Immutable value: assigning it to a -/// Tokenizer copies the configuration, there is no shared state. +/// Base class for all normalizers. +/// +/// A normalizer rewrites text before it is split: cleanup, case-folding, +/// Unicode normalization. Normalizers are immutable values — assigning one to +/// a tokenizer copies it. #[pyclass( frozen, subclass, @@ -61,7 +64,8 @@ pub fn wrap_normalizer(py: Python<'_>, inner: NormalizerWrapper) -> PyResult { + ($pyname:ident, $name:literal, $inner:expr, $doc:literal) => { + #[doc = $doc] #[pyclass(frozen, extends = PyNormalizer, name = $name, module = "tokenizers_pipeline.normalizers")] pub struct $pyname; @@ -75,13 +79,44 @@ macro_rules! unit_normalizer { }; } -unit_normalizer!(PyNFC, "NFC", NFC); -unit_normalizer!(PyNFD, "NFD", NFD); -unit_normalizer!(PyNFKC, "NFKC", NFKC); -unit_normalizer!(PyNFKD, "NFKD", NFKD); -unit_normalizer!(PyLowercase, "Lowercase", Lowercase); -unit_normalizer!(PyStripAccents, "StripAccents", StripAccents); - +unit_normalizer!( + PyNFC, + "NFC", + NFC, + "Unicode NFC: recombines split characters (e + ´ becomes é)." +); +unit_normalizer!( + PyNFD, + "NFD", + NFD, + "Unicode NFD: splits characters into base + accents (é becomes e + ´)." +); +unit_normalizer!( + PyNFKC, + "NFKC", + NFKC, + "Unicode NFKC: NFC, plus compatibility replacements (fi becomes fi)." +); +unit_normalizer!( + PyNFKD, + "NFKD", + NFKD, + "Unicode NFKD: NFD, plus compatibility replacements (fi becomes fi)." +); +unit_normalizer!( + PyLowercase, + "Lowercase", + Lowercase, + "Lowercases everything." +); +unit_normalizer!( + PyStripAccents, + "StripAccents", + StripAccents, + "Removes accents (é becomes e). Only works on decomposed text: put NFD before it." +); + +/// Removes whitespace at the start and/or end of the text. #[pyclass(frozen, extends = PyNormalizer, name = "Strip", module = "tokenizers_pipeline.normalizers")] pub struct PyStrip; @@ -97,6 +132,8 @@ impl PyStrip { } } +/// Replaces every occurrence of `pattern` with `content`. With `regex=True` +/// the pattern is a regular expression. #[pyclass(frozen, extends = PyNormalizer, name = "Replace", module = "tokenizers_pipeline.normalizers")] pub struct PyReplace; @@ -119,6 +156,7 @@ impl PyReplace { } } +/// Puts a fixed string in front of the text (SentencePiece prepends "▁"). #[pyclass(frozen, extends = PyNormalizer, name = "Prepend", module = "tokenizers_pipeline.normalizers")] pub struct PyPrepend; @@ -133,6 +171,10 @@ impl PyPrepend { } } +/// The BERT cleanup: removes control characters, puts spaces around CJK +/// characters, and optionally strips accents and lowercases. +/// `strip_accents=None` means "follow the lowercase setting", like the +/// original BERT. #[pyclass(frozen, extends = PyNormalizer, name = "BertNormalizer", module = "tokenizers_pipeline.normalizers")] pub struct PyBertNormalizer; @@ -154,6 +196,7 @@ impl PyBertNormalizer { } } +/// Runs several normalizers in order. #[pyclass(frozen, extends = PyNormalizer, name = "Sequence", module = "tokenizers_pipeline.normalizers")] pub struct PySequence; @@ -169,6 +212,7 @@ impl PySequence { } } +/// Text cleanup that runs before the text is split. #[pymodule(gil_used = false)] pub mod normalizers { #[pymodule_export] diff --git a/bindings/python-pipeline/src/pre_tokenizers.rs b/bindings/python-pipeline/src/pre_tokenizers.rs index 010d4bda9..d9343dd5b 100644 --- a/bindings/python-pipeline/src/pre_tokenizers.rs +++ b/bindings/python-pipeline/src/pre_tokenizers.rs @@ -29,11 +29,12 @@ pub fn parse_behavior(s: &str) -> PyResult { } } -/// Base class for all pre-tokenizers. Immutable value: assigning it to a -/// Tokenizer copies the configuration, there is no shared state. +/// Base class for all pre-tokenizers. /// -/// Only pre-tokenizers supported by the encode pipeline are constructible here; -/// notably `Metaspace` is not available yet. +/// A pre-tokenizer cuts text into pieces (usually words); the model then turns +/// each piece into token ids. Pre-tokenizers are immutable values — assigning +/// one to a tokenizer copies it. Only pre-tokenizers the encode pipeline can +/// run are constructible here; `Metaspace` is not available yet. #[pyclass( frozen, subclass, @@ -97,7 +98,8 @@ pub fn wrap_pre_tokenizer( } macro_rules! unit_pre_tokenizer { - ($pyname:ident, $name:literal, $inner:expr) => { + ($pyname:ident, $name:literal, $inner:expr, $doc:literal) => { + #[doc = $doc] #[pyclass(frozen, extends = PyPreTokenizer, name = $name, module = "tokenizers_pipeline.pre_tokenizers")] pub struct $pyname; @@ -111,17 +113,39 @@ macro_rules! unit_pre_tokenizer { }; } -unit_pre_tokenizer!(PyWhitespace, "Whitespace", Whitespace); -unit_pre_tokenizer!(PyWhitespaceSplit, "WhitespaceSplit", WhitespaceSplit); -unit_pre_tokenizer!(PyBertPreTokenizer, "BertPreTokenizer", BertPreTokenizer); -unit_pre_tokenizer!(PyUnicodeScripts, "UnicodeScripts", UnicodeScripts); +unit_pre_tokenizer!( + PyWhitespace, + "Whitespace", + Whitespace, + "Splits into runs of letters/digits/underscore or runs of other symbols (the pattern `\\w+|[^\\w\\s]+`)." +); +unit_pre_tokenizer!( + PyWhitespaceSplit, + "WhitespaceSplit", + WhitespaceSplit, + "Splits on whitespace only." +); +unit_pre_tokenizer!( + PyBertPreTokenizer, + "BertPreTokenizer", + BertPreTokenizer, + "The BERT split: on whitespace, and each punctuation character becomes its own piece." +); +unit_pre_tokenizer!( + PyUnicodeScripts, + "UnicodeScripts", + UnicodeScripts, + "Splits where the script changes (Latin to Han, for example), so a piece never mixes alphabets." +); +/// GPT-2 style byte-level splitting: cuts with the GPT-2 regex unless +/// `use_regex=False`. The pipeline does not support `add_prefix_space`, so it +/// is always off. #[pyclass(frozen, extends = PyPreTokenizer, name = "ByteLevel", module = "tokenizers_pipeline.pre_tokenizers")] pub struct PyByteLevel; #[pymethods] impl PyByteLevel { - /// `add_prefix_space` is not supported by the pipeline and is always false. #[new] #[pyo3(signature = (*, use_regex = true))] fn new(use_regex: bool) -> PyClassInitializer { @@ -132,6 +156,7 @@ impl PyByteLevel { } } +/// Splits on one fixed character, dropping it. #[pyclass(frozen, extends = PyPreTokenizer, name = "CharDelimiterSplit", module = "tokenizers_pipeline.pre_tokenizers")] pub struct PyCharDelimiterSplit; @@ -146,6 +171,8 @@ impl PyCharDelimiterSplit { } } +/// Separates digits from everything else. With `individual_digits=True`, +/// every digit becomes its own piece. #[pyclass(frozen, extends = PyPreTokenizer, name = "Digits", module = "tokenizers_pipeline.pre_tokenizers")] pub struct PyDigits; @@ -161,6 +188,8 @@ impl PyDigits { } } +/// Cuts the text into pieces of exactly `length` characters (the last one may +/// be shorter). #[pyclass(frozen, extends = PyPreTokenizer, name = "FixedLength", module = "tokenizers_pipeline.pre_tokenizers")] pub struct PyFixedLength; @@ -176,6 +205,8 @@ impl PyFixedLength { } } +/// Splits on punctuation. `behavior` says what happens to the punctuation +/// itself — see `Split` for the options. #[pyclass(frozen, extends = PyPreTokenizer, name = "Punctuation", module = "tokenizers_pipeline.pre_tokenizers")] pub struct PyPunctuation; @@ -191,6 +222,11 @@ impl PyPunctuation { } } +/// Splits on a pattern: a literal string, or a regular expression with +/// `regex=True`. `behavior` says what to do with each match — "removed" drops +/// it, "isolated" keeps it as its own piece, "merged_with_previous" / +/// "merged_with_next" glue it to a neighbor, "contiguous" merges runs of +/// matches. `invert=True` keeps the matches and splits everything else. #[pyclass(frozen, extends = PyPreTokenizer, name = "Split", module = "tokenizers_pipeline.pre_tokenizers")] pub struct PySplit; @@ -217,6 +253,8 @@ impl PySplit { } } +/// Runs several pre-tokenizers in order, each one further splitting the +/// pieces left by the previous. #[pyclass(frozen, extends = PyPreTokenizer, name = "Sequence", module = "tokenizers_pipeline.pre_tokenizers")] pub struct PySequence; @@ -233,6 +271,7 @@ impl PySequence { } } +/// How text is cut into pieces before the model runs. #[pymodule(gil_used = false)] pub mod pre_tokenizers { #[pymodule_export] diff --git a/bindings/python-pipeline/src/tokenizer.rs b/bindings/python-pipeline/src/tokenizer.rs index 437452166..46371d2de 100644 --- a/bindings/python-pipeline/src/tokenizer.rs +++ b/bindings/python-pipeline/src/tokenizer.rs @@ -54,8 +54,13 @@ fn poisoned(_: std::sync::PoisonError) -> PyErr { } /// A tokenizer: a model plus its optional normalizer and pre-tokenizer. -/// Mutations apply to the serializable definition; encode runs a compiled -/// pipeline that is rebuilt automatically after any change. +/// +/// Create one from a model (`Tokenizer(models.BPE())`), a file +/// (`Tokenizer.from_file`), or the Hub (`Tokenizer.from_pretrained`). +/// Changes — assigning components, training, adding tokens — apply to the +/// serializable definition; encoding runs a compiled pipeline that is rebuilt +/// automatically after any change. A definition the pipeline cannot run +/// raises `TokenizersError` at that point, with the reason. // The lock/GIL ordering rule (never block on the lock while attached) is // enforced by DetachedRwLock: guards are only reachable inside its // detach-first `with` closure. See detached_lock.rs for the rationale and @@ -159,6 +164,7 @@ fn encode_one( #[pymethods] impl PyTokenizer { + /// Create an untrained tokenizer from a model. #[new] fn new(model: PyRef<'_, PyModel>) -> Self { Self::from_spec(SpecTokenizer::new(model.inner.clone())) @@ -290,7 +296,8 @@ impl PyTokenizer { )) } - /// Train the model on text files (one sequence per line). + /// Train the model's vocabulary on text files (one sequence per line). + /// Without a `trainer`, the model's default trainer is used. #[pyo3(signature = (files, *, trainer = None))] fn train( &self, @@ -311,7 +318,8 @@ impl PyTokenizer { }) } - /// Train the model from any iterator of `str`. + /// Train the model's vocabulary from any iterator of `str`. Without a + /// `trainer`, the model's default trainer is used. /// /// The interpreter lock is only re-acquired to refill an internal buffer /// (256 sequences at a time); the training itself runs multi-threaded in @@ -343,13 +351,17 @@ impl PyTokenizer { Ok(()) } - /// Add tokens to the vocabulary, matched literally in the input text. + /// Add tokens to the vocabulary and match them in the input text from now + /// on. Plain strings match with default options; pass `AddedToken` to + /// control matching. Returns how many were actually new. fn add_tokens(&self, py: Python<'_>, tokens: Vec) -> PyResult { let tokens = parse_tokens(tokens, false); self.mutate_spec(py, move |spec| spec.add_tokens(tokens).map_err(to_pyerr)) } - /// Add special tokens (never split, skipped on decode) to the vocabulary. + /// Add special tokens ("", "[CLS]", …) to the vocabulary. Same as + /// `add_tokens`, but every token is marked `special`. Returns how many + /// were actually new. fn add_special_tokens(&self, py: Python<'_>, tokens: Vec) -> PyResult { let tokens = parse_tokens(tokens, true); self.mutate_spec(py, move |spec| { @@ -357,10 +369,12 @@ impl PyTokenizer { }) } + /// The id of `token`, or None if it is not in the vocabulary. fn token_to_id(&self, py: Python<'_>, token: &str) -> PyResult> { self.read_spec(py, |spec| spec.token_to_id(token)) } + /// The token behind `id`, or None if the id is out of range. fn id_to_token(&self, py: Python<'_>, id: u32) -> PyResult> { self.read_spec(py, move |spec| spec.id_to_token(id)) } @@ -372,6 +386,8 @@ impl PyTokenizer { self.read_spec(py, move |spec| spec.get_vocab(with_added_tokens)) } + /// Number of entries in the vocabulary. `with_added_tokens=False` counts + /// only what the model was trained with. #[pyo3(signature = (*, with_added_tokens = true))] fn get_vocab_size(&self, py: Python<'_>, with_added_tokens: bool) -> PyResult { self.read_spec(py, move |spec| spec.get_vocab_size(with_added_tokens)) diff --git a/bindings/python-pipeline/src/trainers.rs b/bindings/python-pipeline/src/trainers.rs index f523eb51f..1edfd094b 100644 --- a/bindings/python-pipeline/src/trainers.rs +++ b/bindings/python-pipeline/src/trainers.rs @@ -6,8 +6,11 @@ use tk_train::trainers::{ use crate::added_token::{TokenInput, parse_tokens}; use crate::error::to_pyerr; -/// Base class for all trainers. A trainer is a plain configuration value: -/// `Tokenizer.train*` copies it, no state is shared or written back. +/// Base class for all trainers. +/// +/// A trainer is the recipe for learning a model's vocabulary from text; pass +/// one to `Tokenizer.train` or `train_from_iterator`. Trainers are plain +/// configuration values — training copies them and writes nothing back. #[pyclass( frozen, subclass, @@ -25,6 +28,11 @@ impl PyTrainer { } } +/// Learns a BPE vocabulary: keeps merging the most frequent pair until +/// `vocab_size` is reached, ignoring pairs seen fewer than `min_frequency` +/// times. `special_tokens` get the first ids. `limit_alphabet` caps how many +/// distinct characters are kept; `initial_alphabet` forces characters in even +/// if the data never shows them; `max_token_length` caps merged token length. #[pyclass(frozen, extends = PyTrainer, name = "BpeTrainer", module = "tokenizers_pipeline.trainers")] pub struct PyBpeTrainer; @@ -69,6 +77,8 @@ impl PyBpeTrainer { } } +/// Learns a WordPiece vocabulary. Same knobs as `BpeTrainer`, plus the +/// continuation prefix ("##" by default). #[pyclass(frozen, extends = PyTrainer, name = "WordPieceTrainer", module = "tokenizers_pipeline.trainers")] pub struct PyWordPieceTrainer; @@ -107,6 +117,9 @@ impl PyWordPieceTrainer { } } +/// Learns a Unigram vocabulary: starts from a large candidate set and prunes +/// it by `shrinking_factor` each round until `vocab_size` pieces remain. +/// `unk_token` names the fallback piece for unknown characters. #[pyclass(frozen, extends = PyTrainer, name = "UnigramTrainer", module = "tokenizers_pipeline.trainers")] pub struct PyUnigramTrainer; @@ -143,6 +156,8 @@ impl PyUnigramTrainer { } } +/// Learns a WordLevel vocabulary: the `vocab_size` most frequent words, +/// keeping only those seen at least `min_frequency` times. #[pyclass(frozen, extends = PyTrainer, name = "WordLevelTrainer", module = "tokenizers_pipeline.trainers")] pub struct PyWordLevelTrainer; @@ -170,6 +185,7 @@ impl PyWordLevelTrainer { } } +/// Recipes for learning a vocabulary from text. #[pymodule(gil_used = false)] pub mod trainers { #[pymodule_export] From fcd5c82bb9e943406079be93ce4b884bcc3611e8 Mon Sep 17 00:00:00 2001 From: SBrandeis <33657802+SBrandeis@users.noreply.github.com> Date: Wed, 22 Jul 2026 12:38:00 +0200 Subject: [PATCH 06/19] render python compare graph --- .github/scripts/render_python_bench.py | 180 +++++++++++++++++++++++++ .github/workflows/pipeline-bench.yml | 49 ++++++- bindings/python-pipeline/README.md | 4 +- 3 files changed, 229 insertions(+), 4 deletions(-) create mode 100644 .github/scripts/render_python_bench.py diff --git a/.github/scripts/render_python_bench.py b/.github/scripts/render_python_bench.py new file mode 100644 index 000000000..d8dd0bbc0 --- /dev/null +++ b/.github/scripts/render_python_bench.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +"""Render the Python-bindings bench JSON (bench_vs_release.py) as a chart + +markdown section for the PR description. + +A sibling of render_pipeline_bench.py's overview chart, reusing its helpers so +the two read as one report: one row per model, log-scale ×speedup vs the +released `tokenizers` wheel (×1.0), two bars per row — single-thread (geomean +across fixtures, with a min–max whisker) and multi-thread (one sweep over the +flattened corpus). Skipped models render as muted rows. Emits +pipeline_bench_pybindings.svg (picked up by the workflow's existing +rasterize/upload globs) and python_bench_section.md (appended to +pipeline_bench.md before the PR upsert). +""" +import argparse +import json +import re +from html import escape +from pathlib import Path + +import render_pipeline_bench as rpb + +# Same-system series colors: pipeline blue for single-thread, the catalog teal +# for multi-thread (CVD ΔE 16.8 protan, both >= 3:1 on the chart surface). +PY_SINK = {"st": "#2a78d6", "mt": "#2a9d8f"} + +SLUG = "pybindings" +ROW_H, BAR_H, GAP = 58, 12, 2 + + +def st_speedups(model): + return [f["speedup"] for f in model.get("fixtures", [])] + + +def agg_mbps(fixtures): + """Byte-weighted single-thread throughput across all fixtures.""" + nbytes = sum(f["bytes"] for f in fixtures) + t = sum(f["bytes"] / (f["pipeline_mbps"] * 1e6) for f in fixtures) + return nbytes / t / 1e6 + + +def skip_reason(message): + """Short form of the compile error ("no Metaspace pre-tokenizer"); the raw + message embeds Rust debug output too noisy for a chart row.""" + m = re.search(r"does not support PreTokenizer: (\w+)", message) + if m: + return f"no {m.group(1)} pre-tokenizer" + return message if len(message) <= 60 else message[:57] + "…" + + +def chart_svg(report, meta): + ink, sink = rpb.INK, PY_SINK + models = report["models"] + ref = f"tokenizers {report['release_version']} (PyPI)" + + vals = [v for m in models for v in st_speedups(m)] + vals += [m["multi_thread"]["speedup"] for m in models if "multi_thread" in m] + lo = min(0.75, min(vals) / 1.08) if vals else 0.75 + hi = max(1.5, max(vals) * 1.08) if vals else 1.5 + x = rpb.log_x(rpb.OV_GUTTER, rpb.OV_PLOT, lo, hi) + ticks = rpb.thin_ticks([t for t in rpb.TICKS if lo <= t <= hi], x, min_px=34, keep=1.0) + + top = 74 + col_x = rpb.CHART_W - 16 + body = [f'fixtures · ids'] + y = top + for m in models: + cy = y + ROW_H / 2 + body.append(f'{escape(m["model"])}') + if "skipped" in m: + body.append(f'not supported — ' + f'{escape(skip_reason(m["skipped"]))}') + body.append(f'—') + y += ROW_H + continue + + st = st_speedups(m) + g, mn, mx = rpb.geomean(st), min(st), max(st) + y_st = cy - BAR_H - GAP / 2 + y_mt = cy + GAP / 2 + body.append(rpb.hbar(x(1.0), x(g), y_st, BAR_H, sink["st"])) + wy = y_st + BAR_H / 2 + body.append(f'') + for v in (mn, mx): + body.append(f'') + body.append(f'×{g:.2f}' + f' ' + f'· {agg_mbps(m["fixtures"]):.0f} MB/s') + + mt = m["multi_thread"]["speedup"] + body.append(rpb.hbar(x(1.0), x(mt), y_mt, BAR_H, sink["mt"])) + body.append(f'×{mt:.2f}' + f' ' + f'· {m["multi_thread"]["pipeline_mbps"]:.0f} MB/s') + + bad = sum(1 for f in m["fixtures"] if not f["ids_match"]) + right, fill = ((f"⚠ {bad} differ", ink["critical"]) if bad + else (f'{len(m["fixtures"])} · ids ok', ink["secondary"])) + body.append(f'{right}') + y += ROW_H + + axis = rpb.speedup_axis(ink, x, ticks, top, y + 4) + y += 30 + legend = rpb.legend_row(ink, sink, y, [ + ("swatch", "st", "single thread (geomean, min–max whisker)"), + ("swatch", "mt", "all threads"), + ("tick", ink["baseline"], f"×1.0 = {ref}"), + ]) + height = y + 34 + subtitle = (f"×speedup per model vs {ref} · timed end-to-end through Python · " + "~10 KiB fixture chunks") + return rpb.svg_doc(ink, height, "Python bindings vs released wheel — encode_batch", + subtitle, axis + "".join(body) + legend, meta) + + +def section_md(report, img_base, run_id): + ref = f"`tokenizers` {report['release_version']} (PyPI)" + lines = [ + f"### Python bindings — `tokenizers_pipeline` vs {ref}", + "", + rpb.picture(img_base, run_id, SLUG, "Python bindings encode_batch speedup " + "vs the released tokenizers wheel", 860), + "", + "
numbers (MB/s)", "", + "| model | pipeline 1t | release 1t | speedup 1t | pipeline mt | release mt | speedup mt |", + "|---|---|---|---|---|---|---|", + ] + for m in report["models"]: + if "skipped" in m: + lines.append(f"| {m['model']} | not supported | | | | | |") + continue + fx = m["fixtures"] + nbytes = sum(f["bytes"] for f in fx) + t_ours = sum(f["bytes"] / (f["pipeline_mbps"] * 1e6) for f in fx) + t_theirs = sum(f["bytes"] / (f["release_mbps"] * 1e6) for f in fx) + mt = m["multi_thread"] + lines.append( + f"| {m['model']} | {nbytes / t_ours / 1e6:.0f} | {nbytes / t_theirs / 1e6:.0f} " + f"| {t_theirs / t_ours:.2f}× " + f"| {mt['pipeline_mbps']:.0f} | {mt['release_mbps']:.0f} | {mt['speedup']:.2f}× |" + ) + lines += ["", "
", ""] + return "\n".join(lines) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("bench_json", type=Path) + parser.add_argument("--img-base", default="") + parser.add_argument("--run-id", default="") + parser.add_argument("--out-dir", type=Path, default=Path(".")) + args = parser.parse_args() + + report = json.loads(args.bench_json.read_text()) + meta = [ + f"tokenizers_pipeline {report['pipeline_version']} vs " + f"tokenizers {report['release_version']}", + f"{report['cpus']} CPUs · median of {report['iters']} runs · " + f"{report['fixture_count']} fixtures", + ] + out = args.out_dir / f"pipeline_bench_{SLUG}.svg" + out.write_text(chart_svg(report, meta)) + print(f"wrote {out}") + out = args.out_dir / "python_bench_section.md" + out.write_text(section_md(report, args.img_base, args.run_id)) + print(f"wrote {out}") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/pipeline-bench.yml b/.github/workflows/pipeline-bench.yml index 9257000a4..f0ef7e2d0 100644 --- a/.github/workflows/pipeline-bench.yml +++ b/.github/workflows/pipeline-bench.yml @@ -241,6 +241,9 @@ jobs: if: github.event_name != 'pull_request' || github.event.label.name == 'run-pipeline-bench' runs-on: group: aws-general-8-plus + env: + RUSTC_WRAPPER: sccache + SCCACHE_GHA_ENABLED: "true" steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 @@ -253,6 +256,12 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v6 + - name: Cache fixtures + model tokenizers + uses: actions/cache@v4 + with: + path: tokenizers/data + key: bench-data-${{ hashFiles('tokenizers/Makefile', 'tokenizers/tk-encode/examples/bench_models.json') }} + - name: Download fixtures and model tokenizers working-directory: tokenizers env: @@ -285,11 +294,15 @@ jobs: retention-days: 30 # Fan-in: concatenate the shard partials (in shard order = manifest order), - # then render + upload charts + update the PR, once. + # then render + upload charts + update the PR, once. Also folds in the Python + # bindings bench (chart + numbers) when its job produced results — but never + # blocks on it: a failed/missing Python bench only drops that section. report: name: PipelineTokenizer vs latest release - needs: bench - if: github.event_name != 'pull_request' || github.event.label.name == 'run-pipeline-bench' + needs: [bench, python-bindings-bench] + if: >- + !cancelled() && needs.bench.result == 'success' && + (github.event_name != 'pull_request' || github.event.label.name == 'run-pipeline-bench') runs-on: group: aws-general-8-plus defaults: @@ -427,6 +440,28 @@ jobs: echo "No base baseline available (no tag, no base-branch artifact) — comparison skipped." echo "has_base=false" >> "$GITHUB_OUTPUT" + # The Python bindings section: render its chart before "Render report" so + # the shared rasterize glob below picks it up. Missing artifact (failed or + # skipped python-bindings-bench) → section skipped, report unaffected. + - name: Download Python bindings bench results + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: python-bindings-bench + path: tokenizers/pybench + + - name: Render Python bindings section + run: | + if [ -f pybench/python_bench.json ]; then + base_url="https://huggingface.co/datasets/hf-internal-testing/tokenizers-bench/resolve/main/charts" + python3 ${{ github.workspace }}/.github/scripts/render_python_bench.py \ + pybench/python_bench.json \ + --img-base "$base_url" \ + --run-id "${{ github.run_id }}" + else + echo "No Python bindings bench results — section skipped." + fi + - name: Render report run: | base_url="https://huggingface.co/datasets/hf-internal-testing/tokenizers-bench/resolve/main/charts" @@ -465,6 +500,14 @@ jobs: charts_upload charts --repo-type dataset \ --commit-message "charts: pipeline-bench run ${{ github.run_id }}" + # Fold the Python bindings section into the report so the step summary and + # the PR-description upsert below carry it inside the same marker block. + - name: Append Python bindings section to the report + run: | + if [ -f python_bench_section.md ]; then + cat python_bench_section.md >> pipeline_bench.md + fi + - name: Write step summary run: cat pipeline_bench.md >> "$GITHUB_STEP_SUMMARY" diff --git a/bindings/python-pipeline/README.md b/bindings/python-pipeline/README.md index c280c6ce7..63d23ef02 100644 --- a/bindings/python-pipeline/README.md +++ b/bindings/python-pipeline/README.md @@ -91,4 +91,6 @@ chunking as the Rust benchmark (`tk-encode/examples/fixture_bench.rs`): every fixture under `data/fixtures/{lang,modalities}`, warmed up, median of N runs, single-thread per fixture plus one multi-thread sweep, ids verified equal before timing. CI runs it in the `python-bindings-bench` job of the Pipeline -Benchmark workflow and posts the table to the run's step summary. +Benchmark workflow, posts the table to the run's step summary, and the report +job renders it as a chart (`.github/scripts/render_python_bench.py`) appended +to the benchmark section in the PR description, next to the Rust charts. From ea3fb02c899818dc05898442e230b84c2f9247a2 Mon Sep 17 00:00:00 2001 From: SBrandeis <33657802+SBrandeis@users.noreply.github.com> Date: Wed, 22 Jul 2026 13:53:19 +0200 Subject: [PATCH 07/19] fold in existing package --- .github/scripts/render_python_bench.py | 6 +- .github/workflows/benchmarks.yml | 141 - .github/workflows/build_documentation.yml | 14 +- .github/workflows/build_pr_documentation.yml | 5 +- .github/workflows/docs-check.yml | 8 +- .github/workflows/pipeline-bench.yml | 33 +- .github/workflows/python-release.yml | 40 +- .github/workflows/python.yml | 162 +- README.md | 15 +- bindings/python-pipeline/Cargo.lock | 1707 ------------ bindings/python-pipeline/Cargo.toml | 24 - bindings/python-pipeline/Makefile | 36 - bindings/python-pipeline/README.md | 96 - .../py_src/tokenizers_pipeline/__init__.py | 20 - .../py_src/tokenizers_pipeline/__init__.pyi | 173 -- .../tokenizers_pipeline/models/__init__.py | 11 - .../normalizers/__init__.py | 31 - .../pre_tokenizers/__init__.py | 31 - .../py_src/tokenizers_pipeline/py.typed | 0 .../tokenizers_pipeline/trainers/__init__.py | 17 - .../tokenizers_pipeline/trainers/__init__.pyi | 53 - bindings/python-pipeline/pyproject.toml | 19 - bindings/python-pipeline/src/error.rs | 9 - bindings/python-pipeline/src/lib.rs | 69 - bindings/python-pipeline/src/models.rs | 162 -- bindings/python-pipeline/src/normalizers.rs | 223 -- .../python-pipeline/src/pre_tokenizers.rs | 283 -- bindings/python-pipeline/src/tokenizer.rs | 542 ---- bindings/python-pipeline/src/trainers.rs | 195 -- .../python-pipeline/tools/stub-gen/Cargo.lock | 178 -- .../python-pipeline/tools/stub-gen/Cargo.toml | 10 - .../tools/stub-gen/src/main.rs | 122 - bindings/python/.cargo/config.toml | 13 - bindings/python/.gitignore | 5 +- bindings/python/CHANGELOG.md | 512 ---- bindings/python/Cargo.lock | 447 +--- bindings/python/Cargo.toml | 41 +- bindings/python/MANIFEST.in | 7 - bindings/python/Makefile | 108 +- bindings/python/README.md | 321 +-- .../benches/bench_vs_release.py | 174 +- bindings/python/benches/test_tiktoken.py | 128 - .../{python-pipeline => python}/clippy.toml | 0 bindings/python/conftest.py | 19 - bindings/python/docs/pyo3.md | 7 - .../examples/01_train_and_encode.py | 2 +- .../examples/02_pretrained.py | 28 +- .../examples/03_threading.py | 2 +- bindings/python/examples/custom_components.py | 79 - bindings/python/examples/example.py | 138 - .../python/examples/train_bert_wordpiece.py | 52 - .../python/examples/train_bytelevel_bpe.py | 56 - .../python/examples/train_with_datasets.py | 23 - .../examples/using_the_visualizer.ipynb | 1056 -------- bindings/python/py_src/tokenizers/__init__.py | 121 +- .../python/py_src/tokenizers/__init__.pyi | 1351 +--------- .../python/py_src/tokenizers/decoders.pyi | 372 --- .../py_src/tokenizers/decoders/__init__.py | 15 - .../tokenizers/implementations/__init__.py | 6 - .../implementations/base_tokenizer.py | 477 ---- .../implementations/bert_wordpiece.py | 151 -- .../implementations/byte_level_bpe.py | 122 - .../implementations/char_level_bpe.py | 150 -- .../implementations/sentencepiece_bpe.py | 103 - .../implementations/sentencepiece_unigram.py | 196 -- bindings/python/py_src/tokenizers/models.pyi | 421 --- .../py_src/tokenizers/models/__init__.py | 16 +- .../py_src/tokenizers}/models/__init__.pyi | 0 .../python/py_src/tokenizers/normalizers.pyi | 399 --- .../py_src/tokenizers/normalizers/__init__.py | 54 +- .../tokenizers}/normalizers/__init__.pyi | 0 .../py_src/tokenizers/pre_tokenizers.pyi | 385 --- .../tokenizers/pre_tokenizers/__init__.py | 44 +- .../tokenizers}/pre_tokenizers/__init__.pyi | 0 .../python/py_src/tokenizers/processors.pyi | 293 --- .../py_src/tokenizers/processors/__init__.py | 10 - .../python/py_src/tokenizers/tokenizers.pyi | 17 - .../py_src/tokenizers/tools/__init__.py | 1 - .../tokenizers/tools/visualizer-styles.css | 170 -- .../py_src/tokenizers/tools/visualizer.py | 420 --- .../python/py_src/tokenizers/trainers.pyi | 399 --- .../py_src/tokenizers/trainers/__init__.py | 23 +- .../py_src/tokenizers/trainers/__init__.pyi | 729 +----- bindings/python/pyproject.toml | 67 +- bindings/python/pytest.ini | 3 - bindings/python/rust-toolchain | 1 - bindings/python/scripts/convert.py | 417 --- .../python/scripts/sentencepiece_extractor.py | 145 -- bindings/python/scripts/spm_parity_check.py | 264 -- bindings/python/setup.cfg | 55 - .../src/added_token.rs | 7 +- bindings/python/src/decoders.rs | 933 ------- .../src/detached_lock.rs | 0 bindings/python/src/encoding.rs | 473 ---- bindings/python/src/error.rs | 45 +- bindings/python/src/lib.rs | 90 +- bindings/python/src/models.rs | 1116 +------- bindings/python/src/normalizers.rs | 1202 ++------- bindings/python/src/pre_tokenizers.rs | 1286 ++------- bindings/python/src/processors.rs | 932 ------- bindings/python/src/token.rs | 50 - bindings/python/src/tokenizer.rs | 2318 +++-------------- bindings/python/src/trainers.rs | 1875 +------------ bindings/python/src/utils/iterators.rs | 134 - bindings/python/src/utils/mod.rs | 75 - bindings/python/src/utils/normalization.rs | 604 ----- bindings/python/src/utils/pretokenization.rs | 335 --- bindings/python/src/utils/regex.rs | 23 - bindings/python/src/utils/serde_pyo3.rs | 773 ------ bindings/python/test.txt | 36 - bindings/python/tests/__init__.py | 0 bindings/python/tests/bindings/__init__.py | 0 .../python/tests/bindings/test_decoders.py | 228 -- .../python/tests/bindings/test_encoding.py | 122 - bindings/python/tests/bindings/test_models.py | 121 - .../python/tests/bindings/test_normalizers.py | 237 -- .../tests/bindings/test_pre_tokenizers.py | 362 --- .../python/tests/bindings/test_processors.py | 256 -- .../python/tests/bindings/test_tokenizer.py | 1107 -------- .../python/tests/bindings/test_trainers.py | 405 --- .../python/tests/documentation/__init__.py | 0 .../tests/documentation/test_pipeline.py | 190 -- .../tests/documentation/test_quicktour.py | 199 -- .../test_tutorial_train_from_iterators.py | 106 - .../python/tests/implementations/__init__.py | 0 .../implementations/test_base_tokenizer.py | 30 - .../implementations/test_bert_wordpiece.py | 56 - .../implementations/test_byte_level_bpe.py | 103 - .../tests/implementations/test_char_bpe.py | 63 - .../implementations/test_sentencepiece.py | 61 - bindings/python/tests/test_benchmarks.py | 204 -- bindings/python/tests/test_freethreaded.py | 163 -- bindings/python/tests/test_serialization.py | 153 -- bindings/python/tests/utils.py | 105 - bindings/python/tools/stub-gen/Cargo.lock | 329 +-- bindings/python/tools/stub-gen/Cargo.toml | 12 +- bindings/python/tools/stub-gen/src/main.rs | 338 +-- 137 files changed, 1973 insertions(+), 30334 deletions(-) delete mode 100644 bindings/python-pipeline/Cargo.lock delete mode 100644 bindings/python-pipeline/Cargo.toml delete mode 100644 bindings/python-pipeline/Makefile delete mode 100644 bindings/python-pipeline/README.md delete mode 100644 bindings/python-pipeline/py_src/tokenizers_pipeline/__init__.py delete mode 100644 bindings/python-pipeline/py_src/tokenizers_pipeline/__init__.pyi delete mode 100644 bindings/python-pipeline/py_src/tokenizers_pipeline/models/__init__.py delete mode 100644 bindings/python-pipeline/py_src/tokenizers_pipeline/normalizers/__init__.py delete mode 100644 bindings/python-pipeline/py_src/tokenizers_pipeline/pre_tokenizers/__init__.py delete mode 100644 bindings/python-pipeline/py_src/tokenizers_pipeline/py.typed delete mode 100644 bindings/python-pipeline/py_src/tokenizers_pipeline/trainers/__init__.py delete mode 100644 bindings/python-pipeline/py_src/tokenizers_pipeline/trainers/__init__.pyi delete mode 100644 bindings/python-pipeline/pyproject.toml delete mode 100644 bindings/python-pipeline/src/error.rs delete mode 100644 bindings/python-pipeline/src/lib.rs delete mode 100644 bindings/python-pipeline/src/models.rs delete mode 100644 bindings/python-pipeline/src/normalizers.rs delete mode 100644 bindings/python-pipeline/src/pre_tokenizers.rs delete mode 100644 bindings/python-pipeline/src/tokenizer.rs delete mode 100644 bindings/python-pipeline/src/trainers.rs delete mode 100644 bindings/python-pipeline/tools/stub-gen/Cargo.lock delete mode 100644 bindings/python-pipeline/tools/stub-gen/Cargo.toml delete mode 100644 bindings/python-pipeline/tools/stub-gen/src/main.rs delete mode 100644 bindings/python/.cargo/config.toml delete mode 100644 bindings/python/CHANGELOG.md delete mode 100644 bindings/python/MANIFEST.in rename bindings/{python-pipeline => python}/benches/bench_vs_release.py (50%) delete mode 100755 bindings/python/benches/test_tiktoken.py rename bindings/{python-pipeline => python}/clippy.toml (100%) delete mode 100644 bindings/python/conftest.py delete mode 100644 bindings/python/docs/pyo3.md rename bindings/{python-pipeline => python}/examples/01_train_and_encode.py (96%) rename bindings/{python-pipeline => python}/examples/02_pretrained.py (59%) rename bindings/{python-pipeline => python}/examples/03_threading.py (98%) delete mode 100644 bindings/python/examples/custom_components.py delete mode 100644 bindings/python/examples/example.py delete mode 100644 bindings/python/examples/train_bert_wordpiece.py delete mode 100644 bindings/python/examples/train_bytelevel_bpe.py delete mode 100644 bindings/python/examples/train_with_datasets.py delete mode 100644 bindings/python/examples/using_the_visualizer.ipynb delete mode 100644 bindings/python/py_src/tokenizers/decoders.pyi delete mode 100644 bindings/python/py_src/tokenizers/decoders/__init__.py delete mode 100644 bindings/python/py_src/tokenizers/implementations/__init__.py delete mode 100644 bindings/python/py_src/tokenizers/implementations/base_tokenizer.py delete mode 100644 bindings/python/py_src/tokenizers/implementations/bert_wordpiece.py delete mode 100644 bindings/python/py_src/tokenizers/implementations/byte_level_bpe.py delete mode 100644 bindings/python/py_src/tokenizers/implementations/char_level_bpe.py delete mode 100644 bindings/python/py_src/tokenizers/implementations/sentencepiece_bpe.py delete mode 100644 bindings/python/py_src/tokenizers/implementations/sentencepiece_unigram.py delete mode 100644 bindings/python/py_src/tokenizers/models.pyi rename bindings/{python-pipeline/py_src/tokenizers_pipeline => python/py_src/tokenizers}/models/__init__.pyi (100%) delete mode 100644 bindings/python/py_src/tokenizers/normalizers.pyi rename bindings/{python-pipeline/py_src/tokenizers_pipeline => python/py_src/tokenizers}/normalizers/__init__.pyi (100%) delete mode 100644 bindings/python/py_src/tokenizers/pre_tokenizers.pyi rename bindings/{python-pipeline/py_src/tokenizers_pipeline => python/py_src/tokenizers}/pre_tokenizers/__init__.pyi (100%) delete mode 100644 bindings/python/py_src/tokenizers/processors.pyi delete mode 100644 bindings/python/py_src/tokenizers/processors/__init__.py delete mode 100644 bindings/python/py_src/tokenizers/tokenizers.pyi delete mode 100644 bindings/python/py_src/tokenizers/tools/__init__.py delete mode 100644 bindings/python/py_src/tokenizers/tools/visualizer-styles.css delete mode 100644 bindings/python/py_src/tokenizers/tools/visualizer.py delete mode 100644 bindings/python/py_src/tokenizers/trainers.pyi delete mode 100644 bindings/python/pytest.ini delete mode 100644 bindings/python/rust-toolchain delete mode 100644 bindings/python/scripts/convert.py delete mode 100644 bindings/python/scripts/sentencepiece_extractor.py delete mode 100644 bindings/python/scripts/spm_parity_check.py delete mode 100644 bindings/python/setup.cfg rename bindings/{python-pipeline => python}/src/added_token.rs (96%) delete mode 100644 bindings/python/src/decoders.rs rename bindings/{python-pipeline => python}/src/detached_lock.rs (100%) delete mode 100644 bindings/python/src/encoding.rs delete mode 100644 bindings/python/src/processors.rs delete mode 100644 bindings/python/src/token.rs delete mode 100644 bindings/python/src/utils/iterators.rs delete mode 100644 bindings/python/src/utils/mod.rs delete mode 100644 bindings/python/src/utils/normalization.rs delete mode 100644 bindings/python/src/utils/pretokenization.rs delete mode 100644 bindings/python/src/utils/regex.rs delete mode 100644 bindings/python/src/utils/serde_pyo3.rs delete mode 100644 bindings/python/test.txt delete mode 100644 bindings/python/tests/__init__.py delete mode 100644 bindings/python/tests/bindings/__init__.py delete mode 100644 bindings/python/tests/bindings/test_decoders.py delete mode 100644 bindings/python/tests/bindings/test_encoding.py delete mode 100644 bindings/python/tests/bindings/test_models.py delete mode 100644 bindings/python/tests/bindings/test_normalizers.py delete mode 100644 bindings/python/tests/bindings/test_pre_tokenizers.py delete mode 100644 bindings/python/tests/bindings/test_processors.py delete mode 100644 bindings/python/tests/bindings/test_tokenizer.py delete mode 100644 bindings/python/tests/bindings/test_trainers.py delete mode 100644 bindings/python/tests/documentation/__init__.py delete mode 100644 bindings/python/tests/documentation/test_pipeline.py delete mode 100644 bindings/python/tests/documentation/test_quicktour.py delete mode 100644 bindings/python/tests/documentation/test_tutorial_train_from_iterators.py delete mode 100644 bindings/python/tests/implementations/__init__.py delete mode 100644 bindings/python/tests/implementations/test_base_tokenizer.py delete mode 100644 bindings/python/tests/implementations/test_bert_wordpiece.py delete mode 100644 bindings/python/tests/implementations/test_byte_level_bpe.py delete mode 100644 bindings/python/tests/implementations/test_char_bpe.py delete mode 100644 bindings/python/tests/implementations/test_sentencepiece.py delete mode 100644 bindings/python/tests/test_benchmarks.py delete mode 100644 bindings/python/tests/test_freethreaded.py delete mode 100644 bindings/python/tests/test_serialization.py delete mode 100644 bindings/python/tests/utils.py diff --git a/.github/scripts/render_python_bench.py b/.github/scripts/render_python_bench.py index d8dd0bbc0..347775d2c 100644 --- a/.github/scripts/render_python_bench.py +++ b/.github/scripts/render_python_bench.py @@ -126,7 +126,7 @@ def chart_svg(report, meta): def section_md(report, img_base, run_id): ref = f"`tokenizers` {report['release_version']} (PyPI)" lines = [ - f"### Python bindings — `tokenizers_pipeline` vs {ref}", + f"### Python bindings — this branch vs {ref}", "", rpb.picture(img_base, run_id, SLUG, "Python bindings encode_batch speedup " "vs the released tokenizers wheel", 860), @@ -163,8 +163,8 @@ def main(): report = json.loads(args.bench_json.read_text()) meta = [ - f"tokenizers_pipeline {report['pipeline_version']} vs " - f"tokenizers {report['release_version']}", + f"tokenizers {report['pipeline_version']} (this branch) vs " + f"{report['release_version']} (PyPI)", f"{report['cpus']} CPUs · median of {report['iters']} runs · " f"{report['fixture_count']} fixtures", ] diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 1465517b1..f2c7b06eb 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -5,7 +5,6 @@ on: branches: [main] paths: - "tokenizers/**" - - "bindings/python/**" - ".github/workflows/benchmarks.yml" workflow_dispatch: inputs: @@ -233,143 +232,3 @@ jobs: -f "output[title]=Benchmarks failed" \ -f "output[summary]=Benchmark workflow failed or was cancelled. Check the workflow run for details." - benchmark-python: - name: Run Python benchmarks - # Same dedicated runner group as the Rust job — keeps Python baselines comparable. - runs-on: - group: aws-general-8-plus - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - - - name: Install Rust stable - uses: dtolnay/rust-toolchain@stable - - - name: Setup sccache - uses: mozilla-actions/sccache-action@v0.0.9 - - - name: Install uv - uses: astral-sh/setup-uv@v6 - - # cairosvg (chart rendering) dlopens the system libcairo, which the - # aws-general-8-plus runner image doesn't ship (ubuntu-latest did) - - name: Install libcairo - run: sudo apt-get update && sudo apt-get install -y --no-install-recommends libcairo2 - - - name: Download benchmark data - env: - HF_TOKEN: ${{ secrets.HF_TOKEN }} - run: uvx --from huggingface_hub hf download hf-internal-testing/tokenizers-bench-data --repo-type dataset --local-dir tokenizers/data - - - name: Install Python bindings + benchmark deps - working-directory: bindings/python - run: | - uv venv .venv - source .venv/bin/activate - uv pip install maturin pytest pytest-benchmark - maturin develop --release --manifest-path Cargo.toml - echo "$VIRTUAL_ENV/bin" >> "$GITHUB_PATH" - - - name: Run Python benchmarks - working-directory: bindings/python - run: | - source .venv/bin/activate - python -m pytest tests/test_benchmarks.py \ - --benchmark-json=bench_output.json \ - --benchmark-min-rounds=15 \ - --benchmark-columns=mean,stddev,rounds \ - --benchmark-sort=name \ - -v - - # Upload as artifact for easy download - - name: Upload Python benchmark results - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: python-benchmark-results - path: bindings/python/bench_output.json - retention-days: 30 - - # Compare against saved baseline - - name: Compare Python results with baseline - id: py_compare - working-directory: bindings/python - env: - HF_TOKEN: ${{ secrets.HF_TOKEN }} - run: | - source .venv/bin/activate - uvx --from huggingface_hub hf download hf-internal-testing/tokenizers-bench \ - python-baseline.json --repo-type dataset --local-dir baseline_dir 2>/dev/null || true - - if [ ! -f baseline_dir/python-baseline.json ]; then - echo "No previous Python baseline found — skipping comparison" - echo "has_comparison=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - - echo "has_comparison=true" >> "$GITHUB_OUTPUT" - - uvx --with cairosvg python ${{ github.workspace }}/.github/scripts/render_bench_svg.py \ - --baseline-json baseline_dir/python-baseline.json \ - --current-json bench_output.json \ - --output python_bench.png \ - --title "Python Benchmarks — ${{ github.sha }}" - - - name: Upload Python chart to HF Hub - if: steps.py_compare.outputs.has_comparison == 'true' && env.HF_TOKEN != '' - continue-on-error: true - env: - HF_TOKEN: ${{ secrets.HF_TOKEN }} - working-directory: bindings/python - run: | - source .venv/bin/activate - uvx --from huggingface_hub hf upload hf-internal-testing/tokenizers-bench \ - python_bench.png "charts/python-${{ github.sha }}.png" --repo-type dataset - - - name: Upload chart artifact - if: steps.py_compare.outputs.has_comparison == 'true' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: python-bench-chart - path: bindings/python/python_bench.png - retention-days: 30 - - # Upload baseline on push to main only - - name: Upload Python baseline to HF Hub - if: github.event_name == 'push' && env.HF_TOKEN != '' - continue-on-error: true - env: - HF_TOKEN: ${{ secrets.HF_TOKEN }} - working-directory: bindings/python - run: | - source .venv/bin/activate - cp bench_output.json "python-output-${{ github.sha }}.json" - uvx --from huggingface_hub hf upload hf-internal-testing/tokenizers-bench \ - bench_output.json python-baseline.json --repo-type dataset - uvx --from huggingface_hub hf upload hf-internal-testing/tokenizers-bench \ - "python-output-${{ github.sha }}.json" "history/python-output-${{ github.sha }}.json" --repo-type dataset - - - name: Post Python results to PR - if: inputs.pr_number != '' && steps.py_compare.outputs.has_comparison == 'true' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - working-directory: bindings/python - run: | - img_url="https://huggingface.co/datasets/hf-internal-testing/tokenizers-bench/resolve/main/charts/python-${{ github.sha }}.png" - - { - echo "## Python Benchmark Results" - echo "" - echo "Commit: \`${{ github.sha }}\`" - echo "" - echo "![Python Benchmarks]($img_url)" - } > py_comparison.md - - existing=$(gh api "repos/${{ github.repository }}/issues/${{ inputs.pr_number }}/comments" \ - --jq '.[] | select(.body | startswith("## Python Benchmark Results")) | .id' | head -1) - - if [ -n "$existing" ]; then - gh api "repos/${{ github.repository }}/issues/comments/$existing" \ - -X PATCH -F "body=@py_comparison.md" - else - gh pr comment "${{ inputs.pr_number }}" --body-file py_comparison.md - fi diff --git a/.github/workflows/build_documentation.yml b/.github/workflows/build_documentation.yml index 19759a4fd..21293fc6c 100644 --- a/.github/workflows/build_documentation.yml +++ b/.github/workflows/build_documentation.yml @@ -1,16 +1,10 @@ name: Build documentation +# docs/source-doc-builder describes the 0.x Python API. The bindings were +# rewritten for 1.0 (PipelineTokenizer); until the docs are rewritten to +# match, the published docs only build on demand — automatic triggers would +# publish documentation for an API the package no longer has. on: - push: - branches: - - main - - doc-builder* - - v*-release - - use_templates - tags: - - v* - release: - types: [published] workflow_dispatch: inputs: ref: diff --git a/.github/workflows/build_pr_documentation.yml b/.github/workflows/build_pr_documentation.yml index f7d278555..188e86499 100644 --- a/.github/workflows/build_pr_documentation.yml +++ b/.github/workflows/build_pr_documentation.yml @@ -1,7 +1,10 @@ name: Build PR Documentation +# docs/source-doc-builder describes the 0.x Python API. The bindings were +# rewritten for 1.0 (PipelineTokenizer); until the docs are rewritten to +# match, PR doc previews only run on demand. on: - pull_request: + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} diff --git a/.github/workflows/docs-check.yml b/.github/workflows/docs-check.yml index 9f6808920..9e4aa1f9d 100644 --- a/.github/workflows/docs-check.yml +++ b/.github/workflows/docs-check.yml @@ -1,10 +1,10 @@ name: Documentation +# The sphinx docs under docs/ describe the 0.x Python API. The bindings were +# rewritten for 1.0 (PipelineTokenizer); until the docs are rewritten to +# match, this check only runs on demand. on: - push: - branches: - - main - pull_request: + workflow_dispatch: jobs: build: diff --git a/.github/workflows/pipeline-bench.yml b/.github/workflows/pipeline-bench.yml index f0ef7e2d0..29596aa48 100644 --- a/.github/workflows/pipeline-bench.yml +++ b/.github/workflows/pipeline-bench.yml @@ -47,7 +47,7 @@ on: paths: - "tokenizers/tk-encode/**" - "tokenizers/src/**" - - "bindings/python-pipeline/**" + - "bindings/python/**" - ".github/workflows/pipeline-bench.yml" - ".github/scripts/render_pipeline_bench.py" - "tokenizers/tk-encode/examples/bench_models.json" @@ -230,12 +230,15 @@ jobs: path: tokenizers/pipeline_bench_${{ matrix.shard }}.json retention-days: 3 - # Python bindings: the tokenizers_pipeline wheel vs the latest *released* - # tokenizers wheel from PyPI, timed end-to-end through Python — input - # conversion, encode, and output objects all count, because that is what a - # user pays. Same models (bench_models.json) and fixture corpora as the Rust - # bench; unsupported models are reported as skipped, an id mismatch fails - # the job. Results land in the run's step summary + an artifact. + # Python bindings: this branch's wheel vs the latest *released* tokenizers + # wheel from PyPI, timed end-to-end through Python — input conversion, + # encode, and output objects all count, because that is what a user pays. + # Both are named `tokenizers`, so the release is installed into its own + # directory (pip --target) and benched in a subprocess with PYTHONPATH + # pointing there. Same models (bench_models.json) and fixture corpora as + # the Rust bench; unsupported models are reported as skipped, an id + # mismatch fails the job. Results land in the run's step summary + an + # artifact. python-bindings-bench: name: Python bindings vs released wheel if: github.event_name != 'pull_request' || github.event.label.name == 'run-pipeline-bench' @@ -268,15 +271,19 @@ jobs: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: make fixtures bench-models HF="uvx --from huggingface_hub hf" - - name: Build and install the tokenizers_pipeline wheel - working-directory: bindings/python-pipeline + - name: Build and install this branch's wheel + working-directory: bindings/python run: | uv venv .venv - uv pip install --python .venv/bin/python maturin numpy tokenizers + uv pip install --python .venv/bin/python maturin numpy source .venv/bin/activate && maturin develop --release + - name: Install the released wheel into its own directory + working-directory: bindings/python + run: uv pip install --python .venv/bin/python --target .release tokenizers + - name: Bench against the released wheel - working-directory: bindings/python-pipeline + working-directory: bindings/python run: | .venv/bin/python benches/bench_vs_release.py \ --manifest ../../tokenizers/tk-encode/examples/bench_models.json \ @@ -289,8 +296,8 @@ jobs: with: name: python-bindings-bench path: | - bindings/python-pipeline/python_bench.json - bindings/python-pipeline/python_bench.md + bindings/python/python_bench.json + bindings/python/python_bench.md retention-days: 30 # Fan-in: concatenate the shard partials (in shard order = manifest order), diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml index 6a8996e36..c857ecda9 100644 --- a/.github/workflows/python-release.yml +++ b/.github/workflows/python-release.yml @@ -1,4 +1,8 @@ name: Python Release + +# All wheels are abi3-py310: one binary per platform covers CPython +# 3.10–3.14. No free-threaded (3.14t) wheels — the 1.x bindings are +# abi3-only. on: push: tags: @@ -19,7 +23,7 @@ jobs: working-directory: ./bindings/python build: - name: build on ${{ matrix.platform || matrix.os }} (${{ matrix.target }} - ${{ matrix.manylinux || 'auto' }} - ${{ matrix.flavor == 'ft' && '3.14t' || matrix.interpreter || '3.14' }}) + name: build on ${{ matrix.platform || matrix.os }} (${{ matrix.target }} - ${{ matrix.manylinux || 'auto' }} - ${{ matrix.interpreter || '3.14' }}) # only run on push to main and on release needs: [lock_exists] if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'Full Build') @@ -29,11 +33,6 @@ jobs: os: [ubuntu, macos, windows] target: [x86_64, aarch64] manylinux: [auto] - # `flavor` discriminates regular abi3 wheels from free-threaded - # (3.14t, non-abi3) wheels so the include: entries below for - # 3.14t create new matrix cells instead of merging with the abi3 - # combos. - flavor: [abi3] include: - os: ubuntu platform: linux @@ -56,10 +55,6 @@ jobs: ls: dir target: aarch64 python-architecture: arm64 - # 3.14t arm64-freethreaded is currently broken upstream: - # actions/python-versions ships a 0-byte python.exe so pip - # install fails with `ModuleNotFoundError: encodings`. Drop - # 3.14t here until the upstream package is fixed. python-install: "3.14" interpreter: "3.14" # - os: windows @@ -97,23 +92,6 @@ jobs: platform: linux target: s390x interpreter: "3.14" - - # --- Free-threaded Python 3.14t wheels ------------------------- - # `flavor: ft` switches the build to non-abi3 (`--no-default-features - # --features ext-module`) and `--interpreter 3.14t`, derived from - # `flavor` in the maturin invocation below. linux container builds - # (manylinux/musllinux) get 3.14t from the docker image, so - # `python-install` is unset for those; macOS/windows host builds - # need it set explicitly so setup-python actually installs 3.14t. - # windows-11-arm 3.14t is intentionally absent — see the abi3 - # windows-11-arm entry above (upstream-broken package). - - { os: ubuntu, platform: linux, target: x86_64, manylinux: auto, flavor: ft } - - { os: ubuntu, platform: linux, target: aarch64, manylinux: auto, flavor: ft } - - { os: ubuntu, platform: linux, target: x86_64, manylinux: musllinux_1_1, flavor: ft } - - { os: ubuntu, platform: linux, target: aarch64, manylinux: musllinux_1_1, flavor: ft } - - { os: macos, target: x86_64, manylinux: auto, flavor: ft, python-install: "3.14t" } - - { os: macos, target: aarch64, manylinux: auto, flavor: ft, python-install: "3.14t" } - - { os: windows, ls: dir, target: x86_64, manylinux: auto, python-architecture: x64, python-install: "3.14t", flavor: ft } exclude: - os: windows target: aarch64 @@ -148,13 +126,9 @@ jobs: working-directory: ./bindings/python manylinux: ${{ matrix.manylinux || 'auto' }} container: ${{ matrix.container }} - # `flavor=ft` builds drop the abi3 cargo feature so the resulting - # wheel is non-abi3 (free-threaded Python can't load limited-API - # extensions). `flavor=abi3` builds use defaults. args: >- --release --out dist - --interpreter ${{ matrix.flavor == 'ft' && '3.14t' || matrix.interpreter || '3.14' }} - ${{ matrix.flavor == 'ft' && '--no-default-features --features ext-module' || '' }} + --interpreter ${{ matrix.interpreter || '3.14' }} rust-toolchain: stable sccache: false docker-options: -e CI @@ -167,7 +141,7 @@ jobs: - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: pypi_files-${{ matrix.os }}-${{ matrix.target }}-${{ matrix.manylinux }}-${{ matrix.flavor || 'abi3' }} + name: pypi_files-${{ matrix.os }}-${{ matrix.target }}-${{ matrix.manylinux }} path: ./bindings/python/dist build-sdist: name: build sdist diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 57eb85754..11dee20aa 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -11,41 +11,17 @@ on: - bindings/node/** jobs: - build_win_32: - name: Check it builds for Windows 32-bit - runs-on: windows-latest - strategy: - matrix: - # The cdylib is built with `abi3-py310`, so a single binary covers - # 3.10–3.14. Free-threaded 3.14t is the only interpreter that needs - # its own build (no abi3 / limited API). - python: ["3.14", "3.14t"] - steps: - - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Install Rust - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - with: - targets: i686-pc-windows-msvc - - - name: Install Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: ${{ matrix.python }} - architecture: x86 - - - name: Build - run: cargo build --target i686-pc-windows-msvc --manifest-path ./bindings/python/Cargo.toml - build_and_test: - name: Check everything builds & tests + name: Build, run the examples, check the stubs runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: [ubuntu-latest, macos-latest] - python: ["3.14", "3.14t"] + # The cdylib is built with `abi3-py310`, so a single binary covers + # 3.10–3.14; test the floor and the newest. Free-threaded 3.14t is + # not supported (abi3 extensions cannot load there). + python: ["3.10", "3.14"] steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -58,68 +34,65 @@ jobs: with: workspaces: bindings/python shared-key: python - # Skip caching ~/.cargo/bin — earlier runs left a stale - # rustup-init binary masquerading as `cargo`, breaking clippy - # on macOS. The cargo toolchain is reinstalled by dtolnay/ - # rust-toolchain each run anyway, so caching it adds risk - # without speedup. cache-bin: false - - name: Install Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: ${{ matrix.python }} - architecture: "x64" + - name: Install uv + uses: astral-sh/setup-uv@v6 - - name: Install + - name: Build and install working-directory: ./bindings/python run: | - python -m venv .env - source .env/bin/activate - pip install -U pip - pip install pytest pytest-asyncio huggingface_hub setuptools_rust numpy pyarrow datasets ruff ty maturin - # Build via maturin so we can drop the abi3 cargo feature on - # free-threaded Python (3.14t can't load abi3 extensions). - # `pip install -e .` would build through maturin's PEP 660 path - # which keeps abi3 enabled regardless of the interpreter. - # `parity-aware-bpe` must be re-listed explicitly on the 3.14t - # branch — `--no-default-features` strips it otherwise, and the - # Python shim unconditionally imports `ParityBpeTrainer`. - if python -c 'import sys; sys.exit(0 if sys._is_gil_enabled() else 1)' 2>/dev/null; then - maturin develop --release - else - maturin develop --release --no-default-features --features ext-module,parity-aware-bpe - fi - - - # Test fixtures are immutable — cache them (plus the huggingface_hub / - # datasets cache used by the test suite at runtime) so CI doesn't - # re-download from the Hub (and get rate-limited) on every run. - - name: Cache HF test data + uv venv .venv --python ${{ matrix.python }} + uv pip install --python .venv/bin/python maturin numpy + source .venv/bin/activate && maturin develop --release + + # The examples read real tokenizer.json files + a text corpus. Cached + # under a key of this workflow's own (not shared with pipeline-bench, + # which needs the fixture corpora on top). + - name: Cache test data uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: - path: | - bindings/python/data - bindings/python/tests/data - ~/.cache/huggingface - key: hf-test-data-${{ runner.os }}-${{ hashFiles('bindings/python/Makefile', 'bindings/python/tests/utils.py') }} - restore-keys: hf-test-data-${{ runner.os }}- - - - name: Run tests - working-directory: ./bindings/python + path: tokenizers/data + key: python-test-data-${{ hashFiles('tokenizers/Makefile', 'tokenizers/tk-encode/examples/bench_models.json') }} + + - name: Download model tokenizers and corpus + working-directory: tokenizers env: HF_TOKEN: ${{ secrets.HF_TOKEN }} + run: make bench-models data/big.txt HF="uvx --from huggingface_hub hf" + + - name: Run the examples + working-directory: ./bindings/python run: | - source .env/bin/activate - # `make test` also runs `cargo test --no-default-features` which - # links libpython via pyo3's `auto-initialize`. On free-threaded - # Python the macOS runner doesn't ship libpython3.14t in the - # framework path, so on 3.14t we run only the python half. - if python -c 'import sys; sys.exit(0 if sys._is_gil_enabled() else 1)' 2>/dev/null; then - make test - else - make test-py - fi + .venv/bin/python examples/01_train_and_encode.py + .venv/bin/python examples/02_pretrained.py + .venv/bin/python examples/03_threading.py + + # The .pyi stubs are generated from the built extension; a diff here + # means someone changed the Rust API without running `make stubs`. + - name: Check the committed stubs are current + working-directory: ./bindings/python + run: | + cargo run --manifest-path tools/stub-gen/Cargo.toml + git diff --exit-code -- py_src + + build_windows: + name: Check it builds on Windows + runs-on: windows-latest + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Install Rust + uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable + + - name: Install Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.14" + + - name: Build + run: cargo build --release --manifest-path ./bindings/python/Cargo.toml quality: name: Lint & format @@ -145,29 +118,6 @@ jobs: - name: Lint with Clippy run: cargo clippy --manifest-path ./bindings/python/Cargo.toml --all-targets --all-features -- -D warnings - - name: Install Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: "3.14" - architecture: "x64" - - # check-style builds the abi3 cdylib and imports it, which only works - # under a GIL interpreter — hence pinned to 3.14, not free-threaded 3.14t. - - name: Install - working-directory: ./bindings/python - run: | - python -m venv .env - source .env/bin/activate - pip install -U pip - pip install pytest pytest-asyncio requests setuptools_rust numpy pyarrow datasets ruff ty maturin - maturin develop --release - - - name: Check style - working-directory: ./bindings/python - run: | - source .env/bin/activate - make check-style - audit: name: Audit dependencies runs-on: ubuntu-latest @@ -187,5 +137,7 @@ jobs: - name: Install cargo-audit run: cargo install cargo-audit + # Both ignores are unmaintained-crate advisories pulled in transitively + # by the core crates (paste via rust-stemmers, fxhash via tk-encode). - name: Run Audit - run: cargo audit -D warnings -f ./bindings/python/Cargo.lock --ignore RUSTSEC-2024-0436 --ignore RUSTSEC-2025-0014 --ignore RUSTSEC-2025-0119 --ignore RUSTSEC-2026-0204 --ignore RUSTSEC-2025-0057 + run: cargo audit -D warnings -f ./bindings/python/Cargo.lock --ignore RUSTSEC-2024-0436 --ignore RUSTSEC-2025-0057 diff --git a/README.md b/README.md index 8258e7534..6b561b662 100644 --- a/README.md +++ b/README.md @@ -28,8 +28,12 @@ versatility. - Does all the pre-processing: Truncate, Pad, add the special tokens your model needs. ## Performances -Performances can vary depending on hardware, but running the [~/bindings/python/benches/test_tiktoken.py](bindings/python/benches/test_tiktoken.py) should give the following on a g6 aws instance: -![image](https://github.com/user-attachments/assets/2b913d4b-e488-4cbc-b542-f90a6c40643d) +Performances can vary depending on hardware. The Python bindings ship a +benchmark against the released wheel +([bindings/python/benches/bench_vs_release.py](bindings/python/benches/bench_vs_release.py)); +the Rust core has the equivalent fixture benchmark +(`tokenizers/tk-encode/examples/fixture_bench.rs`). Both run in the Pipeline +Benchmark workflow in CI. ## Bindings @@ -83,10 +87,13 @@ tokenizer.train(files=["wiki.train.raw", "wiki.valid.raw", "wiki.test.raw"], tra Once your tokenizer is trained, encode any text with just one line: ```python -output = tokenizer.encode("Hello, y'all! How are you 😁 ?") -print(output.tokens) +ids = tokenizer.encode("Hello, y'all! How are you 😁 ?") +print([tokenizer.id_to_token(i) for i in ids]) # ["Hello", ",", "y", "'", "all", "!", "How", "are", "you", "[UNK]", "?"] ``` +`encode` returns the token ids as a `numpy.uint32` array — ready to hand to +your model with no further conversion. + Check the [documentation](https://huggingface.co/docs/tokenizers/index) or the [quicktour](https://huggingface.co/docs/tokenizers/quicktour) to learn more! diff --git a/bindings/python-pipeline/Cargo.lock b/bindings/python-pipeline/Cargo.lock deleted file mode 100644 index 8c20a0a4e..000000000 --- a/bindings/python-pipeline/Cargo.lock +++ /dev/null @@ -1,1707 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "ahash" -version = "0.8.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" -dependencies = [ - "cfg-if", - "getrandom 0.3.4", - "once_cell", - "serde", - "version_check", - "zerocopy", -] - -[[package]] -name = "aho-corasick" -version = "1.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" -dependencies = [ - "memchr", -] - -[[package]] -name = "allocator-api2" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" - -[[package]] -name = "anes" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" - -[[package]] -name = "anstyle" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" - -[[package]] -name = "arbitrary-chunks" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ad8689a486416c401ea15715a4694de30054248ec627edbf31f49cb64ee4086" - -[[package]] -name = "atomsplit" -version = "0.1.0" -dependencies = [ - "memchr", -] - -[[package]] -name = "autocfg" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" - -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - -[[package]] -name = "bit-set" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" -dependencies = [ - "bit-vec", -] - -[[package]] -name = "bit-vec" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" - -[[package]] -name = "bitflags" -version = "2.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" - -[[package]] -name = "bitvec" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" -dependencies = [ - "funty", - "radium", - "tap", - "wyz", -] - -[[package]] -name = "block-pseudorand" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2097358495d244a0643746f4d13eedba4608137008cf9dec54e53a3b700115a6" -dependencies = [ - "chiapos-chacha8", - "nanorand", -] - -[[package]] -name = "bumpalo" -version = "3.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" - -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - -[[package]] -name = "cast" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" - -[[package]] -name = "castaway" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" -dependencies = [ - "rustversion", -] - -[[package]] -name = "cc" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8" -dependencies = [ - "find-msvc-tools", - "shlex", -] - -[[package]] -name = "cfg-if" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" - -[[package]] -name = "chacha20" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" -dependencies = [ - "cfg-if", - "cpufeatures", - "rand_core 0.10.1", -] - -[[package]] -name = "chiapos-chacha8" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33f8be573a85f6c2bc1b8e43834c07e32f95e489b914bf856c0549c3c269cd0a" -dependencies = [ - "rayon", -] - -[[package]] -name = "ciborium" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" -dependencies = [ - "ciborium-io", - "ciborium-ll", - "serde", -] - -[[package]] -name = "ciborium-io" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" - -[[package]] -name = "ciborium-ll" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" -dependencies = [ - "ciborium-io", - "half", -] - -[[package]] -name = "clap" -version = "4.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7" -dependencies = [ - "clap_builder", -] - -[[package]] -name = "clap_builder" -version = "4.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" -dependencies = [ - "anstyle", - "clap_lex", -] - -[[package]] -name = "clap_lex" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" - -[[package]] -name = "colored" -version = "3.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" -dependencies = [ - "windows-sys", -] - -[[package]] -name = "compact_str" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab" -dependencies = [ - "castaway", - "cfg-if", - "itoa", - "rustversion", - "ryu", - "serde", - "static_assertions", -] - -[[package]] -name = "console" -version = "0.16.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fe5f465a4f6fee88fad41b85d990f84c835335e85b5d9e6e63e0d06d28cba7c" -dependencies = [ - "encode_unicode", - "libc", - "unicode-width", - "windows-sys", -] - -[[package]] -name = "cpufeatures" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" -dependencies = [ - "libc", -] - -[[package]] -name = "criterion" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" -dependencies = [ - "anes", - "cast", - "ciborium", - "clap", - "criterion-plot", - "is-terminal", - "itertools 0.10.5", - "num-traits", - "once_cell", - "oorandom", - "plotters", - "rayon", - "regex", - "serde", - "serde_derive", - "serde_json", - "tinytemplate", - "walkdir", -] - -[[package]] -name = "criterion-plot" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" -dependencies = [ - "cast", - "itertools 0.10.5", -] - -[[package]] -name = "crossbeam-deque" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" - -[[package]] -name = "crunchy" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" - -[[package]] -name = "daachorse" -version = "3.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5614204febbc33cc07a2806aa6440b904ac012b68eecc37f4493ea4a76455a3d" - -[[package]] -name = "darling" -version = "0.20.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" -dependencies = [ - "darling_core", - "darling_macro", -] - -[[package]] -name = "darling_core" -version = "0.20.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim", - "syn 2.0.119", -] - -[[package]] -name = "darling_macro" -version = "0.20.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" -dependencies = [ - "darling_core", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "dary_heap" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" -dependencies = [ - "serde", -] - -[[package]] -name = "derive_builder" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" -dependencies = [ - "derive_builder_macro", -] - -[[package]] -name = "derive_builder_core" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" -dependencies = [ - "darling", - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "derive_builder_macro" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" -dependencies = [ - "derive_builder_core", - "syn 2.0.119", -] - -[[package]] -name = "either" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" - -[[package]] -name = "encode_unicode" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" - -[[package]] -name = "equivalent" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" - -[[package]] -name = "errno" -version = "0.3.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" -dependencies = [ - "libc", - "windows-sys", -] - -[[package]] -name = "esaxx-rs" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" -dependencies = [ - "cc", -] - -[[package]] -name = "fancy-regex" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8" -dependencies = [ - "bit-set", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "fastrand" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" - -[[package]] -name = "find-msvc-tools" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" - -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "foldhash" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" - -[[package]] -name = "funty" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" - -[[package]] -name = "futures-core" -version = "0.3.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" - -[[package]] -name = "futures-task" -version = "0.3.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" - -[[package]] -name = "futures-util" -version = "0.3.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" -dependencies = [ - "futures-core", - "futures-task", - "pin-project-lite", - "slab", -] - -[[package]] -name = "fxhash" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" -dependencies = [ - "byteorder", -] - -[[package]] -name = "getrandom" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" -dependencies = [ - "cfg-if", - "libc", - "r-efi 5.3.0", - "wasip2", -] - -[[package]] -name = "getrandom" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" -dependencies = [ - "cfg-if", - "libc", - "r-efi 6.0.0", - "rand_core 0.10.1", -] - -[[package]] -name = "half" -version = "2.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" -dependencies = [ - "cfg-if", - "crunchy", - "zerocopy", -] - -[[package]] -name = "hashbrown" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" -dependencies = [ - "allocator-api2", - "equivalent", - "foldhash", -] - -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - -[[package]] -name = "hermit-abi" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" - -[[package]] -name = "ident_case" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" - -[[package]] -name = "indicatif" -version = "0.18.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9433806cd6b4ec1aba79c021c7e4c58fb4c3b9977c085062e611ac929998fb0c" -dependencies = [ - "console", - "portable-atomic", - "unicode-width", - "unit-prefix", - "web-time", -] - -[[package]] -name = "is-terminal" -version = "0.4.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys", -] - -[[package]] -name = "itertools" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" -dependencies = [ - "either", -] - -[[package]] -name = "itertools" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" -dependencies = [ - "either", -] - -[[package]] -name = "itertools" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" -dependencies = [ - "either", -] - -[[package]] -name = "itoa" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" - -[[package]] -name = "js-sys" -version = "0.3.103" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" -dependencies = [ - "cfg-if", - "futures-util", - "wasm-bindgen", -] - -[[package]] -name = "libc" -version = "0.2.189" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" - -[[package]] -name = "linux-raw-sys" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" - -[[package]] -name = "log" -version = "0.4.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" - -[[package]] -name = "macro_rules_attribute" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65049d7923698040cd0b1ddcced9b0eb14dd22c5f86ae59c3740eab64a676520" -dependencies = [ - "macro_rules_attribute-proc_macro", - "paste", -] - -[[package]] -name = "macro_rules_attribute-proc_macro" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30" - -[[package]] -name = "matrixmultiply" -version = "0.3.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7" -dependencies = [ - "autocfg", - "rawpointer", -] - -[[package]] -name = "mem_dbg" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4ef2d80bfa14894b6d5a3ff537e7e9a908dbf4c95de8a5b8ad2a473301676e6" -dependencies = [ - "bitflags", - "hashbrown", - "mem_dbg-derive", -] - -[[package]] -name = "mem_dbg-derive" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73acd151c6ce84a41d8d6fb0958d9a3d5a18d649ad5a85ad5b719439af8ad257" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "memchr" -version = "2.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" - -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - -[[package]] -name = "monostate" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3341a273f6c9d5bef1908f17b7267bbab0e95c9bf69a0d4dcf8e9e1b2c76ef67" -dependencies = [ - "monostate-impl", - "serde", - "serde_core", -] - -[[package]] -name = "monostate-impl" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "nanorand" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "729eb334247daa1803e0a094d0a5c55711b85571179f5ec6e53eccfdf7008958" - -[[package]] -name = "ndarray" -version = "0.17.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d" -dependencies = [ - "matrixmultiply", - "num-complex", - "num-integer", - "num-traits", - "portable-atomic", - "portable-atomic-util", - "rawpointer", -] - -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - -[[package]] -name = "num-complex" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-integer" -version = "0.1.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-traits" -version = "0.2.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" -dependencies = [ - "autocfg", -] - -[[package]] -name = "numpy" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a5b15d63a5ff39e378daed0e1340d3a5964703ea9712eb09a0dc66fade996f4" -dependencies = [ - "libc", - "ndarray", - "num-complex", - "num-integer", - "num-traits", - "pyo3", - "pyo3-build-config", - "rustc-hash", -] - -[[package]] -name = "once_cell" -version = "1.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" - -[[package]] -name = "oorandom" -version = "11.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" - -[[package]] -name = "partition" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "947f833aaa585cf12b8ec7c0476c98784c49f33b861376ffc84ed92adebf2aba" - -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - -[[package]] -name = "pin-project-lite" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" - -[[package]] -name = "plotters" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" -dependencies = [ - "num-traits", - "plotters-backend", - "plotters-svg", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "plotters-backend" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" - -[[package]] -name = "plotters-svg" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" -dependencies = [ - "plotters-backend", -] - -[[package]] -name = "portable-atomic" -version = "1.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" - -[[package]] -name = "portable-atomic-util" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" -dependencies = [ - "portable-atomic", -] - -[[package]] -name = "ppv-lite86" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" -dependencies = [ - "zerocopy", -] - -[[package]] -name = "prefetch-index" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9057806a8d77d67bccdc0f542db43737a6f19ada3efab2adc63277feea27310f" - -[[package]] -name = "proc-macro2" -version = "1.0.107" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "ptr_hash" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f184d2c69ac0853853275df42e7160a7dc4f3248d93434002c28de27ed3f6d0" -dependencies = [ - "bitvec", - "colored", - "fastrand", - "fxhash", - "itertools 0.15.0", - "log", - "mem_dbg", - "prefetch-index", - "rand 0.10.2", - "rand_chacha 0.10.0", - "rayon", - "rdst", - "serde", - "tempfile", - "xxhash-rust", -] - -[[package]] -name = "pyo3" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd274650b21d4bfc26a0a47587962c1edb425f69287324355cd040c3ea66071c" -dependencies = [ - "libc", - "once_cell", - "portable-atomic", - "pyo3-build-config", - "pyo3-ffi", - "pyo3-macros", -] - -[[package]] -name = "pyo3-build-config" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5e2a7d2f0d013342f295c048ad19237add5154a55b1c5a254c0ec93d4109078" -dependencies = [ - "target-lexicon", -] - -[[package]] -name = "pyo3-ffi" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca85c467da1bbc8d866eea5deff9cf29ea5f7785054a17da36e65bda9c05845b" -dependencies = [ - "libc", - "pyo3-build-config", -] - -[[package]] -name = "pyo3-macros" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ac53762fd065daa3194dd09337a38bd793a188100fd1a9304c4ab312d901771" -dependencies = [ - "proc-macro2", - "pyo3-macros-backend", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "pyo3-macros-backend" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca3a1557399783172dc5bf39cfca835157732532cba56b71d2292161e53b362" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "quote" -version = "1.0.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r-efi" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" - -[[package]] -name = "r-efi" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" - -[[package]] -name = "radium" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" - -[[package]] -name = "rand" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" -dependencies = [ - "rand_chacha 0.9.0", - "rand_core 0.9.5", -] - -[[package]] -name = "rand" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" -dependencies = [ - "chacha20", - "getrandom 0.4.3", - "rand_core 0.10.1", -] - -[[package]] -name = "rand_chacha" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" -dependencies = [ - "ppv-lite86", - "rand_core 0.9.5", -] - -[[package]] -name = "rand_chacha" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e6af7f3e25ded52c41df4e0b1af2d047e45896c2f3281792ed68a1c243daedb" -dependencies = [ - "ppv-lite86", - "rand_core 0.10.1", -] - -[[package]] -name = "rand_core" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" -dependencies = [ - "getrandom 0.3.4", -] - -[[package]] -name = "rand_core" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" - -[[package]] -name = "rawpointer" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" - -[[package]] -name = "rayon" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-cond" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f" -dependencies = [ - "either", - "itertools 0.14.0", - "rayon", -] - -[[package]] -name = "rayon-core" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - -[[package]] -name = "rdst" -version = "0.20.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e7970b4e577b76a96d5e56b5f6662b66d1a4e1f5bb026ee118fc31b373c2752" -dependencies = [ - "arbitrary-chunks", - "block-pseudorand", - "criterion", - "partition", - "rayon", - "tikv-jemallocator", - "voracious_radix_sort", -] - -[[package]] -name = "regex" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.4.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.8.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" - -[[package]] -name = "rustc-hash" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" - -[[package]] -name = "rustix" -version = "1.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" -dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys", - "windows-sys", -] - -[[package]] -name = "rustversion" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" - -[[package]] -name = "ryu" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" - -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - -[[package]] -name = "serde" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" -dependencies = [ - "serde_core", - "serde_derive", -] - -[[package]] -name = "serde_core" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - -[[package]] -name = "serde_json" -version = "1.0.151" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" -dependencies = [ - "itoa", - "memchr", - "serde", - "serde_core", - "zmij", -] - -[[package]] -name = "shlex" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" - -[[package]] -name = "slab" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" - -[[package]] -name = "smallvec" -version = "1.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" - -[[package]] -name = "spm_precompiled" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" -dependencies = [ - "base64", - "nom", - "serde", - "unicode-segmentation", -] - -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - -[[package]] -name = "syn" -version = "2.0.119" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "3.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "tap" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" - -[[package]] -name = "target-lexicon" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" - -[[package]] -name = "tempfile" -version = "3.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" -dependencies = [ - "fastrand", - "getrandom 0.4.3", - "once_cell", - "rustix", - "windows-sys", -] - -[[package]] -name = "thiserror" -version = "2.0.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "2.0.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - -[[package]] -name = "tikv-jemalloc-sys" -version = "0.5.4+5.3.0-patched" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9402443cb8fd499b6f327e40565234ff34dbda27460c5b47db0db77443dd85d1" -dependencies = [ - "cc", - "libc", -] - -[[package]] -name = "tikv-jemallocator" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "965fe0c26be5c56c94e38ba547249074803efd52adfb66de62107d95aab3eaca" -dependencies = [ - "libc", - "tikv-jemalloc-sys", -] - -[[package]] -name = "tinytemplate" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" -dependencies = [ - "serde", - "serde_json", -] - -[[package]] -name = "tinyvec" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - -[[package]] -name = "tk-encode" -version = "0.23.2-dev.0" -dependencies = [ - "ahash", - "atomsplit", - "compact_str", - "daachorse", - "dary_heap", - "derive_builder", - "fancy-regex", - "getrandom 0.3.4", - "indicatif", - "itertools 0.14.0", - "log", - "macro_rules_attribute", - "memchr", - "monostate", - "paste", - "ptr_hash", - "rand 0.9.5", - "rayon", - "rayon-cond", - "regex", - "serde", - "serde_json", - "spm_precompiled", - "thiserror", - "unicode-normalization", - "unicode-normalization-alignments", - "unicode-segmentation", - "unicode_categories", - "yada", -] - -[[package]] -name = "tk-train" -version = "0.23.2-dev.0" -dependencies = [ - "ahash", - "compact_str", - "dary_heap", - "derive_builder", - "esaxx-rs", - "indicatif", - "itertools 0.14.0", - "log", - "rayon", - "serde", - "serde_json", - "thiserror", - "tk-encode", -] - -[[package]] -name = "tokenizers-pipeline-python" -version = "0.1.0" -dependencies = [ - "libc", - "numpy", - "pyo3", - "rayon", - "serde", - "serde_json", - "tk-encode", - "tk-train", -] - -[[package]] -name = "unicode-ident" -version = "1.0.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" - -[[package]] -name = "unicode-normalization" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" -dependencies = [ - "tinyvec", -] - -[[package]] -name = "unicode-normalization-alignments" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" -dependencies = [ - "smallvec", -] - -[[package]] -name = "unicode-segmentation" -version = "1.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" - -[[package]] -name = "unicode-width" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" - -[[package]] -name = "unicode_categories" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" - -[[package]] -name = "unit-prefix" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" - -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - -[[package]] -name = "voracious_radix_sort" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "446e7ffcb6c27a71d05af7e51ef2ee5b71c48424b122a832f2439651e1914899" -dependencies = [ - "rayon", -] - -[[package]] -name = "walkdir" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" -dependencies = [ - "same-file", - "winapi-util", -] - -[[package]] -name = "wasip2" -version = "1.0.4+wasi-0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" -dependencies = [ - "wit-bindgen", -] - -[[package]] -name = "wasm-bindgen" -version = "0.2.126" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" -dependencies = [ - "cfg-if", - "once_cell", - "rustversion", - "wasm-bindgen-macro", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.126" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.126" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" -dependencies = [ - "bumpalo", - "proc-macro2", - "quote", - "syn 2.0.119", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.126" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "web-sys" -version = "0.3.103" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "web-time" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "winapi-util" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" -dependencies = [ - "windows-sys", -] - -[[package]] -name = "windows-link" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" - -[[package]] -name = "windows-sys" -version = "0.61.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" -dependencies = [ - "windows-link", -] - -[[package]] -name = "wit-bindgen" -version = "0.57.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" - -[[package]] -name = "wyz" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" -dependencies = [ - "tap", -] - -[[package]] -name = "xxhash-rust" -version = "0.8.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" - -[[package]] -name = "yada" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53c3bb06259642a57b4ea1bf2a8260f7d94b7b78a096c46f193318918d925f61" - -[[package]] -name = "zerocopy" -version = "0.8.55" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.55" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "zmij" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/bindings/python-pipeline/Cargo.toml b/bindings/python-pipeline/Cargo.toml deleted file mode 100644 index 643f4e5f7..000000000 --- a/bindings/python-pipeline/Cargo.toml +++ /dev/null @@ -1,24 +0,0 @@ -[package] -name = "tokenizers-pipeline-python" -version = "0.1.0" -edition = "2024" -description = "Experimental Python bindings for the tokenizers PipelineTokenizer" -license = "Apache-2.0" - -[lib] -name = "_native" -crate-type = ["cdylib", "rlib"] - -[dependencies] -pyo3 = { version = "=0.29", features = ["abi3-py310", "experimental-inspect"] } -numpy = "0.29" -rayon = "1.10" -libc = "0.2" -serde = "1.0" -serde_json = "1.0" -tk-encode = { path = "../../tokenizers/tk-encode", features = ["fancy-regex"] } -tk-train = { path = "../../tokenizers/tk-train" } - -[features] -default = [] -ext-module = ["pyo3/extension-module"] diff --git a/bindings/python-pipeline/Makefile b/bindings/python-pipeline/Makefile deleted file mode 100644 index af4f2d6cf..000000000 --- a/bindings/python-pipeline/Makefile +++ /dev/null @@ -1,36 +0,0 @@ -PYTHON := .venv/bin/python - -# Everything needed to hack on the bindings: venv, deps, release build. -.PHONY: dev -dev: .venv - . .venv/bin/activate && maturin develop --release - -.venv: - uv venv .venv - uv pip install --python $(PYTHON) maturin numpy tokenizers - -# Regenerate the .pyi stubs from the built extension. Run after `make dev`. -.PHONY: stubs -stubs: - cargo run --manifest-path tools/stub-gen/Cargo.toml - -# Run the end-to-end examples (train, pretrained parity, threading). -.PHONY: examples -examples: dev - $(PYTHON) examples/01_train_and_encode.py - $(PYTHON) examples/02_pretrained.py - $(PYTHON) examples/03_threading.py - -# Benchmark against the released tokenizers wheel (same script CI runs). -.PHONY: bench -bench: dev - $(PYTHON) benches/bench_vs_release.py - -.PHONY: lint -lint: - cargo fmt --check - cargo clippy --all-targets -- -D warnings - -.PHONY: clean -clean: - rm -rf .venv target tools/stub-gen/target diff --git a/bindings/python-pipeline/README.md b/bindings/python-pipeline/README.md deleted file mode 100644 index 63d23ef02..000000000 --- a/bindings/python-pipeline/README.md +++ /dev/null @@ -1,96 +0,0 @@ -# tokenizers-pipeline - -Experimental Python bindings for 🤗 tokenizers, built on the `PipelineTokenizer` -encode path. Same `tokenizer.json` files, same ids, and much faster through -Python: encode never holds the GIL, batches run multi-threaded in Rust, inputs -are borrowed instead of copied, and ids come back as `numpy.uint32` arrays -without a copy. - -```python -import tokenizers_pipeline as tp - -tok = tp.Tokenizer.from_file("tokenizer.json") -ids = tok.encode("Hello world", add_special_tokens=False) # np.ndarray[uint32] -batch = tok.encode_batch(lines, add_special_tokens=False) # list of arrays -``` - -Training and in-place modification work too: - -```python -tok = tp.Tokenizer(tp.models.BPE()) -tok.normalizer = tp.normalizers.Lowercase() -tok.pre_tokenizer = tp.pre_tokenizers.Whitespace() -tok.train_from_iterator(lines, trainer=tp.trainers.BpeTrainer(vocab_size=30000)) -tok.save("tokenizer.json") -``` - -Not there yet (loud errors, never wrong ids): `decode`, post-processor -templates (`[CLS]`/`` insertion — pass `add_special_tokens=False`), and the -`Metaspace` pre-tokenizer (t5-style files). - -## Build and use locally - -Requirements: Rust (stable), [uv](https://docs.astral.sh/uv/), Python ≥ 3.10. - -```sh -cd bindings/python-pipeline -make dev # venv + deps + release build, installed editable -source .venv/bin/activate -python -c "import tokenizers_pipeline; print(tokenizers_pipeline.__version__)" -``` - -Rebuild after changing Rust code with `make dev` again (or `maturin develop ---release` inside the venv). Always use `--release`: a debug build encodes -10-100× slower and any timing you take from it is meaningless. - -To build a distributable wheel instead: `maturin build --release` (find it in -`target/wheels/`). - -Other targets: - -```sh -make examples # run the three end-to-end examples (needs ../../tokenizers/data) -make bench # benchmark against the released tokenizers wheel -make stubs # regenerate the .pyi type stubs from the built extension -make lint # cargo fmt --check + clippy -D warnings -``` - -The examples and the benchmark read test data from `../../tokenizers/data`. -Fetch it once with `make -C ../../tokenizers fixtures bench-models data/big.txt` -(needs `HF_TOKEN` for the mirror repo). - -## Type stubs are generated - -Do not edit the `.pyi` files under `py_src/` by hand. They are produced by -`tools/stub-gen`, which reads the introspection metadata pyo3 embeds in the -built extension — so run `make dev` first, then `make stubs`. Docstrings and -signatures come from the Rust sources; return types that introspection cannot -see (numpy arrays, `Self`) are declared with -`#[pyo3(signature = (...) -> "Type")]` annotations in the Rust code. - -## How it works - -A `Tokenizer` holds two things behind one lock: - -- the **spec** — the plain Rust `Tokenizer`, the serializable source of truth. - Setters, `train*`, and `add_*` write here. -- the **compiled pipeline** — an immutable `Arc` the encode - methods share with worker threads. Any mutation drops it; the next encode - rebuilds it once. Configurations the pipeline cannot run fail at that point - with the reason, never with different ids. - -Every method releases the GIL before touching the lock — enforced at compile -time by `DetachedRwLock` (see `src/detached_lock.rs`), with a clippy ban on -`Python::attach` as the backstop. - -## Benchmark - -`benches/bench_vs_release.py` times `encode_batch` end-to-end through Python -against the latest released `tokenizers` wheel, on the same corpora and ~10 KiB -chunking as the Rust benchmark (`tk-encode/examples/fixture_bench.rs`): every -fixture under `data/fixtures/{lang,modalities}`, warmed up, median of N runs, -single-thread per fixture plus one multi-thread sweep, ids verified equal -before timing. CI runs it in the `python-bindings-bench` job of the Pipeline -Benchmark workflow, posts the table to the run's step summary, and the report -job renders it as a chart (`.github/scripts/render_python_bench.py`) appended -to the benchmark section in the PR description, next to the Rust charts. diff --git a/bindings/python-pipeline/py_src/tokenizers_pipeline/__init__.py b/bindings/python-pipeline/py_src/tokenizers_pipeline/__init__.py deleted file mode 100644 index 003333d64..000000000 --- a/bindings/python-pipeline/py_src/tokenizers_pipeline/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Fast tokenizers built on the pipeline encode path. Start with `Tokenizer`.""" - -from ._native import ( - AddedToken, - Tokenizer, - TokenizersError, - __version__, -) -from . import models, normalizers, pre_tokenizers, trainers - -__all__ = [ - "AddedToken", - "Tokenizer", - "TokenizersError", - "__version__", - "models", - "normalizers", - "pre_tokenizers", - "trainers", -] diff --git a/bindings/python-pipeline/py_src/tokenizers_pipeline/__init__.pyi b/bindings/python-pipeline/py_src/tokenizers_pipeline/__init__.pyi deleted file mode 100644 index 75859f3f1..000000000 --- a/bindings/python-pipeline/py_src/tokenizers_pipeline/__init__.pyi +++ /dev/null @@ -1,173 +0,0 @@ -import numpy as np -import numpy.typing as npt - -""" -Fast tokenizers built on the pipeline encode path. Start with `Tokenizer`. -""" - -from tokenizers_pipeline.models import Model -from tokenizers_pipeline.normalizers import Normalizer -from tokenizers_pipeline.pre_tokenizers import PreTokenizer -from tokenizers_pipeline.trainers import Trainer -from _typeshed import Incomplete -from collections.abc import Sequence -from os import PathLike -from typing import Any, Final, final -__version__: Final[str] - -@final -class AddedToken: - """ - A token added to the vocabulary after training, with options for how it - is matched in text: `single_word` only matches when it stands alone (not - inside a word); `lstrip`/`rstrip` also swallow the whitespace before/after - it; `normalized` matches against normalized instead of raw text (defaults - to the opposite of `special`); `special` marks template tokens like "" - that decoding should be able to skip. - """ - def __new__(cls, /, content: str, *, single_word: bool = False, lstrip: bool = False, rstrip: bool = False, normalized: bool |None = None, special: bool = False) -> AddedToken: ... - def __repr__(self, /) -> str: ... - @property - def content(self, /) -> str: ... - @property - def lstrip(self, /) -> bool: ... - @property - def normalized(self, /) -> bool: ... - @property - def rstrip(self, /) -> bool: ... - @property - def single_word(self, /) -> bool: ... - @property - def special(self, /) -> bool: ... - -@final -class Tokenizer: - """ - A tokenizer: a model plus its optional normalizer and pre-tokenizer. - - Create one from a model (`Tokenizer(models.BPE())`), a file - (`Tokenizer.from_file`), or the Hub (`Tokenizer.from_pretrained`). - Changes — assigning components, training, adding tokens — apply to the - serializable definition; encoding runs a compiled pipeline that is rebuilt - automatically after any change. A definition the pipeline cannot run - raises `TokenizersError` at that point, with the reason. - """ - def __new__(cls, /, model: Model) -> Tokenizer: - """ - Create an untrained tokenizer from a model. - """ - def __reduce__(self, /) -> tuple[Any, tuple[bytes]]: ... - def __repr__(self, /) -> str: ... - def add_special_tokens(self, /, tokens: Sequence[str |AddedToken]) -> int: - """ - Add special tokens ("", "[CLS]", …) to the vocabulary. Same as - `add_tokens`, but every token is marked `special`. Returns how many - were actually new. - """ - def add_tokens(self, /, tokens: Sequence[str |AddedToken]) -> int: - """ - Add tokens to the vocabulary and match them in the input text from now - on. Plain strings match with default options; pass `AddedToken` to - control matching. Returns how many were actually new. - """ - def decode(self, /, ids: Sequence[int], *, skip_special_tokens: bool = True) -> str: - """ - Not implemented yet: decoding is not part of the encode pipeline. - """ - def encode(self, /, text: str, *, add_special_tokens: bool = True) -> "npt.NDArray[np.uint32]": - """ - Encode `text` into token ids. - - Runs entirely outside the interpreter lock and returns a `numpy.uint32` - array backed by the Rust output buffer (no copy). - """ - def encode_batch(self, /, texts: Sequence[str], *, add_special_tokens: bool = True) -> "list[npt.NDArray[np.uint32]]": - """ - Encode a batch of texts, in parallel across Rust threads (respects - `TOKENIZERS_PARALLELISM`), without holding the interpreter lock. - Input strings are borrowed, not copied; each output is a `numpy.uint32` - array backed by its Rust buffer. - """ - @staticmethod - def from_buffer(buffer: Sequence[int]) -> "Tokenizer": - """ - Load a tokenizer from the bytes of a `tokenizer.json` file. - """ - @staticmethod - def from_file(path: str |PathLike[str]) -> "Tokenizer": - """ - Load a tokenizer from a `tokenizer.json` file. - """ - @staticmethod - def from_pretrained(identifier: str, *, revision: str = ..., token: str |None = None) -> "Tokenizer": - """ - Download `tokenizer.json` from a model on the Hugging Face Hub (requires - the `huggingface_hub` package) and load it. - """ - def get_vocab(self, /, *, with_added_tokens: bool = True) -> dict[str, int]: - """ - The whole vocabulary as a dict. This copies every entry; prefer - `token_to_id` for lookups. - """ - def get_vocab_size(self, /, *, with_added_tokens: bool = True) -> int: - """ - Number of entries in the vocabulary. `with_added_tokens=False` counts - only what the model was trained with. - """ - def id_to_token(self, /, id: int) -> str |None: - """ - The token behind `id`, or None if the id is out of range. - """ - @property - def model(self, /) -> Model: - """ - The model in use by this tokenizer (a copy: reassign to change it). - """ - @model.setter - def model(self, /, model: Model) -> None: ... - @property - def normalizer(self, /) -> Normalizer |None: - """ - The optional normalizer in use by this tokenizer (a copy: reassign to - change it). - """ - @normalizer.setter - def normalizer(self, /, normalizer: Normalizer |None) -> None: ... - @property - def pre_tokenizer(self, /) -> PreTokenizer |None: - """ - The optional pre-tokenizer in use by this tokenizer (a copy: reassign - to change it). - """ - @pre_tokenizer.setter - def pre_tokenizer(self, /, pre_tokenizer: PreTokenizer |None) -> None: ... - def save(self, /, path: str |PathLike[str], *, pretty: bool = True) -> None: - """ - Save the tokenizer definition to a `tokenizer.json` file. - """ - def to_str(self, /, *, pretty: bool = False) -> str: - """ - Serialize the tokenizer definition as a `tokenizer.json` string. - """ - def token_to_id(self, /, token: str) -> int |None: - """ - The id of `token`, or None if it is not in the vocabulary. - """ - def train(self, /, files: Sequence[str], *, trainer: Trainer |None = None) -> None: - """ - Train the model's vocabulary on text files (one sequence per line). - Without a `trainer`, the model's default trainer is used. - """ - def train_from_iterator(self, /, iterator: Any, *, trainer: Trainer |None = None) -> None: - """ - Train the model's vocabulary from any iterator of `str`. Without a - `trainer`, the model's default trainer is used. - - The interpreter lock is only re-acquired to refill an internal buffer - (256 sequences at a time); the training itself runs multi-threaded in - Rust with the lock released. - """ - -def __getattr__(name: str) -> Incomplete: ... - -class TokenizersError(Exception): ... diff --git a/bindings/python-pipeline/py_src/tokenizers_pipeline/models/__init__.py b/bindings/python-pipeline/py_src/tokenizers_pipeline/models/__init__.py deleted file mode 100644 index 6ae6ab347..000000000 --- a/bindings/python-pipeline/py_src/tokenizers_pipeline/models/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -"""The algorithms that turn pre-tokenized pieces into token ids.""" - -from .._native import models as _models - -Model = _models.Model -BPE = _models.BPE -WordPiece = _models.WordPiece -WordLevel = _models.WordLevel -Unigram = _models.Unigram - -__all__ = ["Model", "BPE", "WordPiece", "WordLevel", "Unigram"] diff --git a/bindings/python-pipeline/py_src/tokenizers_pipeline/normalizers/__init__.py b/bindings/python-pipeline/py_src/tokenizers_pipeline/normalizers/__init__.py deleted file mode 100644 index c66373610..000000000 --- a/bindings/python-pipeline/py_src/tokenizers_pipeline/normalizers/__init__.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Text cleanup that runs before the text is split.""" - -from .._native import normalizers as _normalizers - -Normalizer = _normalizers.Normalizer -BertNormalizer = _normalizers.BertNormalizer -Lowercase = _normalizers.Lowercase -NFC = _normalizers.NFC -NFD = _normalizers.NFD -NFKC = _normalizers.NFKC -NFKD = _normalizers.NFKD -Prepend = _normalizers.Prepend -Replace = _normalizers.Replace -Sequence = _normalizers.Sequence -Strip = _normalizers.Strip -StripAccents = _normalizers.StripAccents - -__all__ = [ - "Normalizer", - "BertNormalizer", - "Lowercase", - "NFC", - "NFD", - "NFKC", - "NFKD", - "Prepend", - "Replace", - "Sequence", - "Strip", - "StripAccents", -] diff --git a/bindings/python-pipeline/py_src/tokenizers_pipeline/pre_tokenizers/__init__.py b/bindings/python-pipeline/py_src/tokenizers_pipeline/pre_tokenizers/__init__.py deleted file mode 100644 index 49c12046d..000000000 --- a/bindings/python-pipeline/py_src/tokenizers_pipeline/pre_tokenizers/__init__.py +++ /dev/null @@ -1,31 +0,0 @@ -"""How text is cut into pieces before the model runs.""" - -from .._native import pre_tokenizers as _pre_tokenizers - -PreTokenizer = _pre_tokenizers.PreTokenizer -BertPreTokenizer = _pre_tokenizers.BertPreTokenizer -ByteLevel = _pre_tokenizers.ByteLevel -CharDelimiterSplit = _pre_tokenizers.CharDelimiterSplit -Digits = _pre_tokenizers.Digits -FixedLength = _pre_tokenizers.FixedLength -Punctuation = _pre_tokenizers.Punctuation -Sequence = _pre_tokenizers.Sequence -Split = _pre_tokenizers.Split -UnicodeScripts = _pre_tokenizers.UnicodeScripts -Whitespace = _pre_tokenizers.Whitespace -WhitespaceSplit = _pre_tokenizers.WhitespaceSplit - -__all__ = [ - "PreTokenizer", - "BertPreTokenizer", - "ByteLevel", - "CharDelimiterSplit", - "Digits", - "FixedLength", - "Punctuation", - "Sequence", - "Split", - "UnicodeScripts", - "Whitespace", - "WhitespaceSplit", -] diff --git a/bindings/python-pipeline/py_src/tokenizers_pipeline/py.typed b/bindings/python-pipeline/py_src/tokenizers_pipeline/py.typed deleted file mode 100644 index e69de29bb..000000000 diff --git a/bindings/python-pipeline/py_src/tokenizers_pipeline/trainers/__init__.py b/bindings/python-pipeline/py_src/tokenizers_pipeline/trainers/__init__.py deleted file mode 100644 index 99cc2a3eb..000000000 --- a/bindings/python-pipeline/py_src/tokenizers_pipeline/trainers/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Recipes for learning a vocabulary from text.""" - -from .._native import trainers as _trainers - -Trainer = _trainers.Trainer -BpeTrainer = _trainers.BpeTrainer -UnigramTrainer = _trainers.UnigramTrainer -WordLevelTrainer = _trainers.WordLevelTrainer -WordPieceTrainer = _trainers.WordPieceTrainer - -__all__ = [ - "Trainer", - "BpeTrainer", - "UnigramTrainer", - "WordLevelTrainer", - "WordPieceTrainer", -] diff --git a/bindings/python-pipeline/py_src/tokenizers_pipeline/trainers/__init__.pyi b/bindings/python-pipeline/py_src/tokenizers_pipeline/trainers/__init__.pyi deleted file mode 100644 index 42dfd5207..000000000 --- a/bindings/python-pipeline/py_src/tokenizers_pipeline/trainers/__init__.pyi +++ /dev/null @@ -1,53 +0,0 @@ -""" -Recipes for learning a vocabulary from text. -""" - -from tokenizers_pipeline import AddedToken -from collections.abc import Sequence -from typing import final - -@final -class BpeTrainer(Trainer): - """ - Learns a BPE vocabulary: keeps merging the most frequent pair until - `vocab_size` is reached, ignoring pairs seen fewer than `min_frequency` - times. `special_tokens` get the first ids. `limit_alphabet` caps how many - distinct characters are kept; `initial_alphabet` forces characters in even - if the data never shows them; `max_token_length` caps merged token length. - """ - def __new__(cls, /, *, vocab_size: int = 30000, min_frequency: int = 0, special_tokens: Sequence[str |AddedToken] = ..., limit_alphabet: int |None = None, initial_alphabet: Sequence[str] = ..., continuing_subword_prefix: str |None = None, end_of_word_suffix: str |None = None, max_token_length: int |None = None, show_progress: bool = True) -> BpeTrainer: ... - -class Trainer: - """ - Base class for all trainers. - - A trainer is the recipe for learning a model's vocabulary from text; pass - one to `Tokenizer.train` or `train_from_iterator`. Trainers are plain - configuration values — training copies them and writes nothing back. - """ - def __repr__(self, /) -> str: ... - -@final -class UnigramTrainer(Trainer): - """ - Learns a Unigram vocabulary: starts from a large candidate set and prunes - it by `shrinking_factor` each round until `vocab_size` pieces remain. - `unk_token` names the fallback piece for unknown characters. - """ - def __new__(cls, /, *, vocab_size: int = 8000, special_tokens: Sequence[str |AddedToken] = ..., initial_alphabet: Sequence[str] = ..., unk_token: str |None = None, shrinking_factor: float = 0.75, max_piece_length: int = 16, n_sub_iterations: int = 2, show_progress: bool = True) -> UnigramTrainer: ... - -@final -class WordLevelTrainer(Trainer): - """ - Learns a WordLevel vocabulary: the `vocab_size` most frequent words, - keeping only those seen at least `min_frequency` times. - """ - def __new__(cls, /, *, vocab_size: int = 30000, min_frequency: int = 0, special_tokens: Sequence[str |AddedToken] = ..., show_progress: bool = True) -> WordLevelTrainer: ... - -@final -class WordPieceTrainer(Trainer): - """ - Learns a WordPiece vocabulary. Same knobs as `BpeTrainer`, plus the - continuation prefix ("##" by default). - """ - def __new__(cls, /, *, vocab_size: int = 30000, min_frequency: int = 0, special_tokens: Sequence[str |AddedToken] = ..., limit_alphabet: int |None = None, initial_alphabet: Sequence[str] = ..., continuing_subword_prefix: str = ..., end_of_word_suffix: str |None = None, show_progress: bool = True) -> WordPieceTrainer: ... diff --git a/bindings/python-pipeline/pyproject.toml b/bindings/python-pipeline/pyproject.toml deleted file mode 100644 index 9bce55d1b..000000000 --- a/bindings/python-pipeline/pyproject.toml +++ /dev/null @@ -1,19 +0,0 @@ -[build-system] -requires = ["maturin>=1.5,<2.0"] -build-backend = "maturin" - -[project] -name = "tokenizers-pipeline" -description = "Experimental Python bindings for the tokenizers PipelineTokenizer" -requires-python = ">=3.10" -dependencies = ["numpy>=1.24"] -dynamic = ["version"] - -[project.optional-dependencies] -hub = ["huggingface_hub>=0.16.4"] - -[tool.maturin] -python-source = "py_src" -module-name = "tokenizers_pipeline._native" -bindings = "pyo3" -features = ["ext-module"] diff --git a/bindings/python-pipeline/src/error.rs b/bindings/python-pipeline/src/error.rs deleted file mode 100644 index ccd5f7db8..000000000 --- a/bindings/python-pipeline/src/error.rs +++ /dev/null @@ -1,9 +0,0 @@ -use pyo3::PyErr; -use pyo3::create_exception; -use pyo3::exceptions::PyException; - -create_exception!(tokenizers_pipeline, TokenizersError, PyException); - -pub fn to_pyerr(e: tk_encode::Error) -> PyErr { - TokenizersError::new_err(e.to_string()) -} diff --git a/bindings/python-pipeline/src/lib.rs b/bindings/python-pipeline/src/lib.rs deleted file mode 100644 index 6bcc17ebe..000000000 --- a/bindings/python-pipeline/src/lib.rs +++ /dev/null @@ -1,69 +0,0 @@ -#![warn(clippy::all)] - -pub mod added_token; -pub mod detached_lock; -pub mod error; -pub mod models; -pub mod normalizers; -pub mod pre_tokenizers; -pub mod tokenizer; -pub mod trainers; - -use pyo3::prelude::*; - -/// Components repr as their tokenizer.json serialization: compact, and always -/// in sync with what `Tokenizer.save` writes. -pub fn component_repr(component: &T) -> String { - serde_json::to_string(component).unwrap_or_else(|_| "".to_owned()) -} - -// Forked children of a process that used our rayon threads would inherit a -// poisoned thread pool; disable parallelism there unless the user configured -// it explicitly (same behavior as the v1 bindings). -#[cfg(target_family = "unix")] -extern "C" fn child_after_fork() { - use std::sync::atomic::Ordering; - use tk_encode::utils::parallelism::{is_parallelism_configured, set_parallelism}; - if crate::tokenizer::USED_PARALLELISM.load(Ordering::SeqCst) && !is_parallelism_configured() { - set_parallelism(false); - } -} - -/// Fast tokenizers built on the pipeline encode path. Start with `Tokenizer`. -#[pymodule(gil_used = false)] -pub mod _native { - use super::*; - - #[pymodule_export] - pub use super::added_token::PyAddedToken; - #[pymodule_export] - pub use super::error::TokenizersError; - #[pymodule_export] - pub use super::tokenizer::PyTokenizer; - - #[pymodule_export] - pub use super::models::models; - #[pymodule_export] - pub use super::normalizers::normalizers; - #[pymodule_export] - pub use super::pre_tokenizers::pre_tokenizers; - #[pymodule_export] - pub use super::trainers::trainers; - - #[allow(non_upper_case_globals)] - #[pymodule_export] - pub const __version__: &str = env!("CARGO_PKG_VERSION"); - - #[pymodule_init] - fn init(_m: &Bound<'_, PyModule>) -> PyResult<()> { - #[cfg(target_family = "unix")] - { - use std::sync::Once; - static REGISTER_FORK_CALLBACK: Once = Once::new(); - REGISTER_FORK_CALLBACK.call_once(|| unsafe { - libc::pthread_atfork(None, None, Some(child_after_fork)); - }); - } - Ok(()) - } -} diff --git a/bindings/python-pipeline/src/models.rs b/bindings/python-pipeline/src/models.rs deleted file mode 100644 index 0e82449d1..000000000 --- a/bindings/python-pipeline/src/models.rs +++ /dev/null @@ -1,162 +0,0 @@ -use pyo3::prelude::*; -use tk_encode::models::ModelWrapper; -use tk_encode::models::bpe::BPE; -use tk_encode::models::unigram::Unigram; -use tk_encode::models::wordlevel::WordLevel; -use tk_encode::models::wordpiece::WordPiece; - -use crate::error::to_pyerr; - -/// Base class for all models. -/// -/// The model is the trained part of a tokenizer: it turns each pre-tokenized -/// piece into token ids using its vocabulary. Models are immutable values — -/// assigning one to a tokenizer copies it. -#[pyclass( - frozen, - subclass, - name = "Model", - module = "tokenizers_pipeline.models" -)] -pub struct PyModel { - pub inner: ModelWrapper, -} - -#[pymethods] -impl PyModel { - fn __repr__(&self) -> String { - crate::component_repr(&self.inner) - } -} - -pub fn wrap_model(py: Python<'_>, inner: ModelWrapper) -> PyResult> { - let base = PyModel { - inner: inner.clone(), - }; - let init = PyClassInitializer::from(base); - let obj = match inner { - ModelWrapper::BPE(_) => Bound::new(py, init.add_subclass(PyBPE))?.into_super(), - ModelWrapper::WordPiece(_) => Bound::new(py, init.add_subclass(PyWordPiece))?.into_super(), - ModelWrapper::WordLevel(_) => Bound::new(py, init.add_subclass(PyWordLevel))?.into_super(), - ModelWrapper::Unigram(_) => Bound::new(py, init.add_subclass(PyUnigram))?.into_super(), - }; - Ok(obj.unbind()) -} - -/// Byte-Pair Encoding: builds tokens by applying the merges learned during -/// training. `unk_token` stands in for characters the vocabulary cannot -/// represent; `byte_fallback` encodes them as raw bytes instead. `dropout` -/// randomly skips merges (a training-time regularization). `ignore_merges` -/// looks whole pieces up in the vocabulary before merging. -#[pyclass(frozen, extends = PyModel, name = "BPE", module = "tokenizers_pipeline.models")] -pub struct PyBPE; - -#[pymethods] -impl PyBPE { - #[new] - #[pyo3(signature = (*, unk_token = None, dropout = None, fuse_unk = false, byte_fallback = false, ignore_merges = false))] - fn new( - unk_token: Option, - dropout: Option, - fuse_unk: bool, - byte_fallback: bool, - ignore_merges: bool, - ) -> PyResult> { - let mut builder = BPE::builder() - .fuse_unk(fuse_unk) - .byte_fallback(byte_fallback) - .ignore_merges(ignore_merges); - if let Some(unk) = unk_token { - builder = builder.unk_token(unk); - } - if let Some(d) = dropout { - builder = builder.dropout(d); - } - let bpe = builder.build().map_err(to_pyerr)?; - Ok(PyClassInitializer::from(PyModel { inner: bpe.into() }).add_subclass(PyBPE)) - } - - /// Load a BPE from the legacy vocab.json + merges.txt format. - #[staticmethod] - #[pyo3(signature = (vocab, merges, *, unk_token = None) -> "BPE")] - fn from_file( - py: Python<'_>, - vocab: &str, - merges: &str, - unk_token: Option, - ) -> PyResult> { - let mut builder = BPE::from_file(vocab, merges); - if let Some(unk) = unk_token { - builder = builder.unk_token(unk); - } - let bpe = py.detach(|| builder.build()).map_err(to_pyerr)?; - wrap_model(py, bpe.into()) - } -} - -/// The BERT model: greedily matches the longest vocabulary entry, marking -/// word continuations with a prefix ("##" by default). A piece longer than -/// `max_input_chars_per_word` becomes `unk_token` outright. -#[pyclass(frozen, extends = PyModel, name = "WordPiece", module = "tokenizers_pipeline.models")] -pub struct PyWordPiece; - -#[pymethods] -impl PyWordPiece { - #[new] - #[pyo3(signature = (*, unk_token = String::from("[UNK]"), continuing_subword_prefix = String::from("##"), max_input_chars_per_word = 100))] - fn new( - unk_token: String, - continuing_subword_prefix: String, - max_input_chars_per_word: usize, - ) -> PyResult> { - let wp = WordPiece::builder() - .unk_token(unk_token) - .continuing_subword_prefix(continuing_subword_prefix) - .max_input_chars_per_word(max_input_chars_per_word) - .build() - .map_err(to_pyerr)?; - Ok(PyClassInitializer::from(PyModel { inner: wp.into() }).add_subclass(PyWordPiece)) - } -} - -/// The simplest model: one whole word, one id. Words outside the vocabulary -/// become `unk_token`. -#[pyclass(frozen, extends = PyModel, name = "WordLevel", module = "tokenizers_pipeline.models")] -pub struct PyWordLevel; - -#[pymethods] -impl PyWordLevel { - #[new] - #[pyo3(signature = (*, unk_token = String::from("[UNK]")))] - fn new(unk_token: String) -> PyResult> { - let wl = WordLevel::builder() - .unk_token(unk_token) - .build() - .map_err(to_pyerr)?; - Ok(PyClassInitializer::from(PyModel { inner: wl.into() }).add_subclass(PyWordLevel)) - } -} - -/// The SentencePiece Unigram model: picks the most probable segmentation -/// under a learned piece vocabulary. Starts empty — train it, or load a -/// tokenizer.json. -#[pyclass(frozen, extends = PyModel, name = "Unigram", module = "tokenizers_pipeline.models")] -pub struct PyUnigram; - -#[pymethods] -impl PyUnigram { - #[new] - fn new() -> PyClassInitializer { - PyClassInitializer::from(PyModel { - inner: Unigram::default().into(), - }) - .add_subclass(PyUnigram) - } -} - -/// The algorithms that turn pre-tokenized pieces into token ids. -#[pymodule(gil_used = false)] -pub mod models { - #[pymodule_export] - pub use super::{PyBPE, PyModel, PyUnigram, PyWordLevel, PyWordPiece}; -} diff --git a/bindings/python-pipeline/src/normalizers.rs b/bindings/python-pipeline/src/normalizers.rs deleted file mode 100644 index f7a1d0d7b..000000000 --- a/bindings/python-pipeline/src/normalizers.rs +++ /dev/null @@ -1,223 +0,0 @@ -use pyo3::prelude::*; -use tk_encode::normalizers::{ - BertNormalizer, Lowercase, NFC, NFD, NFKC, NFKD, NormalizerWrapper, Prepend, Replace, Sequence, - Strip, StripAccents, -}; - -use crate::error::to_pyerr; - -/// Base class for all normalizers. -/// -/// A normalizer rewrites text before it is split: cleanup, case-folding, -/// Unicode normalization. Normalizers are immutable values — assigning one to -/// a tokenizer copies it. -#[pyclass( - frozen, - subclass, - name = "Normalizer", - module = "tokenizers_pipeline.normalizers" -)] -pub struct PyNormalizer { - pub inner: NormalizerWrapper, -} - -#[pymethods] -impl PyNormalizer { - fn __repr__(&self) -> String { - crate::component_repr(&self.inner) - } -} - -pub fn wrap_normalizer(py: Python<'_>, inner: NormalizerWrapper) -> PyResult> { - let base = PyNormalizer { - inner: inner.clone(), - }; - let init = PyClassInitializer::from(base); - let obj = match inner { - NormalizerWrapper::BertNormalizer(_) => { - Bound::new(py, init.add_subclass(PyBertNormalizer))?.into_super() - } - NormalizerWrapper::StripNormalizer(_) => { - Bound::new(py, init.add_subclass(PyStrip))?.into_super() - } - NormalizerWrapper::StripAccents(_) => { - Bound::new(py, init.add_subclass(PyStripAccents))?.into_super() - } - NormalizerWrapper::NFC(_) => Bound::new(py, init.add_subclass(PyNFC))?.into_super(), - NormalizerWrapper::NFD(_) => Bound::new(py, init.add_subclass(PyNFD))?.into_super(), - NormalizerWrapper::NFKC(_) => Bound::new(py, init.add_subclass(PyNFKC))?.into_super(), - NormalizerWrapper::NFKD(_) => Bound::new(py, init.add_subclass(PyNFKD))?.into_super(), - NormalizerWrapper::Sequence(_) => { - Bound::new(py, init.add_subclass(PySequence))?.into_super() - } - NormalizerWrapper::Lowercase(_) => { - Bound::new(py, init.add_subclass(PyLowercase))?.into_super() - } - NormalizerWrapper::Replace(_) => Bound::new(py, init.add_subclass(PyReplace))?.into_super(), - NormalizerWrapper::Prepend(_) => Bound::new(py, init.add_subclass(PyPrepend))?.into_super(), - // Loadable from tokenizer.json but not constructible from Python: exposed as the base class. - NormalizerWrapper::Nmt(_) - | NormalizerWrapper::Precompiled(_) - | NormalizerWrapper::ByteLevel(_) => Bound::new(py, init)?, - }; - Ok(obj.unbind()) -} - -macro_rules! unit_normalizer { - ($pyname:ident, $name:literal, $inner:expr, $doc:literal) => { - #[doc = $doc] - #[pyclass(frozen, extends = PyNormalizer, name = $name, module = "tokenizers_pipeline.normalizers")] - pub struct $pyname; - - #[pymethods] - impl $pyname { - #[new] - fn new() -> PyClassInitializer { - PyClassInitializer::from(PyNormalizer { inner: $inner.into() }).add_subclass($pyname) - } - } - }; -} - -unit_normalizer!( - PyNFC, - "NFC", - NFC, - "Unicode NFC: recombines split characters (e + ´ becomes é)." -); -unit_normalizer!( - PyNFD, - "NFD", - NFD, - "Unicode NFD: splits characters into base + accents (é becomes e + ´)." -); -unit_normalizer!( - PyNFKC, - "NFKC", - NFKC, - "Unicode NFKC: NFC, plus compatibility replacements (fi becomes fi)." -); -unit_normalizer!( - PyNFKD, - "NFKD", - NFKD, - "Unicode NFKD: NFD, plus compatibility replacements (fi becomes fi)." -); -unit_normalizer!( - PyLowercase, - "Lowercase", - Lowercase, - "Lowercases everything." -); -unit_normalizer!( - PyStripAccents, - "StripAccents", - StripAccents, - "Removes accents (é becomes e). Only works on decomposed text: put NFD before it." -); - -/// Removes whitespace at the start and/or end of the text. -#[pyclass(frozen, extends = PyNormalizer, name = "Strip", module = "tokenizers_pipeline.normalizers")] -pub struct PyStrip; - -#[pymethods] -impl PyStrip { - #[new] - #[pyo3(signature = (*, left = true, right = true))] - fn new(left: bool, right: bool) -> PyClassInitializer { - PyClassInitializer::from(PyNormalizer { - inner: Strip::new(left, right).into(), - }) - .add_subclass(PyStrip) - } -} - -/// Replaces every occurrence of `pattern` with `content`. With `regex=True` -/// the pattern is a regular expression. -#[pyclass(frozen, extends = PyNormalizer, name = "Replace", module = "tokenizers_pipeline.normalizers")] -pub struct PyReplace; - -#[pymethods] -impl PyReplace { - #[new] - #[pyo3(signature = (pattern, content, *, regex = false))] - fn new(pattern: &str, content: &str, regex: bool) -> PyResult> { - use tk_encode::normalizers::replace::ReplacePattern; - let pattern = if regex { - ReplacePattern::Regex(pattern.to_owned()) - } else { - ReplacePattern::String(pattern.to_owned()) - }; - let replace = Replace::new(pattern, content).map_err(to_pyerr)?; - Ok(PyClassInitializer::from(PyNormalizer { - inner: replace.into(), - }) - .add_subclass(PyReplace)) - } -} - -/// Puts a fixed string in front of the text (SentencePiece prepends "▁"). -#[pyclass(frozen, extends = PyNormalizer, name = "Prepend", module = "tokenizers_pipeline.normalizers")] -pub struct PyPrepend; - -#[pymethods] -impl PyPrepend { - #[new] - fn new(prepend: String) -> PyClassInitializer { - PyClassInitializer::from(PyNormalizer { - inner: Prepend::new(prepend).into(), - }) - .add_subclass(PyPrepend) - } -} - -/// The BERT cleanup: removes control characters, puts spaces around CJK -/// characters, and optionally strips accents and lowercases. -/// `strip_accents=None` means "follow the lowercase setting", like the -/// original BERT. -#[pyclass(frozen, extends = PyNormalizer, name = "BertNormalizer", module = "tokenizers_pipeline.normalizers")] -pub struct PyBertNormalizer; - -#[pymethods] -impl PyBertNormalizer { - #[new] - #[pyo3(signature = (*, clean_text = true, handle_chinese_chars = true, strip_accents = None, lowercase = true))] - fn new( - clean_text: bool, - handle_chinese_chars: bool, - strip_accents: Option, - lowercase: bool, - ) -> PyClassInitializer { - let inner = BertNormalizer::new(clean_text, handle_chinese_chars, strip_accents, lowercase); - PyClassInitializer::from(PyNormalizer { - inner: inner.into(), - }) - .add_subclass(PyBertNormalizer) - } -} - -/// Runs several normalizers in order. -#[pyclass(frozen, extends = PyNormalizer, name = "Sequence", module = "tokenizers_pipeline.normalizers")] -pub struct PySequence; - -#[pymethods] -impl PySequence { - #[new] - fn new(normalizers: Vec>) -> PyClassInitializer { - let inner: Vec = normalizers.iter().map(|n| n.inner.clone()).collect(); - PyClassInitializer::from(PyNormalizer { - inner: Sequence::new(inner).into(), - }) - .add_subclass(PySequence) - } -} - -/// Text cleanup that runs before the text is split. -#[pymodule(gil_used = false)] -pub mod normalizers { - #[pymodule_export] - pub use super::{ - PyBertNormalizer, PyLowercase, PyNFC, PyNFD, PyNFKC, PyNFKD, PyNormalizer, PyPrepend, - PyReplace, PySequence, PyStrip, PyStripAccents, - }; -} diff --git a/bindings/python-pipeline/src/pre_tokenizers.rs b/bindings/python-pipeline/src/pre_tokenizers.rs deleted file mode 100644 index d9343dd5b..000000000 --- a/bindings/python-pipeline/src/pre_tokenizers.rs +++ /dev/null @@ -1,283 +0,0 @@ -use pyo3::exceptions::PyValueError; -use pyo3::prelude::*; -use tk_encode::pre_tokenizers::PreTokenizerWrapper; -use tk_encode::pre_tokenizers::bert::BertPreTokenizer; -use tk_encode::pre_tokenizers::byte_level::ByteLevel; -use tk_encode::pre_tokenizers::delimiter::CharDelimiterSplit; -use tk_encode::pre_tokenizers::digits::Digits; -use tk_encode::pre_tokenizers::fixed_length::FixedLength; -use tk_encode::pre_tokenizers::punctuation::Punctuation; -use tk_encode::pre_tokenizers::sequence::Sequence; -use tk_encode::pre_tokenizers::split::{Split, SplitPattern}; -use tk_encode::pre_tokenizers::unicode_scripts::UnicodeScripts; -use tk_encode::pre_tokenizers::whitespace::{Whitespace, WhitespaceSplit}; -use tk_encode::tokenizer::SplitDelimiterBehavior; - -use crate::error::to_pyerr; - -pub fn parse_behavior(s: &str) -> PyResult { - match s { - "removed" => Ok(SplitDelimiterBehavior::Removed), - "isolated" => Ok(SplitDelimiterBehavior::Isolated), - "merged_with_previous" => Ok(SplitDelimiterBehavior::MergedWithPrevious), - "merged_with_next" => Ok(SplitDelimiterBehavior::MergedWithNext), - "contiguous" => Ok(SplitDelimiterBehavior::Contiguous), - other => Err(PyValueError::new_err(format!( - "unknown behavior {other:?}; expected one of: removed, isolated, \ - merged_with_previous, merged_with_next, contiguous" - ))), - } -} - -/// Base class for all pre-tokenizers. -/// -/// A pre-tokenizer cuts text into pieces (usually words); the model then turns -/// each piece into token ids. Pre-tokenizers are immutable values — assigning -/// one to a tokenizer copies it. Only pre-tokenizers the encode pipeline can -/// run are constructible here; `Metaspace` is not available yet. -#[pyclass( - frozen, - subclass, - name = "PreTokenizer", - module = "tokenizers_pipeline.pre_tokenizers" -)] -pub struct PyPreTokenizer { - pub inner: PreTokenizerWrapper, -} - -#[pymethods] -impl PyPreTokenizer { - fn __repr__(&self) -> String { - crate::component_repr(&self.inner) - } -} - -pub fn wrap_pre_tokenizer( - py: Python<'_>, - inner: PreTokenizerWrapper, -) -> PyResult> { - let base = PyPreTokenizer { - inner: inner.clone(), - }; - let init = PyClassInitializer::from(base); - let obj = match inner { - PreTokenizerWrapper::BertPreTokenizer(_) => { - Bound::new(py, init.add_subclass(PyBertPreTokenizer))?.into_super() - } - PreTokenizerWrapper::ByteLevel(_) => { - Bound::new(py, init.add_subclass(PyByteLevel))?.into_super() - } - PreTokenizerWrapper::Delimiter(_) => { - Bound::new(py, init.add_subclass(PyCharDelimiterSplit))?.into_super() - } - PreTokenizerWrapper::Whitespace(_) => { - Bound::new(py, init.add_subclass(PyWhitespace))?.into_super() - } - PreTokenizerWrapper::WhitespaceSplit(_) => { - Bound::new(py, init.add_subclass(PyWhitespaceSplit))?.into_super() - } - PreTokenizerWrapper::Sequence(_) => { - Bound::new(py, init.add_subclass(PySequence))?.into_super() - } - PreTokenizerWrapper::Split(_) => Bound::new(py, init.add_subclass(PySplit))?.into_super(), - PreTokenizerWrapper::Punctuation(_) => { - Bound::new(py, init.add_subclass(PyPunctuation))?.into_super() - } - PreTokenizerWrapper::Digits(_) => Bound::new(py, init.add_subclass(PyDigits))?.into_super(), - PreTokenizerWrapper::UnicodeScripts(_) => { - Bound::new(py, init.add_subclass(PyUnicodeScripts))?.into_super() - } - PreTokenizerWrapper::FixedLength(_) => { - Bound::new(py, init.add_subclass(PyFixedLength))?.into_super() - } - // Loadable from tokenizer.json but not constructible from Python (and - // rejected by the pipeline at compile time): exposed as the base class. - PreTokenizerWrapper::Metaspace(_) => Bound::new(py, init)?, - }; - Ok(obj.unbind()) -} - -macro_rules! unit_pre_tokenizer { - ($pyname:ident, $name:literal, $inner:expr, $doc:literal) => { - #[doc = $doc] - #[pyclass(frozen, extends = PyPreTokenizer, name = $name, module = "tokenizers_pipeline.pre_tokenizers")] - pub struct $pyname; - - #[pymethods] - impl $pyname { - #[new] - fn new() -> PyClassInitializer { - PyClassInitializer::from(PyPreTokenizer { inner: $inner.into() }).add_subclass($pyname) - } - } - }; -} - -unit_pre_tokenizer!( - PyWhitespace, - "Whitespace", - Whitespace, - "Splits into runs of letters/digits/underscore or runs of other symbols (the pattern `\\w+|[^\\w\\s]+`)." -); -unit_pre_tokenizer!( - PyWhitespaceSplit, - "WhitespaceSplit", - WhitespaceSplit, - "Splits on whitespace only." -); -unit_pre_tokenizer!( - PyBertPreTokenizer, - "BertPreTokenizer", - BertPreTokenizer, - "The BERT split: on whitespace, and each punctuation character becomes its own piece." -); -unit_pre_tokenizer!( - PyUnicodeScripts, - "UnicodeScripts", - UnicodeScripts, - "Splits where the script changes (Latin to Han, for example), so a piece never mixes alphabets." -); - -/// GPT-2 style byte-level splitting: cuts with the GPT-2 regex unless -/// `use_regex=False`. The pipeline does not support `add_prefix_space`, so it -/// is always off. -#[pyclass(frozen, extends = PyPreTokenizer, name = "ByteLevel", module = "tokenizers_pipeline.pre_tokenizers")] -pub struct PyByteLevel; - -#[pymethods] -impl PyByteLevel { - #[new] - #[pyo3(signature = (*, use_regex = true))] - fn new(use_regex: bool) -> PyClassInitializer { - PyClassInitializer::from(PyPreTokenizer { - inner: ByteLevel::new(false, true, use_regex).into(), - }) - .add_subclass(PyByteLevel) - } -} - -/// Splits on one fixed character, dropping it. -#[pyclass(frozen, extends = PyPreTokenizer, name = "CharDelimiterSplit", module = "tokenizers_pipeline.pre_tokenizers")] -pub struct PyCharDelimiterSplit; - -#[pymethods] -impl PyCharDelimiterSplit { - #[new] - fn new(delimiter: char) -> PyClassInitializer { - PyClassInitializer::from(PyPreTokenizer { - inner: CharDelimiterSplit::new(delimiter).into(), - }) - .add_subclass(PyCharDelimiterSplit) - } -} - -/// Separates digits from everything else. With `individual_digits=True`, -/// every digit becomes its own piece. -#[pyclass(frozen, extends = PyPreTokenizer, name = "Digits", module = "tokenizers_pipeline.pre_tokenizers")] -pub struct PyDigits; - -#[pymethods] -impl PyDigits { - #[new] - #[pyo3(signature = (*, individual_digits = false))] - fn new(individual_digits: bool) -> PyClassInitializer { - PyClassInitializer::from(PyPreTokenizer { - inner: Digits::new(individual_digits).into(), - }) - .add_subclass(PyDigits) - } -} - -/// Cuts the text into pieces of exactly `length` characters (the last one may -/// be shorter). -#[pyclass(frozen, extends = PyPreTokenizer, name = "FixedLength", module = "tokenizers_pipeline.pre_tokenizers")] -pub struct PyFixedLength; - -#[pymethods] -impl PyFixedLength { - #[new] - #[pyo3(signature = (*, length = 5))] - fn new(length: usize) -> PyClassInitializer { - PyClassInitializer::from(PyPreTokenizer { - inner: FixedLength::new(length).into(), - }) - .add_subclass(PyFixedLength) - } -} - -/// Splits on punctuation. `behavior` says what happens to the punctuation -/// itself — see `Split` for the options. -#[pyclass(frozen, extends = PyPreTokenizer, name = "Punctuation", module = "tokenizers_pipeline.pre_tokenizers")] -pub struct PyPunctuation; - -#[pymethods] -impl PyPunctuation { - #[new] - #[pyo3(signature = (behavior = String::from("isolated")))] - fn new(behavior: String) -> PyResult> { - Ok(PyClassInitializer::from(PyPreTokenizer { - inner: Punctuation::new(parse_behavior(&behavior)?).into(), - }) - .add_subclass(PyPunctuation)) - } -} - -/// Splits on a pattern: a literal string, or a regular expression with -/// `regex=True`. `behavior` says what to do with each match — "removed" drops -/// it, "isolated" keeps it as its own piece, "merged_with_previous" / -/// "merged_with_next" glue it to a neighbor, "contiguous" merges runs of -/// matches. `invert=True` keeps the matches and splits everything else. -#[pyclass(frozen, extends = PyPreTokenizer, name = "Split", module = "tokenizers_pipeline.pre_tokenizers")] -pub struct PySplit; - -#[pymethods] -impl PySplit { - #[new] - #[pyo3(signature = (pattern, behavior = String::from("isolated"), *, invert = false, regex = false))] - fn new( - pattern: &str, - behavior: String, - invert: bool, - regex: bool, - ) -> PyResult> { - let pattern = if regex { - SplitPattern::Regex(pattern.to_owned()) - } else { - SplitPattern::String(pattern.to_owned()) - }; - let split = Split::new(pattern, parse_behavior(&behavior)?, invert).map_err(to_pyerr)?; - Ok(PyClassInitializer::from(PyPreTokenizer { - inner: split.into(), - }) - .add_subclass(PySplit)) - } -} - -/// Runs several pre-tokenizers in order, each one further splitting the -/// pieces left by the previous. -#[pyclass(frozen, extends = PyPreTokenizer, name = "Sequence", module = "tokenizers_pipeline.pre_tokenizers")] -pub struct PySequence; - -#[pymethods] -impl PySequence { - #[new] - fn new(pre_tokenizers: Vec>) -> PyClassInitializer { - let inner: Vec = - pre_tokenizers.iter().map(|p| p.inner.clone()).collect(); - PyClassInitializer::from(PyPreTokenizer { - inner: Sequence::new(inner).into(), - }) - .add_subclass(PySequence) - } -} - -/// How text is cut into pieces before the model runs. -#[pymodule(gil_used = false)] -pub mod pre_tokenizers { - #[pymodule_export] - pub use super::{ - PyBertPreTokenizer, PyByteLevel, PyCharDelimiterSplit, PyDigits, PyFixedLength, - PyPreTokenizer, PyPunctuation, PySequence, PySplit, PyUnicodeScripts, PyWhitespace, - PyWhitespaceSplit, - }; -} diff --git a/bindings/python-pipeline/src/tokenizer.rs b/bindings/python-pipeline/src/tokenizer.rs deleted file mode 100644 index 46371d2de..000000000 --- a/bindings/python-pipeline/src/tokenizer.rs +++ /dev/null @@ -1,542 +0,0 @@ -use std::collections::HashMap; -use std::path::PathBuf; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex}; - -use numpy::{IntoPyArray, PyArray1}; -use pyo3::exceptions::{PyNotImplementedError, PyRuntimeError, PyStopIteration, PyTypeError}; -use pyo3::marker::Ungil; -use pyo3::prelude::*; -use pyo3::pybacked::PyBackedStr; -use pyo3::types::{PyBytes, PyList}; -use rayon::prelude::*; -use tk_encode::Tokenizer as SpecTokenizer; -use tk_encode::pipeline::{ - Model as _, PipelineModelScratch, PipelineToken, PipelineTokenizer, Span, -}; -use tk_encode::tokenizer::PostProcessor as _; -use tk_encode::utils::parallelism::get_parallelism; -use tk_train::{TokenizerTrainExt, Trainable}; - -use crate::added_token::{TokenInput, parse_tokens}; -use crate::detached_lock::{Detached, DetachedRwLock}; -use crate::error::{TokenizersError, to_pyerr}; -use crate::models::{PyModel, wrap_model}; -use crate::normalizers::{PyNormalizer, wrap_normalizer}; -use crate::pre_tokenizers::{PyPreTokenizer, wrap_pre_tokenizer}; -use crate::trainers::PyTrainer; - -/// Set when the bindings actually run a rayon-parallel section, so the -/// pthread_atfork handler only disables parallelism in children of processes -/// that really used it (mirrors the v1 bindings' semantics). -pub static USED_PARALLELISM: AtomicBool = AtomicBool::new(false); - -/// The compiled encode path plus the facts about the spec the encode calls -/// need without re-locking it. -#[derive(Clone)] -struct Compiled { - pipe: Arc, - /// Whether the spec's post-processor would add special tokens. Post-processing - /// is not wired into the pipeline yet, so encode(add_special_tokens=True) - /// must fail loudly instead of silently dropping them. - post_adds_special_tokens: bool, -} - -struct Inner { - /// Source of truth: the mutable, serializable tokenizer definition. - spec: SpecTokenizer, - /// Memoized compilation of `spec`; invalidated by every mutation. - compiled: Option, -} - -fn poisoned(_: std::sync::PoisonError) -> PyErr { - PyRuntimeError::new_err("tokenizer lock poisoned") -} - -/// A tokenizer: a model plus its optional normalizer and pre-tokenizer. -/// -/// Create one from a model (`Tokenizer(models.BPE())`), a file -/// (`Tokenizer.from_file`), or the Hub (`Tokenizer.from_pretrained`). -/// Changes — assigning components, training, adding tokens — apply to the -/// serializable definition; encoding runs a compiled pipeline that is rebuilt -/// automatically after any change. A definition the pipeline cannot run -/// raises `TokenizersError` at that point, with the reason. -// The lock/GIL ordering rule (never block on the lock while attached) is -// enforced by DetachedRwLock: guards are only reachable inside its -// detach-first `with` closure. See detached_lock.rs for the rationale and -// the residual hole. -#[pyclass(frozen, name = "Tokenizer", module = "tokenizers_pipeline")] -pub struct PyTokenizer { - inner: DetachedRwLock, -} - -impl PyTokenizer { - fn from_spec(spec: SpecTokenizer) -> Self { - Self { - inner: DetachedRwLock::new(Inner { - spec, - compiled: None, - }), - } - } - - fn read_spec( - &self, - py: Python<'_>, - f: impl FnOnce(&SpecTokenizer) -> T + Ungil + Send, - ) -> PyResult { - self.inner.with(py, |lock| { - let guard = lock.read().map_err(poisoned)?; - Ok(f(&guard.spec)) - }) - } - - /// Write access to the spec; invalidates the compiled pipeline. - fn mutate_spec( - &self, - py: Python<'_>, - f: impl FnOnce(&mut SpecTokenizer) -> PyResult + Ungil + Send, - ) -> PyResult { - self.inner.with(py, |lock| { - let mut guard = lock.write().map_err(poisoned)?; - let result = f(&mut guard.spec)?; - guard.compiled = None; - Ok(result) - }) - } -} - -/// Get the compiled pipeline, building it from the spec on first use after a -/// mutation. The `Detached` parameter is the proof this runs off the GIL. -fn get_or_compile(lock: &Detached<'_, Inner>) -> PyResult { - { - let guard = lock.read().map_err(poisoned)?; - if let Some(compiled) = &guard.compiled { - return Ok(compiled.clone()); - } - } - let mut guard = lock.write().map_err(poisoned)?; - if guard.compiled.is_none() { - let pipe = PipelineTokenizer::try_from(&guard.spec).map_err(|e| { - TokenizersError::new_err(format!( - "this tokenizer cannot be compiled to an encode pipeline: {e}" - )) - })?; - let post_adds_special_tokens = guard - .spec - .get_post_processor() - .is_some_and(|p| p.added_tokens(false) > 0); - guard.compiled = Some(Compiled { - pipe: Arc::new(pipe), - post_adds_special_tokens, - }); - } - Ok(guard.compiled.clone().expect("just set")) -} - -fn check_special_tokens_flag(compiled: &Compiled, add_special_tokens: bool) -> PyResult<()> { - if add_special_tokens && compiled.post_adds_special_tokens { - return Err(PyNotImplementedError::new_err( - "this tokenizer's post-processor adds special tokens, but post-processing is not \ - implemented in the encode pipeline yet; pass add_special_tokens=False to encode \ - without them", - )); - } - Ok(()) -} - -fn encode_one( - pipe: &PipelineTokenizer, - text: &str, - pre_tokens: &mut Vec, - scratch: &mut PipelineModelScratch, -) -> PyResult> { - let mut output: Vec = Vec::new(); - pipe.encode_generic::<{ PipelineTokenizer::STAGE_MODEL }>( - text, - pre_tokens, - scratch, - &mut output, - ) - .map_err(to_pyerr)?; - Ok(output.iter().map(|t| t.id).collect()) -} - -#[pymethods] -impl PyTokenizer { - /// Create an untrained tokenizer from a model. - #[new] - fn new(model: PyRef<'_, PyModel>) -> Self { - Self::from_spec(SpecTokenizer::new(model.inner.clone())) - } - - /// Load a tokenizer from a `tokenizer.json` file. - #[staticmethod] - #[pyo3(signature = (path) -> "Tokenizer")] - fn from_file(py: Python<'_>, path: PathBuf) -> PyResult { - let spec = py - .detach(|| SpecTokenizer::from_file(path)) - .map_err(to_pyerr)?; - Ok(Self::from_spec(spec)) - } - - /// Load a tokenizer from the bytes of a `tokenizer.json` file. - #[staticmethod] - #[pyo3(signature = (buffer) -> "Tokenizer")] - fn from_buffer(py: Python<'_>, buffer: Vec) -> PyResult { - let spec = py - .detach(|| SpecTokenizer::from_bytes(&buffer)) - .map_err(to_pyerr)?; - Ok(Self::from_spec(spec)) - } - - /// Download `tokenizer.json` from a model on the Hugging Face Hub (requires - /// the `huggingface_hub` package) and load it. - #[staticmethod] - #[pyo3(signature = (identifier, *, revision = String::from("main"), token = None) -> "Tokenizer")] - fn from_pretrained( - py: Python<'_>, - identifier: &str, - revision: String, - token: Option, - ) -> PyResult { - let hub = py.import("huggingface_hub")?; - let kwargs = pyo3::types::PyDict::new(py); - kwargs.set_item("repo_id", identifier)?; - kwargs.set_item("filename", "tokenizer.json")?; - kwargs.set_item("revision", revision)?; - kwargs.set_item("token", token)?; - let path: PathBuf = hub - .getattr("hf_hub_download")? - .call((), Some(&kwargs))? - .extract()?; - Self::from_file(py, path) - } - - /// Serialize the tokenizer definition as a `tokenizer.json` string. - #[pyo3(signature = (*, pretty = false))] - fn to_str(&self, py: Python<'_>, pretty: bool) -> PyResult { - self.read_spec(py, move |spec| spec.to_string(pretty).map_err(to_pyerr))? - } - - /// Save the tokenizer definition to a `tokenizer.json` file. - #[pyo3(signature = (path, *, pretty = true))] - fn save(&self, py: Python<'_>, path: PathBuf, pretty: bool) -> PyResult<()> { - self.read_spec(py, move |spec| spec.save(path, pretty).map_err(to_pyerr))? - } - - /// Encode `text` into token ids. - /// - /// Runs entirely outside the interpreter lock and returns a `numpy.uint32` - /// array backed by the Rust output buffer (no copy). - #[pyo3(signature = (text, *, add_special_tokens = true) -> "npt.NDArray[np.uint32]")] - fn encode<'py>( - &self, - py: Python<'py>, - text: &str, - add_special_tokens: bool, - ) -> PyResult>> { - let ids = self.inner.with(py, |lock| -> PyResult> { - let compiled = get_or_compile(&lock)?; - check_special_tokens_flag(&compiled, add_special_tokens)?; - let mut pre_tokens = Vec::new(); - let mut scratch = compiled.pipe.get_model().init_scratch(); - encode_one(&compiled.pipe, text, &mut pre_tokens, &mut scratch) - })?; - Ok(ids.into_pyarray(py)) - } - - /// Encode a batch of texts, in parallel across Rust threads (respects - /// `TOKENIZERS_PARALLELISM`), without holding the interpreter lock. - /// Input strings are borrowed, not copied; each output is a `numpy.uint32` - /// array backed by its Rust buffer. - #[pyo3(signature = (texts, *, add_special_tokens = true) -> "list[npt.NDArray[np.uint32]]")] - fn encode_batch<'py>( - &self, - py: Python<'py>, - texts: Vec, - add_special_tokens: bool, - ) -> PyResult> { - let batches = self.inner.with(py, |lock| -> PyResult>> { - let compiled = get_or_compile(&lock)?; - check_special_tokens_flag(&compiled, add_special_tokens)?; - if get_parallelism() && texts.len() > 1 { - USED_PARALLELISM.store(true, Ordering::SeqCst); - texts - .par_iter() - .map_init( - || (Vec::new(), compiled.pipe.get_model().init_scratch()), - |(pre_tokens, scratch), text| { - encode_one(&compiled.pipe, text, pre_tokens, scratch) - }, - ) - .collect() - } else { - let mut pre_tokens = Vec::new(); - let mut scratch = compiled.pipe.get_model().init_scratch(); - texts - .iter() - .map(|text| encode_one(&compiled.pipe, text, &mut pre_tokens, &mut scratch)) - .collect() - } - })?; - let list = PyList::empty(py); - for ids in batches { - list.append(ids.into_pyarray(py))?; - } - Ok(list) - } - - /// Not implemented yet: decoding is not part of the encode pipeline. - #[pyo3(signature = (ids, *, skip_special_tokens = true))] - #[allow(unused_variables)] - fn decode(&self, ids: Vec, skip_special_tokens: bool) -> PyResult { - Err(PyNotImplementedError::new_err( - "decode is not implemented in the encode pipeline yet", - )) - } - - /// Train the model's vocabulary on text files (one sequence per line). - /// Without a `trainer`, the model's default trainer is used. - #[pyo3(signature = (files, *, trainer = None))] - fn train( - &self, - py: Python<'_>, - files: Vec, - trainer: Option>, - ) -> PyResult<()> { - let explicit = trainer.map(|t| t.inner.clone()); - self.inner.with(py, |lock| { - let mut guard = lock.write().map_err(poisoned)?; - let mut trainer = explicit.unwrap_or_else(|| guard.spec.get_model().get_trainer()); - guard - .spec - .train_from_files(&mut trainer, files) - .map_err(to_pyerr)?; - guard.compiled = None; - Ok(()) - }) - } - - /// Train the model's vocabulary from any iterator of `str`. Without a - /// `trainer`, the model's default trainer is used. - /// - /// The interpreter lock is only re-acquired to refill an internal buffer - /// (256 sequences at a time); the training itself runs multi-threaded in - /// Rust with the lock released. - #[pyo3(signature = (iterator, *, trainer = None))] - fn train_from_iterator( - &self, - py: Python<'_>, - iterator: &Bound<'_, PyAny>, - trainer: Option>, - ) -> PyResult<()> { - let explicit = trainer.map(|t| t.inner.clone()); - let sequences = BufferedPyIterator::new(iterator)?; - let error = sequences.error.clone(); - self.inner.with(py, |lock| { - USED_PARALLELISM.store(true, Ordering::SeqCst); - let mut guard = lock.write().map_err(poisoned)?; - let mut trainer = explicit.unwrap_or_else(|| guard.spec.get_model().get_trainer()); - guard - .spec - .train(&mut trainer, sequences) - .map_err(to_pyerr)?; - guard.compiled = None; - Ok::<_, PyErr>(()) - })?; - if let Some(err) = error.lock().expect("error slot poisoned").take() { - return Err(err); - } - Ok(()) - } - - /// Add tokens to the vocabulary and match them in the input text from now - /// on. Plain strings match with default options; pass `AddedToken` to - /// control matching. Returns how many were actually new. - fn add_tokens(&self, py: Python<'_>, tokens: Vec) -> PyResult { - let tokens = parse_tokens(tokens, false); - self.mutate_spec(py, move |spec| spec.add_tokens(tokens).map_err(to_pyerr)) - } - - /// Add special tokens ("", "[CLS]", …) to the vocabulary. Same as - /// `add_tokens`, but every token is marked `special`. Returns how many - /// were actually new. - fn add_special_tokens(&self, py: Python<'_>, tokens: Vec) -> PyResult { - let tokens = parse_tokens(tokens, true); - self.mutate_spec(py, move |spec| { - spec.add_special_tokens(tokens).map_err(to_pyerr) - }) - } - - /// The id of `token`, or None if it is not in the vocabulary. - fn token_to_id(&self, py: Python<'_>, token: &str) -> PyResult> { - self.read_spec(py, |spec| spec.token_to_id(token)) - } - - /// The token behind `id`, or None if the id is out of range. - fn id_to_token(&self, py: Python<'_>, id: u32) -> PyResult> { - self.read_spec(py, move |spec| spec.id_to_token(id)) - } - - /// The whole vocabulary as a dict. This copies every entry; prefer - /// `token_to_id` for lookups. - #[pyo3(signature = (*, with_added_tokens = true))] - fn get_vocab(&self, py: Python<'_>, with_added_tokens: bool) -> PyResult> { - self.read_spec(py, move |spec| spec.get_vocab(with_added_tokens)) - } - - /// Number of entries in the vocabulary. `with_added_tokens=False` counts - /// only what the model was trained with. - #[pyo3(signature = (*, with_added_tokens = true))] - fn get_vocab_size(&self, py: Python<'_>, with_added_tokens: bool) -> PyResult { - self.read_spec(py, move |spec| spec.get_vocab_size(with_added_tokens)) - } - - /// The model in use by this tokenizer (a copy: reassign to change it). - #[getter] - fn model(&self, py: Python<'_>) -> PyResult> { - let model = self.read_spec(py, |spec| spec.get_model().clone())?; - wrap_model(py, model) - } - - #[setter] - fn set_model(&self, py: Python<'_>, model: PyRef<'_, PyModel>) -> PyResult<()> { - let model = model.inner.clone(); - self.mutate_spec(py, move |spec| { - spec.with_model(model); - Ok(()) - }) - } - - /// The optional normalizer in use by this tokenizer (a copy: reassign to - /// change it). - #[getter] - fn normalizer(&self, py: Python<'_>) -> PyResult>> { - let normalizer = self.read_spec(py, |spec| spec.get_normalizer().cloned())?; - normalizer.map(|n| wrap_normalizer(py, n)).transpose() - } - - #[setter] - fn set_normalizer( - &self, - py: Python<'_>, - normalizer: Option>, - ) -> PyResult<()> { - let normalizer = normalizer.map(|n| n.inner.clone()); - self.mutate_spec(py, move |spec| { - spec.with_normalizer(normalizer).map_err(to_pyerr)?; - Ok(()) - }) - } - - /// The optional pre-tokenizer in use by this tokenizer (a copy: reassign - /// to change it). - #[getter] - fn pre_tokenizer(&self, py: Python<'_>) -> PyResult>> { - let pre_tokenizer = self.read_spec(py, |spec| spec.get_pre_tokenizer().cloned())?; - pre_tokenizer.map(|p| wrap_pre_tokenizer(py, p)).transpose() - } - - #[setter] - fn set_pre_tokenizer( - &self, - py: Python<'_>, - pre_tokenizer: Option>, - ) -> PyResult<()> { - let pre_tokenizer = pre_tokenizer.map(|p| p.inner.clone()); - self.mutate_spec(py, move |spec| { - spec.with_pre_tokenizer(pre_tokenizer); - Ok(()) - }) - } - - fn __repr__(&self, py: Python<'_>) -> PyResult { - self.read_spec(py, |spec| { - format!( - "Tokenizer(model={}, vocab_size={})", - match spec.get_model() { - tk_encode::ModelWrapper::BPE(_) => "BPE", - tk_encode::ModelWrapper::WordPiece(_) => "WordPiece", - tk_encode::ModelWrapper::WordLevel(_) => "WordLevel", - tk_encode::ModelWrapper::Unigram(_) => "Unigram", - }, - spec.get_vocab_size(true) - ) - }) - } - - fn __reduce__<'py>( - &self, - py: Python<'py>, - ) -> PyResult<(Bound<'py, PyAny>, (Bound<'py, PyBytes>,))> { - let data = self.to_str(py, false)?; - let from_buffer = py.get_type::().getattr("from_buffer")?; - Ok((from_buffer, (PyBytes::new(py, data.as_bytes()),))) - } -} - -/// Pulls a Python iterator of `str` from Rust threads: re-attaches to the -/// interpreter only to refill an internal buffer, `CHUNK` items at a time. -/// A conversion error stops the stream and is stashed in `error` for the -/// caller to surface once training finishes. -struct BufferedPyIterator { - iterator: Py, - buffer: std::collections::VecDeque, - finished: bool, - error: Arc>>, -} - -impl BufferedPyIterator { - const CHUNK: usize = 256; - - fn new(iterable: &Bound<'_, PyAny>) -> PyResult { - Ok(Self { - iterator: iterable.try_iter()?.unbind().into(), - buffer: std::collections::VecDeque::with_capacity(Self::CHUNK), - finished: false, - error: Arc::new(Mutex::new(None)), - }) - } - - // The vetted lock-then-GIL direction: the caller (train) holds the write - // lock and re-attaches here. Safe because no attached thread can be - // blocking on the lock — DetachedRwLock makes that unrepresentable. - #[allow(clippy::disallowed_methods)] - fn refill(&mut self) { - let result = Python::attach(|py| -> PyResult { - let iterator = self.iterator.bind(py); - for _ in 0..Self::CHUNK { - match iterator.call_method0("__next__") { - Ok(item) => { - let sequence = item.extract::().map_err(|_| { - PyTypeError::new_err("train_from_iterator expects an iterator of str") - })?; - self.buffer.push_back(sequence); - } - Err(e) if e.is_instance_of::(py) => return Ok(true), - Err(e) => return Err(e), - } - } - Ok(false) - }); - match result { - Ok(done) => self.finished = done, - Err(e) => { - *self.error.lock().expect("error slot poisoned") = Some(e); - self.finished = true; - } - } - } -} - -impl Iterator for BufferedPyIterator { - type Item = String; - - fn next(&mut self) -> Option { - if self.buffer.is_empty() && !self.finished { - self.refill(); - } - self.buffer.pop_front() - } -} diff --git a/bindings/python-pipeline/src/trainers.rs b/bindings/python-pipeline/src/trainers.rs deleted file mode 100644 index 1edfd094b..000000000 --- a/bindings/python-pipeline/src/trainers.rs +++ /dev/null @@ -1,195 +0,0 @@ -use pyo3::prelude::*; -use tk_train::trainers::{ - BpeTrainer, TrainerWrapper, UnigramTrainer, WordLevelTrainer, WordPieceTrainer, -}; - -use crate::added_token::{TokenInput, parse_tokens}; -use crate::error::to_pyerr; - -/// Base class for all trainers. -/// -/// A trainer is the recipe for learning a model's vocabulary from text; pass -/// one to `Tokenizer.train` or `train_from_iterator`. Trainers are plain -/// configuration values — training copies them and writes nothing back. -#[pyclass( - frozen, - subclass, - name = "Trainer", - module = "tokenizers_pipeline.trainers" -)] -pub struct PyTrainer { - pub inner: TrainerWrapper, -} - -#[pymethods] -impl PyTrainer { - fn __repr__(&self) -> String { - crate::component_repr(&self.inner) - } -} - -/// Learns a BPE vocabulary: keeps merging the most frequent pair until -/// `vocab_size` is reached, ignoring pairs seen fewer than `min_frequency` -/// times. `special_tokens` get the first ids. `limit_alphabet` caps how many -/// distinct characters are kept; `initial_alphabet` forces characters in even -/// if the data never shows them; `max_token_length` caps merged token length. -#[pyclass(frozen, extends = PyTrainer, name = "BpeTrainer", module = "tokenizers_pipeline.trainers")] -pub struct PyBpeTrainer; - -#[pymethods] -impl PyBpeTrainer { - #[new] - #[pyo3(signature = (*, vocab_size = 30000, min_frequency = 0, special_tokens = vec![], limit_alphabet = None, initial_alphabet = vec![], continuing_subword_prefix = None, end_of_word_suffix = None, max_token_length = None, show_progress = true))] - #[allow(clippy::too_many_arguments)] - fn new( - vocab_size: usize, - min_frequency: u64, - special_tokens: Vec, - limit_alphabet: Option, - initial_alphabet: Vec, - continuing_subword_prefix: Option, - end_of_word_suffix: Option, - max_token_length: Option, - show_progress: bool, - ) -> PyResult> { - let mut builder = BpeTrainer::builder() - .vocab_size(vocab_size) - .min_frequency(min_frequency) - .special_tokens(parse_tokens(special_tokens, true)) - .initial_alphabet(initial_alphabet.into_iter().collect()) - .show_progress(show_progress); - if let Some(limit) = limit_alphabet { - builder = builder.limit_alphabet(limit); - } - if let Some(prefix) = continuing_subword_prefix { - builder = builder.continuing_subword_prefix(prefix); - } - if let Some(suffix) = end_of_word_suffix { - builder = builder.end_of_word_suffix(suffix); - } - if let Some(max) = max_token_length { - builder = builder.max_token_length(Some(max)); - } - Ok(PyClassInitializer::from(PyTrainer { - inner: builder.build().into(), - }) - .add_subclass(PyBpeTrainer)) - } -} - -/// Learns a WordPiece vocabulary. Same knobs as `BpeTrainer`, plus the -/// continuation prefix ("##" by default). -#[pyclass(frozen, extends = PyTrainer, name = "WordPieceTrainer", module = "tokenizers_pipeline.trainers")] -pub struct PyWordPieceTrainer; - -#[pymethods] -impl PyWordPieceTrainer { - #[new] - #[pyo3(signature = (*, vocab_size = 30000, min_frequency = 0, special_tokens = vec![], limit_alphabet = None, initial_alphabet = vec![], continuing_subword_prefix = String::from("##"), end_of_word_suffix = None, show_progress = true))] - #[allow(clippy::too_many_arguments)] - fn new( - vocab_size: usize, - min_frequency: u64, - special_tokens: Vec, - limit_alphabet: Option, - initial_alphabet: Vec, - continuing_subword_prefix: String, - end_of_word_suffix: Option, - show_progress: bool, - ) -> PyResult> { - let mut builder = WordPieceTrainer::builder() - .vocab_size(vocab_size) - .min_frequency(min_frequency) - .special_tokens(parse_tokens(special_tokens, true)) - .initial_alphabet(initial_alphabet.into_iter().collect()) - .continuing_subword_prefix(continuing_subword_prefix) - .show_progress(show_progress); - if let Some(limit) = limit_alphabet { - builder = builder.limit_alphabet(limit); - } - if let Some(suffix) = end_of_word_suffix { - builder = builder.end_of_word_suffix(suffix); - } - Ok(PyClassInitializer::from(PyTrainer { - inner: builder.build().into(), - }) - .add_subclass(PyWordPieceTrainer)) - } -} - -/// Learns a Unigram vocabulary: starts from a large candidate set and prunes -/// it by `shrinking_factor` each round until `vocab_size` pieces remain. -/// `unk_token` names the fallback piece for unknown characters. -#[pyclass(frozen, extends = PyTrainer, name = "UnigramTrainer", module = "tokenizers_pipeline.trainers")] -pub struct PyUnigramTrainer; - -#[pymethods] -impl PyUnigramTrainer { - #[new] - #[pyo3(signature = (*, vocab_size = 8000, special_tokens = vec![], initial_alphabet = vec![], unk_token = None, shrinking_factor = 0.75, max_piece_length = 16, n_sub_iterations = 2, show_progress = true))] - #[allow(clippy::too_many_arguments)] - fn new( - vocab_size: u32, - special_tokens: Vec, - initial_alphabet: Vec, - unk_token: Option, - shrinking_factor: f64, - max_piece_length: usize, - n_sub_iterations: u32, - show_progress: bool, - ) -> PyResult> { - let trainer = UnigramTrainer::builder() - .vocab_size(vocab_size) - .special_tokens(parse_tokens(special_tokens, true)) - .initial_alphabet(initial_alphabet.into_iter().collect()) - .unk_token(unk_token) - .shrinking_factor(shrinking_factor) - .max_piece_length(max_piece_length) - .n_sub_iterations(n_sub_iterations) - .show_progress(show_progress) - .build() - .map_err(|e| to_pyerr(e.to_string().into()))?; - Ok(PyClassInitializer::from(PyTrainer { - inner: trainer.into(), - }) - .add_subclass(PyUnigramTrainer)) - } -} - -/// Learns a WordLevel vocabulary: the `vocab_size` most frequent words, -/// keeping only those seen at least `min_frequency` times. -#[pyclass(frozen, extends = PyTrainer, name = "WordLevelTrainer", module = "tokenizers_pipeline.trainers")] -pub struct PyWordLevelTrainer; - -#[pymethods] -impl PyWordLevelTrainer { - #[new] - #[pyo3(signature = (*, vocab_size = 30000, min_frequency = 0, special_tokens = vec![], show_progress = true))] - fn new( - vocab_size: usize, - min_frequency: u64, - special_tokens: Vec, - show_progress: bool, - ) -> PyResult> { - let trainer = WordLevelTrainer::builder() - .vocab_size(vocab_size) - .min_frequency(min_frequency) - .special_tokens(parse_tokens(special_tokens, true)) - .show_progress(show_progress) - .build() - .map_err(|e| to_pyerr(e.to_string().into()))?; - Ok(PyClassInitializer::from(PyTrainer { - inner: trainer.into(), - }) - .add_subclass(PyWordLevelTrainer)) - } -} - -/// Recipes for learning a vocabulary from text. -#[pymodule(gil_used = false)] -pub mod trainers { - #[pymodule_export] - pub use super::{ - PyBpeTrainer, PyTrainer, PyUnigramTrainer, PyWordLevelTrainer, PyWordPieceTrainer, - }; -} diff --git a/bindings/python-pipeline/tools/stub-gen/Cargo.lock b/bindings/python-pipeline/tools/stub-gen/Cargo.lock deleted file mode 100644 index c00f814df..000000000 --- a/bindings/python-pipeline/tools/stub-gen/Cargo.lock +++ /dev/null @@ -1,178 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "anyhow" -version = "1.0.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" - -[[package]] -name = "goblin" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17582616a7718cca54cec18e534a76c7c4aec11a8b9a85695712f262fd15a4c8" -dependencies = [ - "log", - "plain", - "scroll", -] - -[[package]] -name = "itoa" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" - -[[package]] -name = "log" -version = "0.4.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" - -[[package]] -name = "memchr" -version = "2.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" - -[[package]] -name = "plain" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" - -[[package]] -name = "proc-macro2" -version = "1.0.107" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "pyo3-introspection" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7775fcc875acdce3872dcb91a4b7bd155ffba6e0ea8be88b8caab7d0b34539a6" -dependencies = [ - "anyhow", - "goblin", - "serde", - "serde_json", -] - -[[package]] -name = "quote" -version = "1.0.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "scroll" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1257cd4248b4132760d6524d6dda4e053bc648c9070b960929bf50cfb1e7add" -dependencies = [ - "scroll_derive", -] - -[[package]] -name = "scroll_derive" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed76efe62313ab6610570951494bdaa81568026e0318eaa55f167de70eeea67d" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "serde" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" -dependencies = [ - "serde_core", - "serde_derive", -] - -[[package]] -name = "serde_core" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - -[[package]] -name = "serde_json" -version = "1.0.151" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" -dependencies = [ - "itoa", - "memchr", - "serde", - "serde_core", - "zmij", -] - -[[package]] -name = "stub-gen" -version = "0.1.0" -dependencies = [ - "pyo3-introspection", -] - -[[package]] -name = "syn" -version = "2.0.119" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "3.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "unicode-ident" -version = "1.0.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" - -[[package]] -name = "zmij" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/bindings/python-pipeline/tools/stub-gen/Cargo.toml b/bindings/python-pipeline/tools/stub-gen/Cargo.toml deleted file mode 100644 index afb83b1bc..000000000 --- a/bindings/python-pipeline/tools/stub-gen/Cargo.toml +++ /dev/null @@ -1,10 +0,0 @@ -[package] -name = "stub-gen" -version = "0.1.0" -edition = "2024" -description = "Generates py_src .pyi stubs by introspecting the built cdylib" - -[dependencies] -pyo3-introspection = "0.29" - -[workspace] diff --git a/bindings/python-pipeline/tools/stub-gen/src/main.rs b/bindings/python-pipeline/tools/stub-gen/src/main.rs deleted file mode 100644 index 8acfd4c44..000000000 --- a/bindings/python-pipeline/tools/stub-gen/src/main.rs +++ /dev/null @@ -1,122 +0,0 @@ -//! Generates the `.pyi` stubs under `py_src/tokenizers_pipeline/` from the -//! introspection metadata pyo3 embeds in the built extension (the -//! `experimental-inspect` feature). Run after `maturin develop --release`: -//! -//! ```sh -//! cargo run --manifest-path tools/stub-gen/Cargo.toml -//! ``` -//! -//! Return types beyond introspection's reach come from the -//! `#[pyo3(signature = (...) -> "Type")]` annotations in the sources; numpy -//! imports for those annotations are injected here. - -use std::path::{Path, PathBuf}; - -const MODULE: &str = "tokenizers_pipeline"; -/// The `#[pymodule]` name inside the cdylib. -const NATIVE_MODULE: &str = "_native"; - -fn main() -> Result<(), Box> { - let crate_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .ancestors() - .nth(2) - .expect("tools/stub-gen sits two levels under the crate root") - .to_path_buf(); - let cdylib = crate_dir.join(format!( - "target/release/{}{NATIVE_MODULE}.{}", - std::env::consts::DLL_PREFIX, - std::env::consts::DLL_EXTENSION - )); - let out_dir = crate_dir.join("py_src").join(MODULE); - - if !cdylib.is_file() { - return Err(format!( - "no cdylib at {} — run `maturin develop --release` first", - cdylib.display() - ) - .into()); - } - - let module = pyo3_introspection::introspect_cdylib(&cdylib, NATIVE_MODULE)?; - assert_has_docstrings(&module); - - for (rel_path, contents) in pyo3_introspection::module_stub_files(&module) { - let out_path = out_dir.join(place(&rel_path)); - if let Some(parent) = out_path.parent() { - std::fs::create_dir_all(parent)?; - } - let mut contents = postprocess(&contents); - if rel_path == Path::new("__init__.pyi") { - // `create_exception!` types carry no introspection metadata. - contents.push_str("\nclass TokenizersError(Exception): ...\n"); - } - std::fs::write(&out_path, &contents)?; - println!("generated {}", out_path.display()); - } - Ok(()) -} - -/// Map the introspected layout onto the package layout: the root module stub -/// becomes `__init__.pyi`, and each submodule stub lands inside its runtime -/// shim package (`models.pyi` -> `models/__init__.pyi`) so it shadows the -/// `.py` re-exports for type checkers. -fn place(rel_path: &Path) -> PathBuf { - let name = rel_path - .file_name() - .and_then(|n| n.to_str()) - .expect("stub paths are utf-8 files"); - match name.strip_suffix(".pyi") { - Some("__init__") | None => rel_path.to_path_buf(), - Some(module) => rel_path.with_file_name(module).join("__init__.pyi"), - } -} - -fn postprocess(contents: &str) -> String { - // Cross-submodule references come out relative to the extension root; - // absolutize them to the package. - let mut contents = contents - .replace("from . import", &format!("from {MODULE} import")) - .replace("from .", &format!("from {MODULE}.")); - // Annotated numpy return types need their imports. - if contents.contains("npt.") || contents.contains("np.") { - contents = format!( - "import numpy as np\nimport numpy.typing as npt\n\n{contents}" - ); - } - contents -} - -/// Fail loudly if introspection came back without docstrings — that means the -/// cdylib was built without `experimental-inspect` (or the feature broke) and -/// the stubs would silently lose all documentation. -fn assert_has_docstrings(module: &pyo3_introspection::model::Module) { - fn count(module: &pyo3_introspection::model::Module) -> (usize, usize) { - let mut with_doc = 0; - let mut total = 0; - for f in &module.functions { - total += 1; - with_doc += f.docstring.is_some() as usize; - } - for c in &module.classes { - total += 1; - with_doc += c.docstring.is_some() as usize; - for m in &c.methods { - total += 1; - with_doc += m.docstring.is_some() as usize; - } - } - for sub in &module.modules { - let (w, t) = count(sub); - with_doc += w; - total += t; - } - (with_doc, total) - } - let (with_doc, total) = count(module); - println!("docstring coverage: {with_doc}/{total}"); - assert!( - with_doc > 0, - "introspection returned 0/{total} docstrings — was the cdylib built \ - with the `experimental-inspect` pyo3 feature?" - ); -} diff --git a/bindings/python/.cargo/config.toml b/bindings/python/.cargo/config.toml deleted file mode 100644 index dd4042827..000000000 --- a/bindings/python/.cargo/config.toml +++ /dev/null @@ -1,13 +0,0 @@ -[target.x86_64-apple-darwin] -rustflags = [ - "-C", "link-arg=-undefined", - "-C", "link-arg=dynamic_lookup", - "-C", "link-arg=-mmacosx-version-min=10.11", -] - -[target.aarch64-apple-darwin] -rustflags = [ - "-C", "link-arg=-undefined", - "-C", "link-arg=dynamic_lookup", - "-C", "link-arg=-mmacosx-version-min=10.11", -] diff --git a/bindings/python/.gitignore b/bindings/python/.gitignore index 1269488f7..9cbe9d5c8 100644 --- a/bindings/python/.gitignore +++ b/bindings/python/.gitignore @@ -1 +1,4 @@ -data +.venv/ +.release/ +python_bench.json +python_bench.md diff --git a/bindings/python/CHANGELOG.md b/bindings/python/CHANGELOG.md deleted file mode 100644 index 56c8af841..000000000 --- a/bindings/python/CHANGELOG.md +++ /dev/null @@ -1,512 +0,0 @@ -# Changelog -All notable changes to this project will be documented in this file. - -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), -and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - -## [0.13.2] - -- [#1096] Python 3.11 support - -## [0.13.1] - -- [#1072] Fixing Roberta type ids. - -## [0.13.0] - -- [#956] PyO3 version upgrade -- [#1055] M1 automated builds -- [#1008] `Decoder` is now a composable trait, but without being backward incompatible -- [#1047, #1051, #1052] `Processor` is now a composable trait, but without being backward incompatible - -Both trait changes warrant a "major" number since, despite best efforts to not break backward - compatibility, the code is different enough that we cannot be exactly sure. - -## [0.12.1] - -- [#938] **Reverted breaking change**. https://github.com/huggingface/transformers/issues/16520 - -## [0.12.0] YANKED - -Bump minor version because of a breaking change. - -- [#938] [REVERTED IN 0.12.1] **Breaking change**. Decoder trait is modified to be composable. This is only breaking if you are using decoders on their own. tokenizers should be error free. -- [#939] Making the regex in `ByteLevel` pre_tokenizer optional (necessary for BigScience) - -- [#952] Fixed the vocabulary size of UnigramTrainer output (to respect added tokens) -- [#954] Fixed not being able to save vocabularies with holes in vocab (ConvBert). Yell warnings instead, but stop panicking. -- [#962] Fix tests for python 3.10 -- [#961] Added link for Ruby port of `tokenizers` - -## [0.11.6] - -- [#919] Fixing single_word AddedToken. (regression from 0.11.2) -- [#916] Deserializing faster `added_tokens` by loading them in batch. - -## [0.11.5] - -- [#895] Build `python 3.10` wheels. - -## [0.11.4] - -- [#884] Fixing bad deserialization following inclusion of a default for Punctuation - -## [0.11.3] - -- [#882] Fixing Punctuation deserialize without argument. -- [#868] Fixing missing direction in TruncationParams -- [#860] Adding TruncationSide to TruncationParams - -## [0.11.0] - -### Fixed - -- [#585] Conda version should now work on old CentOS -- [#844] Fixing interaction between `is_pretokenized` and `trim_offsets`. -- [#851] Doc links - -### Added -- [#657]: Add SplitDelimiterBehavior customization to Punctuation constructor -- [#845]: Documentation for `Decoders`. - -### Changed -- [#850]: Added a feature gate to enable disabling `http` features -- [#718]: Fix `WordLevel` tokenizer determinism during training -- [#762]: Add a way to specify the unknown token in `SentencePieceUnigramTokenizer` -- [#770]: Improved documentation for `UnigramTrainer` -- [#780]: Add `Tokenizer.from_pretrained` to load tokenizers from the Hugging Face Hub -- [#793]: Saving a pretty JSON file by default when saving a tokenizer - -## [0.10.3] - -### Fixed -- [#686]: Fix SPM conversion process for whitespace deduplication -- [#707]: Fix stripping strings containing Unicode characters - -### Added -- [#693]: Add a CTC Decoder for Wave2Vec models - -### Removed -- [#714]: Removed support for Python 3.5 - -## [0.10.2] - -### Fixed -- [#652]: Fix offsets for `Precompiled` corner case -- [#656]: Fix BPE `continuing_subword_prefix` -- [#674]: Fix `Metaspace` serialization problems - -## [0.10.1] - -### Fixed -- [#616]: Fix SentencePiece tokenizers conversion -- [#617]: Fix offsets produced by Precompiled Normalizer (used by tokenizers converted from SPM) -- [#618]: Fix Normalizer.normalize with `PyNormalizedStringRefMut` -- [#620]: Fix serialization/deserialization for overlapping models -- [#621]: Fix `ByteLevel` instantiation from a previously saved state (using `__getstate__()`) - -## [0.10.0] - -### Added -- [#508]: Add a Visualizer for notebooks to help understand how the tokenizers work -- [#519]: Add a `WordLevelTrainer` used to train a `WordLevel` model -- [#533]: Add support for conda builds -- [#542]: Add Split pre-tokenizer to easily split using a pattern -- [#544]: Ability to train from memory. This also improves the integration with `datasets` -- [#590]: Add getters/setters for components on BaseTokenizer -- [#574]: Add `fust_unk` option to SentencePieceBPETokenizer - -### Changed -- [#509]: Automatically stubbing the `.pyi` files -- [#519]: Each `Model` can return its associated `Trainer` with `get_trainer()` -- [#530]: The various attributes on each component can be get/set (ie. -`tokenizer.model.dropout = 0.1`) -- [#538]: The API Reference has been improved and is now up-to-date. - -### Fixed -- [#519]: During training, the `Model` is now trained in-place. This fixes several bugs that were -forcing to reload the `Model` after a training. -- [#539]: Fix `BaseTokenizer` enable_truncation docstring - -## [0.9.4] - -### Fixed -- [#492]: Fix `from_file` on `BertWordPieceTokenizer` -- [#498]: Fix the link to download `sentencepiece_model_pb2.py` -- [#500]: Fix a typo in the docs quicktour - -### Changed -- [#506]: Improve Encoding mappings for pairs of sequence - -## [0.9.3] - -### Fixed -- [#470]: Fix hanging error when training with custom component -- [#476]: TemplateProcessing serialization is now deterministic -- [#481]: Fix SentencePieceBPETokenizer.from_files - -### Added -- [#477]: UnicodeScripts PreTokenizer to avoid merges between various scripts -- [#480]: Unigram now accepts an `initial_alphabet` and handles `special_tokens` correctly - -## [0.9.2] - -### Fixed -- [#464]: Fix a problem with RobertaProcessing being deserialized as BertProcessing - -## [0.9.1] - -### Fixed -- [#459]: Fix a problem with deserialization - -## [0.9.0] - -### Fixed -- [#362]: Fix training deadlock with Python components. -- [#363]: Fix a crash when calling `.train` with some non-existent files -- [#355]: Remove a lot of possible crashes -- [#389]: Improve truncation (crash and consistency) - -### Added -- [#379]: Add the ability to call `encode`/`encode_batch` with numpy arrays -- [#292]: Support for the Unigram algorithm -- [#378], [#394], [#416], [#417]: Many new Normalizer and PreTokenizer -- [#403]: Add `TemplateProcessing` `PostProcessor`. -- [#420]: Ability to fuse the "unk" token in BPE. - -### Changed -- [#360]: Lots of improvements related to words/alignment tracking -- [#426]: Improvements on error messages thanks to PyO3 0.12 - -## [0.8.1] - -### Fixed -- [#333]: Fix deserialization of `AddedToken`, where the content was not restored properly - -### Changed -- [#329]: Improved warning and behavior when we detect a fork -- [#330]: BertNormalizer now keeps the same behavior than the original implementation when -`strip_accents` is not specified. - -## [0.8.0] - -### Highlights of this release -- We can now encode both pre-tokenized inputs, and raw strings. This is especially usefull when -processing datasets that are already pre-tokenized like for NER (Name Entity Recognition), and helps -while applying labels to each word. -- Full tokenizer serialization. It is now easy to save a tokenizer to a single JSON file, to later -load it back with just one line of code. That's what sharing a Tokenizer means now: 1 line of code. -- With the serialization comes the compatibility with `Pickle`! The Tokenizer, all of its components, -Encodings, everything can be pickled! -- Training a tokenizer is now even faster (up to 5-10x) than before! -- Compatibility with `multiprocessing`, even when using the `fork` start method. Since this library -makes heavy use of the multithreading capacities of our computers to allows a very fast tokenization, -this led to problems (deadlocks) when used with `multiprocessing`. This version now allows to -disable the parallelism, and will warn you if this is necessary. -- And a lot of other improvements, and fixes. - -### Fixed -- [#286]: Fix various crash when training a BPE model -- [#309]: Fixed a few bugs related to additional vocabulary/tokens - -### Added -- [#272]: Serialization of the `Tokenizer` and all the parts (`PreTokenizer`, `Normalizer`, ...). -This adds some methods to easily save/load an entire tokenizer (`from_str`, `from_file`). -- [#273]: `Tokenizer` and its parts are now pickable -- [#289]: Ability to pad to a multiple of a specified value. This is especially useful to ensure -activation of the Tensor Cores, while ensuring padding to a multiple of 8. Use with -`enable_padding(pad_to_multiple_of=8)` for example. -- [#298]: Ability to get the currently set truncation/padding params -- [#311]: Ability to enable/disable the parallelism using the `TOKENIZERS_PARALLELISM` environment -variable. This is especially usefull when using `multiprocessing` capabilities, with the `fork` -start method, which happens to be the default on Linux systems. Without disabling the parallelism, -the process dead-locks while encoding. (Cf [#187] for more information) - -### Changed -- Improved errors generated during truncation: When the provided max length is too low are -now handled properly. -- [#249] `encode` and `encode_batch` now accept pre-tokenized inputs. When the input is pre-tokenized, -the argument `is_pretokenized=True` must be specified. -- [#276]: Improve BPE training speeds, by reading files sequentially, but parallelizing the -processing of each file -- [#280]: Use `onig` for byte-level pre-tokenization to remove all the differences with the original -implementation from GPT-2 -- [#309]: Improved the management of the additional vocabulary. This introduces an option -`normalized`, controlling whether a token should be extracted from the normalized version of the -input text. - -## [0.7.0] - -### Changed -- Only one progress bar while reading files during training. This is better for use-cases with -a high number of files as it avoids having too many progress bars on screen. Also avoids reading the -size of each file before starting to actually read these files, as this process could take really -long. -- [#193]: `encode` and `encode_batch` now take a new optional argument, specifying whether we -should add the special tokens. This is activated by default. -- [#197]: `original_str` and `normalized_str` have been removed from the `Encoding` returned by -`encode` and `encode_batch`. This brings a reduction of 70% of the memory footprint. -- [#197]: The offsets provided on `Encoding` are now relative to the original string, and not the -normalized one anymore. -- The added token given to `add_special_tokens` or `add_tokens` on a `Tokenizer`, or while using -`train(special_tokens=...)` can now be instances of `AddedToken` to provide more control over these -tokens. -- [#136]: Updated Pyo3 version -- [#136]: Static methods `Model.from_files` and `Model.empty` are removed in favor of using -constructors. -- [#239]: `CharBPETokenizer` now corresponds to OpenAI GPT BPE implementation by default. - -### Added -- [#188]: `ByteLevel` is also a `PostProcessor` now and handles trimming the offsets if activated. -This avoids the unintuitive inclusion of the whitespaces in the produced offsets, even if these -whitespaces are part of the actual token. -It has been added to `ByteLevelBPETokenizer` but it is off by default (`trim_offsets=False`). -- [#236]: `RobertaProcessing` also handles trimming the offsets. -- [#234]: New alignment mappings on the `Encoding`. Provide methods to easily convert between `char` -or `word` (input space) and `token` (output space). -- `post_process` can be called on the `Tokenizer` -- [#208]: Ability to retrieve the vocabulary from the `Tokenizer` with -`get_vocab(with_added_tokens: bool)` -- [#136] Models can now be instantiated through object constructors. - -### Fixed -- [#193]: Fix some issues with the offsets being wrong with the `ByteLevel` BPE: - - when `add_prefix_space=True` - - [#156]: when a Unicode character gets split-up in multiple byte-level characters -- Fix a bug where offsets were wrong when there was any added tokens in the sequence being encoded. -- [#175]: Fix a bug that prevented the addition of more than a certain amount of tokens (even if -not advised, but that's not the question). -- [#205]: Trim the decoded string in `BPEDecoder` used by `CharBPETokenizer` - -### How to migrate -- Add the `ByteLevel` `PostProcessor` to your byte-level BPE tokenizers if relevant. If you are -using `ByteLevelBPETokenizer`, this option is disabled by default (`trim_offsets=False`). -- `BertWordPieceTokenizer` option to `add_special_tokens` must now be given to `encode` or -`encode_batch` -- Access to the `original_str` on the `Encoding` has been removed. The original string is the input -of `encode` so it didn't make sense to keep it here. -- No need to call `original_str.offsets(offsets[N])` to convert offsets to the original string. They -are now relative to the original string by default. -- Access to the `normalized_str` on the `Encoding` has been removed. Can be retrieved by calling -`normalize(sequence)` on the `Tokenizer` -- Change `Model.from_files` and `Model.empty` to use constructor. The model constructor should take -the same arguments as the old methods. (ie `BPE(vocab, merges)` or `BPE()`) -- If you were using the `CharBPETokenizer` and want to keep the same behavior as before, set -`bert_normalizer=False` and `split_on_whitespace_only=True`. - -## [0.6.0] - -### Changed -- [#165]: Big improvements in speed for BPE (Both training and tokenization) - -### Fixed -- [#160]: Some default tokens were missing from `BertWordPieceTokenizer` -- [#156]: There was a bug in ByteLevel PreTokenizer that caused offsets to be wrong if a char got -split up in multiple bytes. -- [#174]: The `longest_first` truncation strategy had a bug - -## [0.5.2] -- [#163]: Do not open all files directly while training - -### Fixed -- We introduced a bug related to the saving of the WordPiece model in 0.5.1: The `vocab.txt` file -was named `vocab.json`. This is now fixed. -- The `WordLevel` model was also saving its vocabulary to the wrong format. - -## [0.5.1] - -### Changed -- `name` argument is now optional when saving a `Model`'s vocabulary. When the name is not -specified, the files get a more generic naming, like `vocab.json` or `merges.txt`. - -## [0.5.0] - -### Changed -- [#145]: `BertWordPieceTokenizer` now cleans up some tokenization artifacts while decoding -- [#149]: `ByteLevelBPETokenizer` now has `dropout`. -- `do_lowercase` has been changed to `lowercase` for consistency between the different tokenizers. -(Especially `ByteLevelBPETokenizer` and `CharBPETokenizer`) -- [#139]: Expose `__len__` on `Encoding` -- Improved padding performances. - -### Added -- Added a new `Strip` normalizer - -### Fixed -- [#145]: Decoding was buggy on `BertWordPieceTokenizer`. -- [#152]: Some documentation and examples were still using the old `BPETokenizer` - -### How to migrate -- Use `lowercase` when initializing `ByteLevelBPETokenizer` or `CharBPETokenizer` instead of -`do_lowercase`. - -## [0.4.2] - -### Fixed -- [#137]: Fix a bug in the class `WordPieceTrainer` that prevented `BertWordPieceTokenizer` from -being trained. - -## [0.4.1] - -### Fixed -- [#134]: Fix a bug related to the punctuation in BertWordPieceTokenizer - -## [0.4.0] - -### Changed -- [#131]: Replaced all .new() class methods by a proper __new__ implementation -- Improved typings - -### How to migrate -- Remove all `.new` on all classe instanciations - -## [0.3.0] - -### Changed -- BPETokenizer has been renamed to CharBPETokenizer for clarity. -- Improve truncation/padding and the handling of overflowing tokens. Now when a sequence gets -truncated, we provide a list of overflowing `Encoding` that are ready to be processed by a language -model, just as the main `Encoding`. -- Provide mapping to the original string offsets using: -``` -output = tokenizer.encode(...) -print(output.original_str.offsets(output.offsets[3])) -``` -- [#99]: Exposed the vocabulary size on all tokenizers - -### Added -- Added `CharDelimiterSplit`: a new `PreTokenizer` that allows splitting sequences on the given -delimiter (Works like `.split(delimiter)`) -- Added `WordLevel`: a new model that simply maps `tokens` to their `ids`. - -### Fixed -- Fix a bug with IndexableString -- Fix a bug with truncation - -### How to migrate -- Rename `BPETokenizer` to `CharBPETokenizer` -- `Encoding.overflowing` is now a List instead of a `Optional[Encoding]` - -## [0.2.1] - -### Fixed -- Fix a bug with the IDs associated with added tokens. -- Fix a bug that was causing crashes in Python 3.5 - -[#1096]: https://github.com/huggingface/tokenizers/pull/1096 -[#1072]: https://github.com/huggingface/tokenizers/pull/1072 -[#956]: https://github.com/huggingface/tokenizers/pull/956 -[#1008]: https://github.com/huggingface/tokenizers/pull/1008 -[#1009]: https://github.com/huggingface/tokenizers/pull/1009 -[#1047]: https://github.com/huggingface/tokenizers/pull/1047 -[#1055]: https://github.com/huggingface/tokenizers/pull/1055 -[#1051]: https://github.com/huggingface/tokenizers/pull/1051 -[#1052]: https://github.com/huggingface/tokenizers/pull/1052 -[#938]: https://github.com/huggingface/tokenizers/pull/938 -[#939]: https://github.com/huggingface/tokenizers/pull/939 -[#952]: https://github.com/huggingface/tokenizers/pull/952 -[#954]: https://github.com/huggingface/tokenizers/pull/954 -[#962]: https://github.com/huggingface/tokenizers/pull/962 -[#961]: https://github.com/huggingface/tokenizers/pull/961 -[#960]: https://github.com/huggingface/tokenizers/pull/960 -[#919]: https://github.com/huggingface/tokenizers/pull/919 -[#916]: https://github.com/huggingface/tokenizers/pull/916 -[#895]: https://github.com/huggingface/tokenizers/pull/895 -[#884]: https://github.com/huggingface/tokenizers/pull/884 -[#882]: https://github.com/huggingface/tokenizers/pull/882 -[#868]: https://github.com/huggingface/tokenizers/pull/868 -[#860]: https://github.com/huggingface/tokenizers/pull/860 -[#850]: https://github.com/huggingface/tokenizers/pull/850 -[#844]: https://github.com/huggingface/tokenizers/pull/844 -[#845]: https://github.com/huggingface/tokenizers/pull/845 -[#851]: https://github.com/huggingface/tokenizers/pull/851 -[#585]: https://github.com/huggingface/tokenizers/pull/585 -[#793]: https://github.com/huggingface/tokenizers/pull/793 -[#780]: https://github.com/huggingface/tokenizers/pull/780 -[#770]: https://github.com/huggingface/tokenizers/pull/770 -[#762]: https://github.com/huggingface/tokenizers/pull/762 -[#718]: https://github.com/huggingface/tokenizers/pull/718 -[#714]: https://github.com/huggingface/tokenizers/pull/714 -[#707]: https://github.com/huggingface/tokenizers/pull/707 -[#693]: https://github.com/huggingface/tokenizers/pull/693 -[#686]: https://github.com/huggingface/tokenizers/pull/686 -[#674]: https://github.com/huggingface/tokenizers/pull/674 -[#657]: https://github.com/huggingface/tokenizers/pull/657 -[#656]: https://github.com/huggingface/tokenizers/pull/656 -[#652]: https://github.com/huggingface/tokenizers/pull/652 -[#621]: https://github.com/huggingface/tokenizers/pull/621 -[#620]: https://github.com/huggingface/tokenizers/pull/620 -[#618]: https://github.com/huggingface/tokenizers/pull/618 -[#617]: https://github.com/huggingface/tokenizers/pull/617 -[#616]: https://github.com/huggingface/tokenizers/pull/616 -[#590]: https://github.com/huggingface/tokenizers/pull/590 -[#574]: https://github.com/huggingface/tokenizers/pull/574 -[#544]: https://github.com/huggingface/tokenizers/pull/544 -[#542]: https://github.com/huggingface/tokenizers/pull/542 -[#539]: https://github.com/huggingface/tokenizers/pull/539 -[#538]: https://github.com/huggingface/tokenizers/pull/538 -[#533]: https://github.com/huggingface/tokenizers/pull/533 -[#530]: https://github.com/huggingface/tokenizers/pull/530 -[#519]: https://github.com/huggingface/tokenizers/pull/519 -[#509]: https://github.com/huggingface/tokenizers/pull/509 -[#508]: https://github.com/huggingface/tokenizers/pull/508 -[#506]: https://github.com/huggingface/tokenizers/pull/506 -[#500]: https://github.com/huggingface/tokenizers/pull/500 -[#498]: https://github.com/huggingface/tokenizers/pull/498 -[#492]: https://github.com/huggingface/tokenizers/pull/492 -[#481]: https://github.com/huggingface/tokenizers/pull/481 -[#480]: https://github.com/huggingface/tokenizers/pull/480 -[#477]: https://github.com/huggingface/tokenizers/pull/477 -[#476]: https://github.com/huggingface/tokenizers/pull/476 -[#470]: https://github.com/huggingface/tokenizers/pull/470 -[#464]: https://github.com/huggingface/tokenizers/pull/464 -[#459]: https://github.com/huggingface/tokenizers/pull/459 -[#420]: https://github.com/huggingface/tokenizers/pull/420 -[#417]: https://github.com/huggingface/tokenizers/pull/417 -[#416]: https://github.com/huggingface/tokenizers/pull/416 -[#403]: https://github.com/huggingface/tokenizers/pull/403 -[#394]: https://github.com/huggingface/tokenizers/pull/394 -[#389]: https://github.com/huggingface/tokenizers/pull/389 -[#379]: https://github.com/huggingface/tokenizers/pull/379 -[#378]: https://github.com/huggingface/tokenizers/pull/378 -[#363]: https://github.com/huggingface/tokenizers/pull/363 -[#362]: https://github.com/huggingface/tokenizers/pull/362 -[#360]: https://github.com/huggingface/tokenizers/pull/360 -[#355]: https://github.com/huggingface/tokenizers/pull/355 -[#333]: https://github.com/huggingface/tokenizers/pull/333 -[#330]: https://github.com/huggingface/tokenizers/pull/330 -[#329]: https://github.com/huggingface/tokenizers/pull/329 -[#311]: https://github.com/huggingface/tokenizers/pull/311 -[#309]: https://github.com/huggingface/tokenizers/pull/309 -[#292]: https://github.com/huggingface/tokenizers/pull/292 -[#289]: https://github.com/huggingface/tokenizers/pull/289 -[#286]: https://github.com/huggingface/tokenizers/pull/286 -[#280]: https://github.com/huggingface/tokenizers/pull/280 -[#276]: https://github.com/huggingface/tokenizers/pull/276 -[#273]: https://github.com/huggingface/tokenizers/pull/273 -[#272]: https://github.com/huggingface/tokenizers/pull/272 -[#249]: https://github.com/huggingface/tokenizers/pull/249 -[#239]: https://github.com/huggingface/tokenizers/pull/239 -[#236]: https://github.com/huggingface/tokenizers/pull/236 -[#234]: https://github.com/huggingface/tokenizers/pull/234 -[#208]: https://github.com/huggingface/tokenizers/pull/208 -[#205]: https://github.com/huggingface/tokenizers/issues/205 -[#197]: https://github.com/huggingface/tokenizers/pull/197 -[#193]: https://github.com/huggingface/tokenizers/pull/193 -[#190]: https://github.com/huggingface/tokenizers/pull/190 -[#188]: https://github.com/huggingface/tokenizers/pull/188 -[#187]: https://github.com/huggingface/tokenizers/issues/187 -[#175]: https://github.com/huggingface/tokenizers/issues/175 -[#174]: https://github.com/huggingface/tokenizers/issues/174 -[#165]: https://github.com/huggingface/tokenizers/pull/165 -[#163]: https://github.com/huggingface/tokenizers/issues/163 -[#160]: https://github.com/huggingface/tokenizers/issues/160 -[#156]: https://github.com/huggingface/tokenizers/pull/156 -[#152]: https://github.com/huggingface/tokenizers/issues/152 -[#149]: https://github.com/huggingface/tokenizers/issues/149 -[#145]: https://github.com/huggingface/tokenizers/issues/145 -[#139]: https://github.com/huggingface/tokenizers/issues/139 -[#137]: https://github.com/huggingface/tokenizers/issues/137 -[#134]: https://github.com/huggingface/tokenizers/issues/134 -[#131]: https://github.com/huggingface/tokenizers/issues/131 -[#99]: https://github.com/huggingface/tokenizers/pull/99 diff --git a/bindings/python/Cargo.lock b/bindings/python/Cargo.lock index acbc8bf8d..db9054852 100644 --- a/bindings/python/Cargo.lock +++ b/bindings/python/Cargo.lock @@ -37,56 +37,12 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" -[[package]] -name = "anstream" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "is_terminal_polyfill", - "utf8parse", -] - [[package]] name = "anstyle" version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" -[[package]] -name = "anstyle-parse" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-query" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" -dependencies = [ - "windows-sys", -] - -[[package]] -name = "anstyle-wincon" -version = "3.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" -dependencies = [ - "anstyle", - "once_cell_polyfill", - "windows-sys", -] - [[package]] name = "arbitrary-chunks" version = "0.4.1" @@ -129,9 +85,9 @@ checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitflags" -version = "2.13.0" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" [[package]] name = "bitvec" @@ -184,9 +140,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.65" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8" dependencies = [ "find-msvc-tools", "shlex", @@ -247,9 +203,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.2" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd059f9da4f5c36b3787f65d38ccaab1cc315f07b01f89abc8359ee6a8205011" +checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7" dependencies = [ "clap_builder", ] @@ -270,12 +226,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" -[[package]] -name = "colorchoice" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" - [[package]] name = "colored" version = "3.1.1" @@ -302,9 +252,9 @@ dependencies = [ [[package]] name = "console" -version = "0.16.3" +version = "0.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d64e8af5551369d19cf50138de61f1c42074ab970f74e99be916646777f8fc87" +checksum = "4fe5f465a4f6fee88fad41b85d990f84c835335e85b5d9e6e63e0d06d28cba7c" dependencies = [ "encode_unicode", "libc", @@ -359,9 +309,9 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -369,18 +319,18 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.21" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" [[package]] name = "crunchy" @@ -390,9 +340,9 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "daachorse" -version = "3.0.2" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99251f238b74cd219a86fe6ea9328308ebb223fcbb5b8eb5aa400b847a41dded" +checksum = "5614204febbc33cc07a2806aa6440b904ac012b68eecc37f4493ea4a76455a3d" [[package]] name = "darling" @@ -415,7 +365,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn", + "syn 2.0.119", ] [[package]] @@ -426,7 +376,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -456,7 +406,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -466,7 +416,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core", - "syn", + "syn 2.0.119", ] [[package]] @@ -481,29 +431,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" -[[package]] -name = "env_filter" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32e90c2accc4b07a8456ea0debdc2e7587bdd890680d71173a15d4ae604f6eef" -dependencies = [ - "log", - "regex", -] - -[[package]] -name = "env_logger" -version = "0.11.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0621c04f2196ac3f488dd583365b9c09be011a4ab8b9f37248ffcc8f6198b56a" -dependencies = [ - "anstream", - "anstyle", - "env_filter", - "jiff", - "log", -] - [[package]] name = "equivalent" version = "1.0.2" @@ -542,9 +469,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "find-msvc-tools" @@ -570,46 +497,25 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" -[[package]] -name = "futures-channel" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" -dependencies = [ - "futures-core", -] - [[package]] name = "futures-core" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" - -[[package]] -name = "futures-macro" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" [[package]] name = "futures-task" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" [[package]] name = "futures-util" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" dependencies = [ "futures-core", - "futures-macro", "futures-task", "pin-project-lite", "slab", @@ -690,9 +596,9 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "indicatif" -version = "0.18.4" +version = "0.18.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25470f23803092da7d239834776d653104d551bc4d7eacaf31e6837854b8e9eb" +checksum = "9433806cd6b4ec1aba79c021c7e4c58fb4c3b9977c085062e611ac929998fb0c" dependencies = [ "console", "portable-atomic", @@ -712,12 +618,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "is_terminal_polyfill" -version = "1.70.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" - [[package]] name = "itertools" version = "0.10.5" @@ -751,35 +651,11 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" -[[package]] -name = "jiff" -version = "0.2.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4603d3033e49e2b0e31229fcab20a5d40089c607d975cd9c80551dc69eed9102" -dependencies = [ - "jiff-static", - "log", - "portable-atomic", - "portable-atomic-util", - "serde_core", -] - -[[package]] -name = "jiff-static" -version = "0.2.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "782d32378dddf207193ac91cefb848ad41abb58195c95168e1291227a0832b47" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "js-sys" -version = "0.3.102" +version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" dependencies = [ "cfg-if", "futures-util", @@ -788,9 +664,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "linux-raw-sys" @@ -800,9 +676,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "log" -version = "0.4.32" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "macro_rules_attribute" @@ -822,9 +698,9 @@ checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30" [[package]] name = "matrixmultiply" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08" +checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7" dependencies = [ "autocfg", "rawpointer", @@ -849,14 +725,14 @@ checksum = "73acd151c6ce84a41d8d6fb0958d9a3d5a18d649ad5a85ad5b719439af8ad257" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "memchr" -version = "2.8.2" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "minimal-lexical" @@ -864,17 +740,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" -[[package]] -name = "mio" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" -dependencies = [ - "libc", - "wasi", - "windows-sys", -] - [[package]] name = "monostate" version = "0.1.18" @@ -894,7 +759,7 @@ checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -903,21 +768,6 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "729eb334247daa1803e0a094d0a5c55711b85571179f5ec6e53eccfdf7008958" -[[package]] -name = "ndarray" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841" -dependencies = [ - "matrixmultiply", - "num-complex", - "num-integer", - "num-traits", - "portable-atomic", - "portable-atomic-util", - "rawpointer", -] - [[package]] name = "ndarray" version = "0.17.2" @@ -977,7 +827,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a5b15d63a5ff39e378daed0e1340d3a5964703ea9712eb09a0dc66fade996f4" dependencies = [ "libc", - "ndarray 0.17.2", + "ndarray", "num-complex", "num-integer", "num-traits", @@ -992,12 +842,6 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" -[[package]] -name = "once_cell_polyfill" -version = "1.70.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" - [[package]] name = "oorandom" version = "11.1.5" @@ -1052,9 +896,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.13.1" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" +checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" [[package]] name = "portable-atomic-util" @@ -1082,9 +926,9 @@ checksum = "9057806a8d77d67bccdc0f542db43737a6f19ada3efab2adc63277feea27310f" [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] @@ -1126,20 +970,6 @@ dependencies = [ "pyo3-macros", ] -[[package]] -name = "pyo3-async-runtimes" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3ef68daa7316a3fac65e5e18b2203f010346de1c1c53456811a2624673ab046" -dependencies = [ - "futures-channel", - "futures-util", - "once_cell", - "pin-project-lite", - "pyo3", - "tokio", -] - [[package]] name = "pyo3-build-config" version = "0.29.0" @@ -1168,7 +998,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1180,14 +1010,14 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "quote" -version = "1.0.45" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -1212,9 +1042,9 @@ checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" [[package]] name = "rand" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -1320,9 +1150,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.4" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -1332,9 +1162,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.14" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" dependencies = [ "aho-corasick", "memchr", @@ -1349,9 +1179,9 @@ checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "rustc-hash" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" [[package]] name = "rustix" @@ -1368,9 +1198,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" [[package]] name = "ryu" @@ -1389,9 +1219,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -1399,29 +1229,29 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "itoa", "memchr", @@ -1436,16 +1266,6 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" -[[package]] -name = "signal-hook-registry" -version = "1.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" -dependencies = [ - "errno", - "libc", -] - [[package]] name = "slab" version = "0.4.12" @@ -1484,9 +1304,20 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "syn" -version = "2.0.118" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" dependencies = [ "proc-macro2", "quote", @@ -1520,22 +1351,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.18" +version = "2.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.18" +version = "2.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] @@ -1603,7 +1434,7 @@ dependencies = [ "monostate", "paste", "ptr_hash", - "rand 0.9.4", + "rand 0.9.5", "rayon", "rayon-cond", "regex", @@ -1637,60 +1468,18 @@ dependencies = [ "tk-encode", ] -[[package]] -name = "tokenizers" -version = "0.23.2-dev.0" -dependencies = [ - "tk-encode", - "tk-train", -] - [[package]] name = "tokenizers-python" -version = "0.23.2-dev.0" +version = "1.0.0-dev.0" dependencies = [ - "ahash", - "env_logger", - "itertools 0.14.0", "libc", - "ndarray 0.16.1", "numpy", - "once_cell", "pyo3", - "pyo3-async-runtimes", - "pyo3-build-config", - "pyo3-ffi", "rayon", "serde", "serde_json", - "tempfile", - "tokenizers", - "tokio", -] - -[[package]] -name = "tokio" -version = "1.52.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" -dependencies = [ - "libc", - "mio", - "pin-project-lite", - "signal-hook-registry", - "tokio-macros", - "windows-sys", -] - -[[package]] -name = "tokio-macros" -version = "2.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" -dependencies = [ - "proc-macro2", - "quote", - "syn", + "tk-encode", + "tk-train", ] [[package]] @@ -1741,12 +1530,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" -[[package]] -name = "utf8parse" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" - [[package]] name = "version_check" version = "0.9.5" @@ -1772,12 +1555,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "wasi" -version = "0.11.1+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" - [[package]] name = "wasip2" version = "1.0.4+wasi-0.2.12" @@ -1789,9 +1566,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.125" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" dependencies = [ "cfg-if", "once_cell", @@ -1802,9 +1579,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.125" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1812,31 +1589,31 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.125" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.119", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.125" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" dependencies = [ "unicode-ident", ] [[package]] name = "web-sys" -version = "0.3.102" +version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" dependencies = [ "js-sys", "wasm-bindgen", @@ -1893,9 +1670,9 @@ dependencies = [ [[package]] name = "xxhash-rust" -version = "0.8.17" +version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985eec839aaf2a1270af8f4ebcf63cf9401cfd90f0902f97c28d9f104ffbde72" +checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" [[package]] name = "yada" @@ -1905,26 +1682,26 @@ checksum = "53c3bb06259642a57b4ea1bf2a8260f7d94b7b78a096c46f193318918d925f61" [[package]] name = "zerocopy" -version = "0.8.52" +version = "0.8.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.52" +version = "0.8.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "zmij" -version = "1.0.21" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index df887ae19..bd728de19 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -1,41 +1,26 @@ [package] name = "tokenizers-python" -version = "0.23.2-dev.0" -authors = ["Anthony MOI "] +version = "1.0.0-dev.0" edition = "2024" +description = "Fast Python bindings for 🤗 tokenizers, built on the PipelineTokenizer encode path" +license = "Apache-2.0" [lib] -name = "tokenizers" -crate-type = ["cdylib"] +name = "_native" +crate-type = ["cdylib", "rlib"] [dependencies] +pyo3 = { version = "=0.29", features = ["abi3-py310", "experimental-inspect"] } +numpy = "0.29" rayon = "1.10" -serde = { version = "1.0", features = ["rc", "derive"] } +serde = "1.0" serde_json = "1.0" -libc = "0.2" -env_logger = "0.11" -pyo3 = { version = "=0.29", default-features = false, features = ["py-clone", "experimental-inspect"] } -pyo3-async-runtimes = { version = "0.29", features = ["tokio-runtime"] } -tokio = { version = "1.47.1", features = ["rt", "rt-multi-thread", "macros", "signal"] } -once_cell = "1.19.0" -numpy = "0.29" -ndarray = "0.16" -itertools = "0.14" -ahash = { version = "0.8.11", features = ["serde"] } -pyo3-ffi = { version = "=0.29" } +tk-encode = { path = "../../tokenizers/tk-encode", features = ["fancy-regex"] } +tk-train = { path = "../../tokenizers/tk-train" } -[dependencies.tokenizers] -path = "../../tokenizers" - -[build-dependencies] -pyo3-build-config = "=0.29" - -[dev-dependencies] -tempfile = "3.10" -pyo3 = { version = "0.29", features = ["auto-initialize", "experimental-inspect"] } +[target.'cfg(unix)'.dependencies] +libc = "0.2" [features] -default = ["ext-module", "abi3", "parity-aware-bpe"] +default = [] ext-module = ["pyo3/extension-module"] -abi3 = ["pyo3/abi3", "pyo3/abi3-py310"] -parity-aware-bpe = ["tokenizers/parity-aware-bpe"] diff --git a/bindings/python/MANIFEST.in b/bindings/python/MANIFEST.in deleted file mode 100644 index 74c4f2c76..000000000 --- a/bindings/python/MANIFEST.in +++ /dev/null @@ -1,7 +0,0 @@ -include Cargo.toml -include pyproject.toml -include rust-toolchain -include ../../LICENSE -recursive-include src * -recursive-include tokenizers-lib * -recursive-exclude tokenizers-lib/target * diff --git a/bindings/python/Makefile b/bindings/python/Makefile index 4b0367e5f..67f13839b 100644 --- a/bindings/python/Makefile +++ b/bindings/python/Makefile @@ -1,67 +1,41 @@ -.PHONY: style check-style test - -DATA_DIR = data - -dir_guard=@mkdir -p $(@D) -check_dirs := examples py_src/tokenizers tests -check_stubs := $(filter-out py_src/tokenizers/tokenizers.pyi,$(wildcard py_src/tokenizers/*.pyi py_src/tokenizers/*/*.pyi)) - -# Detect uv and set env vars to work around broken dylib install names -# in python-build-standalone distributions. -# See: https://github.com/astral-sh/uv/issues/11006 -HAS_UV := $(shell command -v uv >/dev/null 2>&1 && echo 1 || echo 0) -ifeq ($(HAS_UV),1) - PIP := uv pip - CARGO_ENV := DYLD_FALLBACK_LIBRARY_PATH=$(shell python3 -c "import sysconfig; print(sysconfig.get_config_var('LIBDIR'))") \ - PYTHONHOME=$(shell python3 -c "import sys; print(sys.base_prefix)") -else - PIP := pip - CARGO_ENV := -endif - -# Format source code automatically -style: - $(CARGO_ENV) cargo run --manifest-path ./tools/stub-gen/Cargo.toml - ruff check $(check_stubs) --select I --fix - ruff check $(check_dirs) --fix - ruff format $(check_dirs) - ty check py_src --exclude py_src/tokenizers/implementations --exclude py_src/tokenizers/tools/visualizer.py - - - -# Check the source code is formatted correctly -check-style: - $(CARGO_ENV) cargo run --manifest-path ./tools/stub-gen/Cargo.toml - ruff check $(check_stubs) --select I --fix - ruff check $(check_dirs) - ruff format py_src/tokenizers/*.pyi - ruff format --check $(check_dirs) - ty check py_src --exclude py_src/tokenizers/implementations --exclude py_src/tokenizers/tools/visualizer.py - - -TESTS_RESOURCES = $(DATA_DIR)/small.txt $(DATA_DIR)/roberta.json - -# Launch the Python test suite -test-py: $(TESTS_RESOURCES) - $(PIP) install pytest pytest-asyncio huggingface_hub setuptools_rust numpy pyarrow datasets - python -m pytest -s -v tests - -# Launch the Rust test suite -# (cargo test --no-default-features uses pyo3's `auto-initialize` which -# links libpython; this doesn't work on free-threaded Python on macOS -# runners where libpython3.14t isn't installed in the framework path.) -test-rs: - $(CARGO_ENV) cargo test --no-default-features - -# Full test suite -test: test-py test-rs - -HF_TEST_REPO = hf-internal-testing/tokenizers-test-data -# Fetch fixtures through the huggingface_hub CLI: it handles auth (HF_TOKEN), -# retry/backoff on rate limits, and caching for us. CI overrides HF with -# `uvx --from huggingface_hub hf` to avoid a separate install step. -HF ?= hf - -$(DATA_DIR)/% : - $(dir_guard) - $(HF) download $(HF_TEST_REPO) $* --repo-type dataset --local-dir $(DATA_DIR) +PYTHON := .venv/bin/python + +# Everything needed to hack on the bindings: venv, deps, release build. +.PHONY: dev +dev: .venv + . .venv/bin/activate && maturin develop --release + +.venv: + uv venv .venv + uv pip install --python $(PYTHON) maturin numpy + +# Regenerate the .pyi stubs from the built extension. Run after `make dev`. +.PHONY: stubs +stubs: + cargo run --manifest-path tools/stub-gen/Cargo.toml + +# Run the end-to-end examples (train, pretrained parity, threading). +.PHONY: examples +examples: dev + $(PYTHON) examples/01_train_and_encode.py + $(PYTHON) examples/02_pretrained.py + $(PYTHON) examples/03_threading.py + +# The released PyPI wheel shares our package name, so it lives in its own +# directory that the bench subprocess puts on PYTHONPATH. +.release: | .venv + uv pip install --python $(PYTHON) --target .release tokenizers + +# Benchmark against the released tokenizers wheel (same script CI runs). +.PHONY: bench +bench: dev .release + $(PYTHON) benches/bench_vs_release.py + +.PHONY: lint +lint: + cargo fmt --check + cargo clippy --all-targets -- -D warnings + +.PHONY: clean +clean: + rm -rf .venv .release target tools/stub-gen/target diff --git a/bindings/python/README.md b/bindings/python/README.md index e371c43be..21967f5f5 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -1,252 +1,115 @@ -

-
- -
-

-

- - Build - - - GitHub - -

-
- -# Tokenizers - -Provides an implementation of today's most used tokenizers, with a focus on performance and -versatility. - -Bindings over the [Rust](https://github.com/huggingface/tokenizers/tree/master/tokenizers) implementation. -If you are interested in the High-level design, you can go check it there. - -Otherwise, let's dive in! - -## Main features: - - - Train new vocabularies and tokenize using 4 pre-made tokenizers (Bert WordPiece and the 3 - most common BPE versions). - - Extremely fast (both training and tokenization), thanks to the Rust implementation. Takes - less than 20 seconds to tokenize a GB of text on a server's CPU. - - Easy to use, but also extremely versatile. - - Designed for research and production. - - Normalization comes with alignments tracking. It's always possible to get the part of the - original sentence that corresponds to a given token. - - Does all the pre-processing: Truncate, Pad, add the special tokens your model needs. - -### Installation - -#### With pip: - -```bash -pip install tokenizers -``` - -#### From sources: - -To use this method, you need to have the Rust installed: - -```bash -# Install with: -curl https://sh.rustup.rs -sSf | sh -s -- -y -export PATH="$HOME/.cargo/bin:$PATH" -``` - -Once Rust is installed, you can compile doing the following - -```bash -git clone https://github.com/huggingface/tokenizers -cd tokenizers/bindings/python - -# Create a virtual env (you can use yours as well) -python -m venv .env -source .env/bin/activate - -# Install `tokenizers` in the current virtual env -pip install -e . -``` - -### Free-threaded Python (3.14t) - -`tokenizers` ships dedicated wheels for the [free-threaded build of CPython](https://docs.python.org/3.14/howto/free-threading-python.html) -(`python3.14t`). These wheels declare `Py_MOD_GIL_NOT_USED`, so importing -`tokenizers` does **not** force the GIL back on — multi-threaded code stays -GIL-free. - -The full mutable API works on 3.14t — the same as on regular CPython. -Setters are thread-safe: the inner tokenizer state is wrapped in a -`std::sync::RwLock`, so concurrent `tokenizer.X = …` from multiple threads -serialize correctly and concurrent encode operations take a read guard -that blocks writers only briefly. - -```python -from tokenizers import Tokenizer -from tokenizers.models import BPE -from tokenizers.pre_tokenizers import Whitespace -from tokenizers.processors import ByteLevel - -tok = Tokenizer(BPE()) -tok.pre_tokenizer = Whitespace() # ✅ thread-safe on 3.14t -tok.post_processor = ByteLevel(trim_offsets=True) -``` - -**Caveat — compound mutations are not atomic.** Statements like -`tokenizer.post_processor.special_tokens = X` evaluate in two steps from -Python's point of view (read attribute → set attribute on the result). If -another thread swaps `tokenizer.post_processor` between those steps, the -mutation lands on an orphaned component. This is the same class of race -as `dict[k] = v` interleaved with `dict.clear()` — coordinate with a Python -lock if you need the compound to be atomic. - -For the full thread-safety analysis, see -[`docs/free-threading-audit.md`](./docs/free-threading-audit.md). - -### Load a pretrained tokenizer from the Hub - -```python -from tokenizers import Tokenizer - -tokenizer = Tokenizer.from_pretrained("bert-base-cased") -``` - -### Using the provided Tokenizers - -We provide some pre-build tokenizers to cover the most common cases. You can easily load one of -these using some `vocab.json` and `merges.txt` files: +# tokenizers (Python bindings) -```python -from tokenizers import CharBPETokenizer - -# Initialize a tokenizer -vocab = "./path/to/vocab.json" -merges = "./path/to/merges.txt" -tokenizer = CharBPETokenizer(vocab, merges) - -# And then encode: -encoded = tokenizer.encode("I can feel the magic, can you?") -print(encoded.ids) -print(encoded.tokens) -``` - -And you can train them just as simply: +Python bindings for 🤗 tokenizers, built on the `PipelineTokenizer` encode +path. This is the 1.x rewrite of the bindings: same `tokenizer.json` files, +same ids as 0.x, and much faster through Python — encode never holds the GIL, +batches run multi-threaded in Rust, inputs are borrowed instead of copied, and +ids come back as `numpy.uint32` arrays without a copy. ```python -from tokenizers import CharBPETokenizer - -# Initialize a tokenizer -tokenizer = CharBPETokenizer() - -# Then train it! -tokenizer.train([ "./path/to/files/1.txt", "./path/to/files/2.txt" ]) +import tokenizers as tk -# Now, let's use it: -encoded = tokenizer.encode("I can feel the magic, can you?") - -# And finally save it somewhere -tokenizer.save("./path/to/directory/my-bpe.tokenizer.json") +tok = tk.Tokenizer.from_file("tokenizer.json") +ids = tok.encode("Hello world", add_special_tokens=False) # np.ndarray[uint32] +batch = tok.encode_batch(lines, add_special_tokens=False) # list of arrays ``` -#### Provided Tokenizers - - - `CharBPETokenizer`: The original BPE - - `ByteLevelBPETokenizer`: The byte level version of the BPE - - `SentencePieceBPETokenizer`: A BPE implementation compatible with the one used by SentencePiece - - `BertWordPieceTokenizer`: The famous Bert tokenizer, using WordPiece - -All of these can be used and trained as explained above! - -### Build your own - -Whenever these provided tokenizers don't give you enough freedom, you can build your own tokenizer, -by putting all the different parts you need together. -You can check how we implemented the [provided tokenizers](https://github.com/huggingface/tokenizers/tree/master/bindings/python/py_src/tokenizers/implementations) and adapt them easily to your own needs. - -#### Building a byte-level BPE - -Here is an example showing how to build your own byte-level BPE by putting all the different pieces -together, and then saving it to a single file: - -```python -from tokenizers import Tokenizer, models, pre_tokenizers, decoders, trainers, processors - -# Initialize a tokenizer -tokenizer = Tokenizer(models.BPE()) - -# Customize pre-tokenization and decoding -tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=True) -tokenizer.decoder = decoders.ByteLevel() -tokenizer.post_processor = processors.ByteLevel(trim_offsets=True) - -# And then train -trainer = trainers.BpeTrainer( - vocab_size=20000, - min_frequency=2, - initial_alphabet=pre_tokenizers.ByteLevel.alphabet() -) -tokenizer.train([ - "./path/to/dataset/1.txt", - "./path/to/dataset/2.txt", - "./path/to/dataset/3.txt" -], trainer=trainer) - -# And Save it -tokenizer.save("byte-level-bpe.tokenizer.json", pretty=True) -``` - -Now, when you want to use this tokenizer, this is as simple as: +Training and in-place modification work too: ```python -from tokenizers import Tokenizer - -tokenizer = Tokenizer.from_file("byte-level-bpe.tokenizer.json") - -encoded = tokenizer.encode("I can feel the magic, can you?") +tok = tk.Tokenizer(tk.models.BPE()) +tok.normalizer = tk.normalizers.Lowercase() +tok.pre_tokenizer = tk.pre_tokenizers.Whitespace() +tok.train_from_iterator(lines, trainer=tk.trainers.BpeTrainer(vocab_size=30000)) +tok.save("tokenizer.json") ``` -### Typing support and stub generation +## Breaking changes vs 0.x -The compiled PyO3 extension does not expose type annotations, so editors and type checkers would otherwise see most objects as `Any`. To provide full typing support, we use a two-step stub generation process: +1.x is a ground-up rewrite with a smaller, faster API. The headline changes: -1. **Rust introspection** (`tools/stub-gen/`): Uses `pyo3-introspection` to analyze the compiled extension and generate `.pyi` stub files -2. **Python enrichment** (`stub.py`): Adds docstrings from the runtime module and generates forwarding `__init__.py` shims +- `encode` returns a `numpy.uint32` array of ids, not an `Encoding` object. + Offsets, type ids, and attention masks are gone from the encode path. +- Not implemented yet (loud errors, never wrong ids): `decode`, + post-processor templates (`[CLS]`/`` insertion — pass + `add_special_tokens=False`), and the `Metaspace` pre-tokenizer + (t5-style files). +- Custom Python components (normalizers/pre-tokenizers written in Python) + are not supported; components are plain values you assign, not objects + you subclass. +- `decoders`, `processors`, and the `implementations` helpers + (`BertWordPieceTokenizer`, …) are gone. +- The wheel is abi3 (one binary for CPython 3.10–3.14); free-threaded + interpreters (3.14t) are not supported yet. Multi-threaded encode does not + need them — the GIL is released. -#### Running stub generation +## Build and use locally -The easiest way to regenerate stubs is via `make style`: +Requirements: Rust (stable), [uv](https://docs.astral.sh/uv/), Python ≥ 3.10. -```bash +```sh cd bindings/python -make style +make dev # venv + deps + release build, installed editable +source .venv/bin/activate +python -c "import tokenizers; print(tokenizers.__version__)" ``` -This will: -1. Build the extension with `maturin develop --release` -2. Run introspection to generate `.pyi` files -3. Enrich stubs with docstrings via `stub.py` -4. Format with `ruff` +Rebuild after changing Rust code with `make dev` again (or `maturin develop +--release` inside the venv). Always use `--release`: a debug build encodes +10-100× slower and any timing you take from it is meaningless. -#### Running manually +To build a distributable wheel instead: `maturin build --release` (find it in +`target/wheels/`). -To run the stub generator directly: +Other targets: -```bash -cd bindings/python -cargo run --manifest-path tools/stub-gen/Cargo.toml -python stub.py +```sh +make examples # run the three end-to-end examples (needs ../../tokenizers/data) +make bench # benchmark against the released tokenizers wheel from PyPI +make stubs # regenerate the .pyi type stubs from the built extension +make lint # cargo fmt --check + clippy -D warnings ``` -The stub generator automatically: -- Builds the extension using maturin -- Copies the built `.so` to the project root for introspection -- Detects and sets `PYTHONHOME` for embedded Python (handles uv/venv environments) -- Generates stubs to `py_src/tokenizers/` - -#### Troubleshooting - -If you encounter Python initialization errors, you can manually set `PYTHONHOME`: - -```bash -export PYTHONHOME=$(python3 -c 'import sys; print(sys.base_prefix)') -cargo run --manifest-path tools/stub-gen/Cargo.toml -``` +The examples and the benchmark read test data from `../../tokenizers/data`. +Fetch it once with `make -C ../../tokenizers fixtures bench-models data/big.txt` +(needs `HF_TOKEN` for the mirror repo). + +## Type stubs are generated + +Do not edit the `.pyi` files under `py_src/` by hand. They are produced by +`tools/stub-gen`, which reads the introspection metadata pyo3 embeds in the +built extension — so run `make dev` first, then `make stubs`. Docstrings and +signatures come from the Rust sources; return types that introspection cannot +see (numpy arrays, `Self`) are declared with +`#[pyo3(signature = (...) -> "Type")]` annotations in the Rust code. + +## How it works + +A `Tokenizer` holds two things behind one lock: + +- the **spec** — the plain Rust `Tokenizer`, the serializable source of truth. + Setters, `train*`, and `add_*` write here. +- the **compiled pipeline** — an immutable `Arc` the encode + methods share with worker threads. Any mutation drops it; the next encode + rebuilds it once. Configurations the pipeline cannot run fail at that point + with the reason, never with different ids. + +Every method releases the GIL before touching the lock — enforced at compile +time by `DetachedRwLock` (see `src/detached_lock.rs`), with a clippy ban on +`Python::attach` as the backstop. + +## Benchmark + +`benches/bench_vs_release.py` times `encode_batch` end-to-end through Python +against the latest released `tokenizers` wheel, on the same corpora and ~10 KiB +chunking as the Rust benchmark (`tk-encode/examples/fixture_bench.rs`): every +fixture under `data/fixtures/{lang,modalities}`, warmed up, median of N runs, +single-thread per fixture plus one multi-thread sweep, ids verified equal +before the run counts. Because the released wheel and this build share the +package name, the release is installed into `.release/` (`make bench` does +this) and benched in a subprocess with `PYTHONPATH` pointing there. + +CI runs it in the `python-bindings-bench` job of the Pipeline Benchmark +workflow, posts the table to the run's step summary, and the report job +renders it as a chart (`.github/scripts/render_python_bench.py`) appended to +the benchmark section in the PR description, next to the Rust charts. diff --git a/bindings/python-pipeline/benches/bench_vs_release.py b/bindings/python/benches/bench_vs_release.py similarity index 50% rename from bindings/python-pipeline/benches/bench_vs_release.py rename to bindings/python/benches/bench_vs_release.py index 005d7d5ad..a07bff242 100644 --- a/bindings/python-pipeline/benches/bench_vs_release.py +++ b/bindings/python/benches/bench_vs_release.py @@ -1,29 +1,44 @@ -"""Benchmark tokenizers_pipeline against the released `tokenizers` wheel. +"""Benchmark these bindings against the released `tokenizers` wheel from PyPI. Mirrors tk-encode/examples/fixture_bench.rs: every `.txt` corpus under data/fixtures/{lang,modalities}, cut into ~10 KiB multi-line chunks (at most 100 per fixture), single-thread throughput per fixture plus one multi-thread sweep over all fixtures flattened. Timing is end-to-end through Python — input conversion, encode, and output objects all count, because that is what -a user pays. Ids are checked to match on every fixture before anything is -timed; a mismatch fails the run. +a user pays. Ids are checked to match on every fixture; a mismatch fails the +run. + +The local build and the released wheel share the package name `tokenizers`, +so they cannot be imported into one process. The released wheel lives in its +own directory (`pip install --target tokenizers`) and this script +re-runs itself in a subprocess with PYTHONPATH pointing there — PYTHONPATH +wins over site-packages, so the subprocess sees the release while the main +process sees the local build. Ids cross the process boundary as one SHA-1 +digest per chunk. Usage: python benches/bench_vs_release.py [--manifest bench_models.json] [--data-dir ../../tokenizers/data] [--iters 3] - [--json out.json] [--markdown out.md] + [--release-dir .release] [--json out.json] [--markdown out.md] + +Set up the release directory once with: + uv pip install --target .release tokenizers """ import argparse +import hashlib import json import os import statistics +import subprocess import sys +import tempfile import time from pathlib import Path -import tokenizers as release -import tokenizers_pipeline as pipeline +import numpy as np + +import tokenizers # Keep in sync with fixture_bench.rs (CHUNK_BYTES, MAX_CHUNKS). CHUNK_BYTES = 10 * 1024 @@ -86,36 +101,52 @@ def timed(fn, iters: int) -> float: return statistics.median(samples) -def bench_model(model: dict, fixtures: list[dict], iters: int) -> dict: - row = {"model": model["name"], "fixtures": []} - try: - ours = pipeline.Tokenizer.from_file(model["path"]) - ours.encode("warmup", add_special_tokens=False) - except (pipeline.TokenizersError, NotImplementedError) as e: - row["skipped"] = str(e) - return row - theirs = release.Tokenizer.from_file(str(model["path"])) +def digests(batch) -> list[str]: + return [hashlib.sha1(np.asarray(ids, dtype=np.uint32).tobytes()).hexdigest() for ids in batch] + + +def bench_release_side(models: list[dict], fixtures: list[dict], iters: int) -> dict: + """Runs in the subprocess where `tokenizers` is the released wheel.""" + all_chunks = [c for f in fixtures for c in f["chunks"]] + nbytes = sum(f["bytes"] for f in fixtures) + out = {"version": tokenizers.__version__, "models": {}} + for model in models: + tok = tokenizers.Tokenizer.from_file(model["path"]) + os.environ["TOKENIZERS_PARALLELISM"] = "false" + rows = [] + for fixture in fixtures: + chunks = fixture["chunks"] + encoded = tok.encode_batch_fast(chunks, add_special_tokens=False) + t = timed(lambda: tok.encode_batch_fast(chunks, add_special_tokens=False), iters) + rows.append( + { + "mbps": fixture["bytes"] / t / 1e6, + "digests": digests([e.ids for e in encoded]), + } + ) + os.environ["TOKENIZERS_PARALLELISM"] = "true" + t = timed(lambda: tok.encode_batch_fast(all_chunks, add_special_tokens=False), iters) + out["models"][model["name"]] = {"fixtures": rows, "multi_thread_mbps": nbytes / t / 1e6} + return out + +def bench_local_side(tok, fixtures: list[dict], release_row: dict, iters: int) -> dict: + row = {"fixtures": []} os.environ["TOKENIZERS_PARALLELISM"] = "false" - for fixture in fixtures: + for fixture, rel in zip(fixtures, release_row["fixtures"], strict=True): chunks = fixture["chunks"] - parity = ours.encode_batch(chunks, add_special_tokens=False) - reference = theirs.encode_batch_fast(chunks, add_special_tokens=False) - t_ours = timed(lambda: ours.encode_batch(chunks, add_special_tokens=False), iters) - t_theirs = timed( - lambda: theirs.encode_batch_fast(chunks, add_special_tokens=False), iters - ) + encoded = tok.encode_batch(chunks, add_special_tokens=False) + t = timed(lambda: tok.encode_batch(chunks, add_special_tokens=False), iters) + mbps = fixture["bytes"] / t / 1e6 row["fixtures"].append( { "fixture": fixture["name"], "group": fixture["group"], "bytes": fixture["bytes"], - "ids_match": all( - a.tolist() == b.ids for a, b in zip(parity, reference, strict=True) - ), - "pipeline_mbps": fixture["bytes"] / t_ours / 1e6, - "release_mbps": fixture["bytes"] / t_theirs / 1e6, - "speedup": t_theirs / t_ours, + "ids_match": digests(encoded) == rel["digests"], + "pipeline_mbps": mbps, + "release_mbps": rel["mbps"], + "speedup": mbps / rel["mbps"], } ) @@ -123,31 +154,29 @@ def bench_model(model: dict, fixtures: list[dict], iters: int) -> dict: all_chunks = [c for f in fixtures for c in f["chunks"]] nbytes = sum(f["bytes"] for f in fixtures) os.environ["TOKENIZERS_PARALLELISM"] = "true" - t_ours = timed(lambda: ours.encode_batch(all_chunks, add_special_tokens=False), iters) - t_theirs = timed( - lambda: theirs.encode_batch_fast(all_chunks, add_special_tokens=False), iters - ) + t = timed(lambda: tok.encode_batch(all_chunks, add_special_tokens=False), iters) + mbps = nbytes / t / 1e6 row["multi_thread"] = { "bytes": nbytes, - "pipeline_mbps": nbytes / t_ours / 1e6, - "release_mbps": nbytes / t_theirs / 1e6, - "speedup": t_theirs / t_ours, + "pipeline_mbps": mbps, + "release_mbps": release_row["multi_thread_mbps"], + "speedup": mbps / release_row["multi_thread_mbps"], } return row def render_markdown(report: dict) -> str: lines = [ - "## Python bindings: `tokenizers_pipeline` vs released `tokenizers` " - f"{report['release_version']}", + "## Python bindings: this branch vs `tokenizers` " + f"{report['release_version']} (PyPI)", "", f"{report['fixture_count']} fixtures (~10 KiB chunks, ≤100/fixture), median of " f"{report['iters']} runs, {report['cpus']} CPUs. Single-thread numbers aggregate " "all fixtures (speedup range = slowest…fastest fixture); multi-thread runs the " - "flattened corpus. Speedup >1 means the pipeline bindings are faster.", + "flattened corpus. Speedup >1 means this branch is faster.", "", - "| model | ids | pipeline 1t (MB/s) | release 1t (MB/s) | speedup 1t (range) " - "| pipeline mt (MB/s) | release mt (MB/s) | speedup mt |", + "| model | ids | branch 1t (MB/s) | release 1t (MB/s) | speedup 1t (range) " + "| branch mt (MB/s) | release mt (MB/s) | speedup mt |", "|---|---|---|---|---|---|---|---|", ] for row in report["models"]: @@ -176,23 +205,78 @@ def main() -> int: parser.add_argument("--manifest", type=Path, help="bench_models.json to take the model list from") parser.add_argument("--data-dir", type=Path, default=Path(__file__).parents[3] / "tokenizers" / "data") parser.add_argument("--iters", type=int, default=3) + parser.add_argument("--release-dir", type=Path, default=Path(__file__).parents[1] / ".release") parser.add_argument("--json", type=Path, help="write the full report here") parser.add_argument("--markdown", type=Path, help="write the summary table here") + parser.add_argument("--side", choices=["release"], help=argparse.SUPPRESS) + parser.add_argument("--models-json", type=Path, help=argparse.SUPPRESS) + parser.add_argument("--out", type=Path, help=argparse.SUPPRESS) args = parser.parse_args() + if args.side == "release": + models = json.loads(args.models_json.read_text()) + fixtures = load_fixtures(args.data_dir) + args.out.write_text(json.dumps(bench_release_side(models, fixtures, args.iters))) + return 0 + + if not (args.release_dir / "tokenizers").is_dir(): + sys.exit( + f"released wheel not found in {args.release_dir} — run " + f"`uv pip install --target {args.release_dir} tokenizers`" + ) + models = json.load(open(args.manifest)) if args.manifest else DEFAULT_MODELS for model in models: - model["path"] = args.data_dir / model.get("file", model["name"] + ".json") - models = [m for m in models if m["path"].is_file()] or sys.exit("no model files found") + model["path"] = str(args.data_dir / model.get("file", model["name"] + ".json")) + models = [m for m in models if Path(m["path"]).is_file()] or sys.exit("no model files found") fixtures = load_fixtures(args.data_dir) + + compiled: dict[str, object] = {} + skipped: dict[str, str] = {} + for m in models: + try: + tok = tokenizers.Tokenizer.from_file(m["path"]) + tok.encode("warmup", add_special_tokens=False) + compiled[m["name"]] = tok + except (tokenizers.TokenizersError, NotImplementedError) as e: + skipped[m["name"]] = str(e) + + with tempfile.TemporaryDirectory() as td: + models_json = Path(td) / "models.json" + models_json.write_text( + json.dumps([{"name": m["name"], "path": m["path"]} for m in models if m["name"] in compiled]) + ) + release_out = Path(td) / "release.json" + subprocess.run( + [ + sys.executable, __file__, + "--side", "release", + "--models-json", str(models_json), + "--out", str(release_out), + "--data-dir", str(args.data_dir), + "--iters", str(args.iters), + ], + env=os.environ | {"PYTHONPATH": str(args.release_dir)}, + check=True, + ) + release = json.loads(release_out.read_text()) + + rows = [] + for m in models: + if m["name"] in skipped: + rows.append({"model": m["name"], "skipped": skipped[m["name"]]}) + continue + row = bench_local_side(compiled[m["name"]], fixtures, release["models"][m["name"]], args.iters) + rows.append({"model": m["name"], **row}) + report = { - "release_version": release.__version__, - "pipeline_version": pipeline.__version__, + "release_version": release["version"], + "pipeline_version": tokenizers.__version__, "iters": args.iters, "fixture_count": len(fixtures), "cpus": os.cpu_count(), - "models": [bench_model(m, fixtures, args.iters) for m in models], + "models": rows, } markdown = render_markdown(report) diff --git a/bindings/python/benches/test_tiktoken.py b/bindings/python/benches/test_tiktoken.py deleted file mode 100755 index f88c18e0f..000000000 --- a/bindings/python/benches/test_tiktoken.py +++ /dev/null @@ -1,128 +0,0 @@ -import os -import time -import argparse -from datasets import load_dataset -from tiktoken.load import load_tiktoken_bpe # type: ignore[import] -import tiktoken # type: ignore[import] -from tokenizers import Tokenizer -from huggingface_hub import hf_hub_download -from typing import Tuple, List -from multiprocessing import Process - -MODEL_ID = "meta-llama/Llama-3.2-1B" -DATASET = "facebook/xnli" -DATASET_CONFIG = "all_languages" -DEFAULT_THREADS = [2**i for i in range(8) if 2**i] - - -def format_byte_size(num_bytes: int) -> Tuple[str, str]: - """Convert bytes to a human-readable format (KB, MB, GB).""" - num_bytes_f = float(num_bytes) - for unit in ["B", "KB", "MB", "GB", "TB"]: - if num_bytes_f < 1024: - return f"{num_bytes_f:.2f} {unit}", unit - num_bytes_f /= 1024 - return f"{num_bytes_f:.2f} PB", "PB" - - -def benchmark_batch(model: str, documents: list[str], num_threads: int, document_length: float) -> None: - os.environ["RAYON_NUM_THREADS"] = str(num_threads) - num_bytes = sum(map(len, map(str.encode, documents))) - readable_size, unit = format_byte_size(num_bytes) - print(f"==============") - print( - f"num_threads: {num_threads}, data size: {readable_size}, documents: {len(documents)} Avg Length: {document_length:.0f}" - ) - filename = hf_hub_download(MODEL_ID, "original/tokenizer.model") - mergeable_ranks = load_tiktoken_bpe(filename) - pat_str = r"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+" - num_reserved_special_tokens = 256 - special_tokens = [ - "<|begin_of_text|>", - "<|end_of_text|>", - "<|reserved_special_token_0|>", - "<|reserved_special_token_1|>", - "<|reserved_special_token_2|>", - "<|reserved_special_token_3|>", - "<|start_header_id|>", - "<|end_header_id|>", - "<|reserved_special_token_4|>", - "<|eot_id|>", # end of turn - ] + [f"<|reserved_special_token_{i}|>" for i in range(5, num_reserved_special_tokens - 5)] - num_base_tokens = len(mergeable_ranks) - special_tokens = {token: num_base_tokens + i for i, token in enumerate(special_tokens)} - enc = tiktoken.Encoding( - name=model, - pat_str=pat_str, - mergeable_ranks=mergeable_ranks, - special_tokens=special_tokens, - ) - out = enc.encode("This is a test") - - hf_enc = Tokenizer.from_pretrained(model) - out2 = hf_enc.encode("This is a test", add_special_tokens=False).ids - - assert out == out2, "sanity check" - - start = time.perf_counter_ns() - enc.encode_ordinary_batch(documents, num_threads=num_threads) - end = time.perf_counter_ns() - - readable_size, unit = format_byte_size(num_bytes / (end - start) * 1e9) - print(f"tiktoken \t{readable_size} / s") - - start = time.perf_counter_ns() - hf_enc.encode_batch_fast(documents) - end = time.perf_counter_ns() - readable_size, unit = format_byte_size(num_bytes / (end - start) * 1e9) - print(f"huggingface \t{readable_size} / s") - - -def test(model: str, dataset: str, dataset_config: str, threads: List[int]): - dataset_xnli = load_dataset(dataset, dataset_config) - - input_lengths = [(10, False, True), (10_000, False, True), (10_000, False, False)] - - for num_threads in threads: - for length, fuse, long in input_lengths: - documents = [] - for i, item in enumerate(dataset_xnli["train"]): - if i >= length: - break - if long: - documents.append("".join(item["premise"].values())) - else: - documents.append(item["premise"]["en"]) - if fuse: - documents = ["".join(documents)] - - document_length = sum(len(d) for d in documents) / len(documents) - - # Rayon thread pool is global to a process, we need to launch - # separate processes in order to accurately use the correct number of threads. - # Otherwise, we're simply running tokenizers in whatever tests comes first. - # tokenizers does NOT provide a method to change the number of threads during - # runtime. - p = Process(target=benchmark_batch, args=(model, documents, num_threads, document_length)) - p.start() - p.join() - - # benchmark_batch(model, documents, num_threads) - - -def main(): - parser = argparse.ArgumentParser( - prog="bench_tokenizer", - description="Getting a feel for speed when tokenizing", - ) - parser.add_argument("-m", "--model", default=MODEL_ID, type=str) - parser.add_argument("-d", "--dataset", default=DATASET, type=str) - parser.add_argument("-ds", "--dataset-config", default=DATASET_CONFIG, type=str) - parser.add_argument("-t", "--threads", nargs="+", default=DEFAULT_THREADS, type=int) - args = parser.parse_args() - test(args.model, args.dataset, args.dataset_config, args.threads) - - -# Call the function to run the benchmark -if __name__ == "__main__": - main() diff --git a/bindings/python-pipeline/clippy.toml b/bindings/python/clippy.toml similarity index 100% rename from bindings/python-pipeline/clippy.toml rename to bindings/python/clippy.toml diff --git a/bindings/python/conftest.py b/bindings/python/conftest.py deleted file mode 100644 index 3178a8fd5..000000000 --- a/bindings/python/conftest.py +++ /dev/null @@ -1,19 +0,0 @@ -import pytest - - -def pytest_addoption(parser): - parser.addoption("--runslow", action="store_true", default=False, help="run slow tests") - - -def pytest_configure(config): - config.addinivalue_line("markers", "slow: mark test as slow to run") - - -def pytest_collection_modifyitems(config, items): - if config.getoption("--runslow"): - # --runslow given in cli: do not skip slow tests - return - skip_slow = pytest.mark.skip(reason="need --runslow option to run") - for item in items: - if "slow" in item.keywords: - item.add_marker(skip_slow) diff --git a/bindings/python/docs/pyo3.md b/bindings/python/docs/pyo3.md deleted file mode 100644 index 1bc8f9bd6..000000000 --- a/bindings/python/docs/pyo3.md +++ /dev/null @@ -1,7 +0,0 @@ -# PyO3 Usage Notes - -## Why we take `self_: PyRef<'_, Self>` - -Most of the Python-facing structs are declared with `#[pyclass(extends = ...)]`. The actual data (for example the `processor` field in `PyPostProcessor`) lives in the base class, while the derived Rust structs are often just markers so that Python sees a proper subclass. When we implement a method on the subclass, we still need to reach into the base storage without downcasting a `PyAny` or re-wrapping objects. - -Using `self_: PyRef<'_, Self>` gives us a borrowed reference to the Python-owned value that keeps the GIL lifetime, reference counts, and the inheritance chain intact. With it we can call `self_.as_ref()` to view the base `PyPostProcessor` directly and access shared helpers like the processor getters/setters. If we used a plain `&self` we would only see the zero-sized derived struct and would have to convert through a super type just to touch the processors, which adds boilerplate and loses the link to the Python inheritance model. This is the PyO3 equivalent of Python’s `super()`—it keeps the Rust type information while letting us operate on the underlying parent. diff --git a/bindings/python-pipeline/examples/01_train_and_encode.py b/bindings/python/examples/01_train_and_encode.py similarity index 96% rename from bindings/python-pipeline/examples/01_train_and_encode.py rename to bindings/python/examples/01_train_and_encode.py index c7463234e..436a794b9 100644 --- a/bindings/python-pipeline/examples/01_train_and_encode.py +++ b/bindings/python/examples/01_train_and_encode.py @@ -7,7 +7,7 @@ import numpy as np -from tokenizers_pipeline import AddedToken, Tokenizer, models, normalizers, pre_tokenizers, trainers +from tokenizers import AddedToken, Tokenizer, models, normalizers, pre_tokenizers, trainers DATA = Path(__file__).resolve().parents[3] / "tokenizers" / "data" diff --git a/bindings/python-pipeline/examples/02_pretrained.py b/bindings/python/examples/02_pretrained.py similarity index 59% rename from bindings/python-pipeline/examples/02_pretrained.py rename to bindings/python/examples/02_pretrained.py index 3eccd15d6..78a57cbe2 100644 --- a/bindings/python-pipeline/examples/02_pretrained.py +++ b/bindings/python/examples/02_pretrained.py @@ -1,12 +1,14 @@ -"""Load real tokenizer.json files and check id parity against the released -`tokenizers` package on a real corpus. Also demonstrates the two loud failure -modes: unsupported pre-tokenizers and unwired post-processing.""" +"""Load real tokenizer.json files and encode a real corpus. Also demonstrates +the two loud failure modes: unsupported pre-tokenizers and unwired +post-processing. (Id parity against the released wheel is checked by +benches/bench_vs_release.py — the released package shares our name, so the +comparison needs two processes.)""" from pathlib import Path -import tokenizers as reference +import numpy as np -from tokenizers_pipeline import Tokenizer, TokenizersError +from tokenizers import Tokenizer, TokenizersError DATA = Path(__file__).resolve().parents[3] / "tokenizers" / "data" @@ -21,16 +23,12 @@ ("bert-base-uncased", "bert-base-uncased.json"), ]: tok = Tokenizer.from_file(DATA / file) - ref = reference.Tokenizer.from_file(str(DATA / file)) - - ours = tok.encode_batch(LINES, add_special_tokens=False) - theirs = ref.encode_batch_fast(LINES, add_special_tokens=False) - mismatches = sum( - 1 for a, b in zip(ours, theirs, strict=True) if a.tolist() != b.ids - ) - total = sum(len(a) for a in ours) - assert mismatches == 0, f"{name}: {mismatches} mismatching lines" - print(f"{name}: {total} tokens, ids identical to `tokenizers` {reference.__version__}") + batch = tok.encode_batch(LINES, add_special_tokens=False) + assert all(ids.dtype == np.uint32 for ids in batch) + total = sum(len(ids) for ids in batch) + round_trip = tok.id_to_token(int(batch[0][0])) + assert round_trip is not None + print(f"{name}: {total} tokens, first token {round_trip!r}") # Expected failure 1: post-processor would add special tokens -> loud error, # not silently wrong ids diff --git a/bindings/python-pipeline/examples/03_threading.py b/bindings/python/examples/03_threading.py similarity index 98% rename from bindings/python-pipeline/examples/03_threading.py rename to bindings/python/examples/03_threading.py index cc12166f2..cbe5a8b74 100644 --- a/bindings/python-pipeline/examples/03_threading.py +++ b/bindings/python/examples/03_threading.py @@ -6,7 +6,7 @@ from concurrent.futures import ThreadPoolExecutor from pathlib import Path -from tokenizers_pipeline import Tokenizer +from tokenizers import Tokenizer DATA = Path(__file__).resolve().parents[3] / "tokenizers" / "data" N_THREADS = 4 diff --git a/bindings/python/examples/custom_components.py b/bindings/python/examples/custom_components.py deleted file mode 100644 index 10875cdeb..000000000 --- a/bindings/python/examples/custom_components.py +++ /dev/null @@ -1,79 +0,0 @@ -from typing import List - -import jieba -from tokenizers import NormalizedString, PreTokenizedString, Regex, Tokenizer -from tokenizers.decoders import Decoder -from tokenizers.models import BPE -from tokenizers.normalizers import Normalizer -from tokenizers.pre_tokenizers import PreTokenizer - - -class JiebaPreTokenizer: - def jieba_split(self, i: int, normalized_string: NormalizedString) -> List[NormalizedString]: - splits = [] - # we need to call `str(normalized_string)` because jieba expects a str, - # not a NormalizedString - for token, start, stop in jieba.tokenize(str(normalized_string)): - splits.append(normalized_string[start:stop]) - - return splits - # We can also easily do it in one line: - # return [normalized_string[w[1] : w[2]] for w in jieba.tokenize(str(normalized_string))] - - def odd_number_split(self, i: int, normalized_string: NormalizedString) -> List[NormalizedString]: - # Just an odd example... - splits = [] - last = 0 - for i, char in enumerate(str(normalized_string)): - if char.isnumeric() and int(char) % 2 == 1: - splits.append(normalized_string[last:i]) - last = i - # Don't forget the last one - splits.append(normalized_string[last:]) - return splits - - def pre_tokenize(self, pretok: PreTokenizedString): - # Let's call split on the PreTokenizedString to split using `self.jieba_split` - pretok.split(self.jieba_split) - # Here we can call `pretok.split` multiple times if we want to apply - # different algorithm, but we generally just need to call it once. - pretok.split(self.odd_number_split) - - -class CustomDecoder: - def decode(self, tokens: List[str]) -> str: - return "".join(tokens) - - -class CustomNormalizer: - def normalize(self, normalized: NormalizedString): - # Most of these can be replaced by a `Sequence` combining some provided Normalizer, - # (ie Sequence([ NFKC(), Replace(Regex("\s+"), " "), Lowercase() ]) - # and it should be the preferred way. That being said, here is an example of the kind - # of things that can be done here: - normalized.nfkc() - normalized.filter(lambda char: not char.isnumeric()) - normalized.replace(Regex("\s+"), " ") - normalized.lowercase() - - -# This section shows how to attach these custom components to the Tokenizer -tok = Tokenizer(BPE()) -tok.normalizer = Normalizer.custom(CustomNormalizer()) -tok.pre_tokenizer = PreTokenizer.custom(JiebaPreTokenizer()) -tok.decoder = Decoder.custom(CustomDecoder()) - -input = "永和服装饰品有限公司" -print("PreTokenize:", input) -print(tok.pre_tokenizer.pre_tokenize_str(input)) -# [('永和', (0, 2)), ('服装', (2, 4)), ('饰品', (4, 6)), ('有限公司', (6, 10))] - -input = "112233" -print("PreTokenize:", input) -print(tok.pre_tokenizer.pre_tokenize_str(input)) -# [('1', (0, 1)), ('122', (1, 4)), ('3', (4, 5)), ('3', (5, 6))] - -input = "1234 ℌ𝔢𝔩𝔩𝔬 𝔱𝔥𝔢𝔯𝔢 𝓂𝓎 𝒹ℯ𝒶𝓇 𝕕𝕖𝕒𝕣 𝕗𝕣𝕚𝕖𝕟𝕕!" -print("Normalize:", input) -print(tok.normalizer.normalize_str(input)) -# " hello there my dear dear friend!" diff --git a/bindings/python/examples/example.py b/bindings/python/examples/example.py deleted file mode 100644 index d165d34ba..000000000 --- a/bindings/python/examples/example.py +++ /dev/null @@ -1,138 +0,0 @@ -import argparse -import logging -import time - -from tqdm import tqdm - -from tokenizers import Tokenizer, decoders, pre_tokenizers -from tokenizers.models import BPE, WordPiece -from tokenizers.normalizers import BertNormalizer -from tokenizers.processors import BertProcessing -from transformers import BertTokenizer, GPT2Tokenizer # type: ignore[import] - -logging.getLogger("transformers").disabled = True -logging.getLogger("transformers.tokenization_utils").disabled = True - - -parser = argparse.ArgumentParser() -parser.add_argument("--type", default="gpt2", type=str, help="The type of tokenizer (bert|gpt2)") -parser.add_argument("--file", default=None, type=str, help="The file to encode") -parser.add_argument("--vocab", default=None, type=str, required=True, help="The vocab file") -parser.add_argument("--merges", default=None, type=str, help="The merges.txt file") -parser.add_argument("--debug", action="store_true", help="Verbose output") -args = parser.parse_args() - -if args.type == "gpt2" and args.merges is None: - raise Exception("Expected merges.txt file") - -if args.file is not None: - with open(args.file, "r") as fp: - text = [line.strip() for line in fp] -else: - text = """ -The Zen of Python, by Tim Peters -Beautiful is better than ugly. -Explicit is better than implicit. -Simple is better than complex. -Complex is better than complicated. -Flat is better than nested. -Sparse is better than dense. -Readability counts. -Special cases aren't special enough to break the rules. -Although practicality beats purity. -Errors should never pass silently. -Unless explicitly silenced. -In the face of ambiguity, refuse the temptation to guess. -There should be one-- and preferably only one --obvious way to do it. -Although that way may not be obvious at first unless you're Dutch. -Now is better than never. -Although never is often better than *right* now. -If the implementation is hard to explain, it's a bad idea. -If the implementation is easy to explain, it may be a good idea. -Namespaces are one honking great idea -- let's do more of those! -""".split("\n") - -if args.type == "gpt2": - print("Running GPT-2 tokenizer") - tok_p = GPT2Tokenizer.from_pretrained("gpt2") - - # Create a Tokenizer using BPE - tok_r = Tokenizer(BPE(args.vocab, args.merges)) - # Use ByteLevel PreTokenizer - tok_r.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False) - # Use ByteLevel Decoder - tok_r.decoder = decoders.ByteLevel() -elif args.type == "bert": - print("Running Bert tokenizer") - tok_p = BertTokenizer.from_pretrained(args.vocab) - - tok_r = Tokenizer(WordPiece(args.vocab, unk_token="[UNK]", max_input_chars_per_word=100)) - tok_r.normalizer = BertNormalizer( - clean_text=True, - handle_chinese_chars=True, - strip_accents=True, - lowercase=True, - ) - # tok_r.pre_tokenizer = pre_tokenizers.Whitespace() - tok_r.pre_tokenizer = pre_tokenizers.BertPreTokenizer() - tok_r.decoder = decoders.WordPiece() - tok_r.post_processor = BertProcessing( - ("[SEP]", tok_r.token_to_id("[SEP]")), - ("[CLS]", tok_r.token_to_id("[CLS]")), - ) -else: - raise Exception(f"Unknown type {args.type}") - - -def tokenize_r(): - return tok_r.encode_batch(text) - - -def tokenize_p(): - return [tok_p.encode(sentence, add_special_tokens=True) for sentence in tqdm(text)] - - -print(f"Tokenizing {len(text)} lines") - -# Rust version -start = time.time() -encoded_r = tokenize_r() -end = time.time() -time_r = end - start -print(f"Rust tokenizer took: {time_r} sec") - -# Python version -start = time.time() -encoded_p = tokenize_p() -end = time.time() -time_p = end - start -print(f"Transformer tokenizer took: {time_p} sec") - -print(f"SpeedUp Ratio: {time_p / time_r}") - -ids_r = [sentence.ids for sentence in encoded_r] -diff_ids = 0 -for i in range(0, len(encoded_r)): - if encoded_r[i].ids != encoded_p[i]: - diff_ids += 1 - if args.debug: - print(encoded_r[i].ids) - print(encoded_p[i]) - print(encoded_r[i].tokens) - print(tok_p.tokenize(text[i])) - print(text[i]) - print("") -print(f"Ids differences: {diff_ids}") - -decoded_r = tok_r.decode_batch([sentence.ids for sentence in encoded_r], False) -decoded_p = [tok_p.decode(en) for en in encoded_p] -diff_decoded = 0 -for i in range(0, len(text)): - if decoded_r[i] != decoded_p[i]: - diff_decoded += 1 - if args.debug: - print(f"Original: {text[i]}") - print(f"Rust: {decoded_r[i]}") - print(f"Python: {decoded_p[i]}") - print("") -print(f"Decoding differences: {diff_decoded}") diff --git a/bindings/python/examples/train_bert_wordpiece.py b/bindings/python/examples/train_bert_wordpiece.py deleted file mode 100644 index b6be85d61..000000000 --- a/bindings/python/examples/train_bert_wordpiece.py +++ /dev/null @@ -1,52 +0,0 @@ -import argparse -import glob - -from tokenizers import BertWordPieceTokenizer - - -parser = argparse.ArgumentParser() -parser.add_argument( - "--files", - default=None, - metavar="path", - type=str, - required=True, - help="The files to use as training; accept '**/*.txt' type of patterns \ - if enclosed in quotes", -) -parser.add_argument( - "--out", - default="./", - type=str, - help="Path to the output directory, where the files will be saved", -) -parser.add_argument("--name", default="bert-wordpiece", type=str, help="The name of the output vocab files") -args = parser.parse_args() - -files = glob.glob(args.files) -if not files: - print(f"File does not exist: {args.files}") - exit(1) - - -# Initialize an empty tokenizer -tokenizer = BertWordPieceTokenizer( - clean_text=True, - handle_chinese_chars=True, - strip_accents=True, - lowercase=True, -) - -# And then train -tokenizer.train( - files, - vocab_size=10000, - min_frequency=2, - show_progress=True, - special_tokens=["[PAD]", "[UNK]", "[CLS]", "[SEP]", "[MASK]"], - limit_alphabet=1000, - wordpieces_prefix="##", -) - -# Save the files -tokenizer.save_model(args.out, args.name) diff --git a/bindings/python/examples/train_bytelevel_bpe.py b/bindings/python/examples/train_bytelevel_bpe.py deleted file mode 100644 index 728550d93..000000000 --- a/bindings/python/examples/train_bytelevel_bpe.py +++ /dev/null @@ -1,56 +0,0 @@ -import argparse -import glob -from os.path import join - -from tokenizers import ByteLevelBPETokenizer - - -parser = argparse.ArgumentParser() -parser.add_argument( - "--files", - default=None, - metavar="path", - type=str, - required=True, - help="The files to use as training; accept '**/*.txt' type of patterns \ - if enclosed in quotes", -) -parser.add_argument( - "--out", - default="./", - type=str, - help="Path to the output directory, where the files will be saved", -) -parser.add_argument("--name", default="bpe-bytelevel", type=str, help="The name of the output vocab files") -args = parser.parse_args() - -files = glob.glob(args.files) -if not files: - print(f"File does not exist: {args.files}") - exit(1) - - -# Initialize an empty tokenizer -tokenizer = ByteLevelBPETokenizer(add_prefix_space=True) - -# And then train -tokenizer.train( - files, - vocab_size=10000, - min_frequency=2, - show_progress=True, - special_tokens=["", "", ""], -) - -# Save the files -tokenizer.save_model(args.out, args.name) - -# Restoring model from learned vocab/merges -tokenizer = ByteLevelBPETokenizer( - join(args.out, "{}-vocab.json".format(args.name)), - join(args.out, "{}-merges.txt".format(args.name)), - add_prefix_space=True, -) - -# Test encoding -print(tokenizer.encode("Training ByteLevel BPE is very easy").tokens) diff --git a/bindings/python/examples/train_with_datasets.py b/bindings/python/examples/train_with_datasets.py deleted file mode 100644 index 956c8542d..000000000 --- a/bindings/python/examples/train_with_datasets.py +++ /dev/null @@ -1,23 +0,0 @@ -import datasets - -from tokenizers import Tokenizer, models, normalizers, pre_tokenizers - - -# Build a tokenizer -bpe_tokenizer = Tokenizer(models.BPE()) -bpe_tokenizer.pre_tokenizer = pre_tokenizers.Whitespace() -bpe_tokenizer.normalizer = normalizers.Lowercase() - -# Initialize a dataset -dataset = datasets.load_dataset("wikitext", "wikitext-103-raw-v1", split="train") - - -# Build an iterator over this dataset -def batch_iterator(): - batch_size = 1000 - for batch in dataset.iter(batch_size=batch_size): # type: ignore[attr-defined] - yield batch["text"] - - -# And finally train -bpe_tokenizer.train_from_iterator(batch_iterator(), length=len(dataset)) # type: ignore[arg-type] diff --git a/bindings/python/examples/using_the_visualizer.ipynb b/bindings/python/examples/using_the_visualizer.ipynb deleted file mode 100644 index 61d6fb845..000000000 --- a/bindings/python/examples/using_the_visualizer.ipynb +++ /dev/null @@ -1,1056 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "--2020-12-04 09:25:00-- https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt\n", - "Loaded CA certificate '/etc/ssl/certs/ca-certificates.crt'\n", - "Resolving s3.amazonaws.com (s3.amazonaws.com)... 52.216.104.253\n", - "Connecting to s3.amazonaws.com (s3.amazonaws.com)|52.216.104.253|:443... connected.\n", - "HTTP request sent, awaiting response... 200 OK\n", - "Length: 231508 (226K) [text/plain]\n", - "Saving to: ‘/tmp/bert-base-uncased-vocab.txt’\n", - "\n", - "/tmp/bert-base-unca 100%[===================>] 226.08K --.-KB/s in 0.06s \n", - "\n", - "2020-12-04 09:25:00 (3.87 MB/s) - ‘/tmp/bert-base-uncased-vocab.txt’ saved [231508/231508]\n", - "\n" - ] - } - ], - "source": [ - "!wget https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt -O /tmp/bert-base-uncased-vocab.txt" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "from tokenizers import BertWordPieceTokenizer\n", - "from tokenizers.tools import EncodingVisualizer" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "EncodingVisualizer.unk_token_regex.search(\"aaa[udsnk]aaa\")" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "text = \"\"\"Mathias Bynens 'Z͑ͫ̓ͪ̂ͫ̽͏̴̙̤̞͉͚̯̞̠͍A̴̵̜̰͔ͫ͗͢L̠ͨͧͩ͘G̴̻͈͍͔̹̑͗̎̅͛́Ǫ̵̹̻̝̳͂̌̌͘!͖̬̰̙̗̿̋ͥͥ̂ͣ̐́́͜͞': Whenever you’re working on a piece of JavaScript code that deals with strings or regular expressions in some way, just add a unit test that contains a pile of poo (💩) in a string, 💩💩💩💩💩💩💩💩💩💩💩💩 and see if anything breaks. It’s a quick, fun, and easy way to see if your code supports astral symbols. Once you’ve found a Unicode-related bug in your code, all you need to do is apply the techniques discussed in this post to fix it.\"\"\"" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "tokenizer = BertWordPieceTokenizer(\"/tmp/bert-base-uncased-vocab.txt\", lowercase=True)\n", - "visualizer = EncodingVisualizer(tokenizer=tokenizer)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Visualizing Tokens With No Annotations" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
\n", - " Mathias Bynens 'Z͑ͫ̓ͪ̂ͫ̽͏̴̙̤̞͉͚̯̞̠͍A̴̵̜̰͔ͫ͗͢L̠ͨͧͩ͘G̴̻͈͍͔̹̑͗̎̅͛́Ǫ̵̹̻̝̳͂̌̌͘!͖̬̰̙̗̿̋ͥͥ̂ͣ̐́́͜͞': Whenever you’re working on a piece of JavaScript code that deals with strings or regular expressions in some way, just add a unit test that contains a pile of poo (💩) in a string, 💩💩💩💩💩💩💩💩💩💩💩💩 and see if anything breaks. It’s a quick, fun, and easy way to see if your code supports astral symbols. Once you’ve found a Unicode-related bug in your code, all you need to do is apply the techniques discussed in this post to fix it.\n", - "
\n", - " \n", - " \n", - " " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "visualizer(text)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Visualizing Tokens With Aligned Annotations\n", - "First we make some annotations with the Annotation class" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "from tokenizers.tools import Annotation" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "anno1 = Annotation(start=0, end=2, label=\"foo\")\n", - "anno2 = Annotation(start=2, end=4, label=\"bar\")\n", - "anno3 = Annotation(start=6, end=8, label=\"poo\")\n", - "anno4 = Annotation(start=9, end=12, label=\"shoe\")\n", - "annotations = [\n", - " anno1,\n", - " anno2,\n", - " anno3,\n", - " anno4,\n", - " Annotation(start=23, end=30, label=\"random tandem bandem sandem landem fandom\"),\n", - " Annotation(start=63, end=70, label=\"foo\"),\n", - " Annotation(start=80, end=95, label=\"bar\"),\n", - " Annotation(start=120, end=128, label=\"bar\"),\n", - " Annotation(start=152, end=155, label=\"poo\"),\n", - "]" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
\n", - " Mathias Bynens 'Z͑ͫ̓ͪ̂ͫ̽͏̴̙̤̞͉͚̯̞̠͍A̴̵̜̰͔ͫ͗͢L̠ͨͧͩ͘G̴̻͈͍͔̹̑͗̎̅͛́Ǫ̵̹̻̝̳͂̌̌͘!͖̬̰̙̗̿̋ͥͥ̂ͣ̐́́͜͞': Whenever you’re working on a piece of JavaScript code that deals with strings or regular expressions in some way, just add a unit test that contains a pile of poo (💩) in a string, 💩💩💩💩💩💩💩💩💩💩💩💩 and see if anything breaks. It’s a quick, fun, and easy way to see if your code supports astral symbols. Once you’ve found a Unicode-related bug in your code, all you need to do is apply the techniques discussed in this post to fix it.\n", - "
\n", - " \n", - " \n", - " " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "visualizer(text, annotations=annotations)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using A Custom Annotation Format\n", - "Every system has its own representation of annotations. That's why we can instantiate the EncodingVisualizer with a convertion function." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[{'startPlace': 0, 'endPlace': 3, 'theTag': '0'},\n", - " {'startPlace': 4, 'endPlace': 7, 'theTag': '4'},\n", - " {'startPlace': 8, 'endPlace': 11, 'theTag': '8'},\n", - " {'startPlace': 12, 'endPlace': 15, 'theTag': '12'},\n", - " {'startPlace': 16, 'endPlace': 19, 'theTag': '16'}]" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "funnyAnnotations = [dict(startPlace=i, endPlace=i + 3, theTag=str(i)) for i in range(0, 20, 4)]\n", - "funnyAnnotations" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "def converter(funny):\n", - " return Annotation(start=funny[\"startPlace\"], end=funny[\"endPlace\"], label=funny[\"theTag\"])\n", - "\n", - "\n", - "visualizer = EncodingVisualizer(tokenizer=tokenizer, default_to_notebook=True, annotation_converter=converter)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
\n", - " Mathias Bynens 'Z͑ͫ̓ͪ̂ͫ̽͏̴̙̤̞͉͚̯̞̠͍A̴̵̜̰͔ͫ͗͢L̠ͨͧͩ͘G̴̻͈͍͔̹̑͗̎̅͛́Ǫ̵̹̻̝̳͂̌̌͘!͖̬̰̙̗̿̋ͥͥ̂ͣ̐́́͜͞': Whenever you’re working on a piece of JavaScript code that deals with strings or regular expressions in some way, just add a unit test that contains a pile of poo (💩) in a string, 💩💩💩💩💩💩💩💩💩💩💩💩 and see if anything breaks. It’s a quick, fun, and easy way to see if your code supports astral symbols. Once you’ve found a Unicode-related bug in your code, all you need to do is apply the techniques discussed in this post to fix it.\n", - "
\n", - " \n", - " \n", - " " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "visualizer(text, annotations=funnyAnnotations)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Trying with Roberta\n" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "--2020-12-04 09:25:00-- https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json\n", - "Loaded CA certificate '/etc/ssl/certs/ca-certificates.crt'\n", - "Resolving s3.amazonaws.com (s3.amazonaws.com)... 52.216.226.19\n", - "Connecting to s3.amazonaws.com (s3.amazonaws.com)|52.216.226.19|:443... connected.\n", - "HTTP request sent, awaiting response... 200 OK\n", - "Length: 898823 (878K) [application/json]\n", - "Saving to: ‘/tmp/roberta-base-vocab.json’\n", - "\n", - "/tmp/roberta-base-v 100%[===================>] 877.76K 4.35MB/s in 0.2s \n", - "\n", - "2020-12-04 09:25:00 (4.35 MB/s) - ‘/tmp/roberta-base-vocab.json’ saved [898823/898823]\n", - "\n", - "--2020-12-04 09:25:00-- https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-merges.txt\n", - "Loaded CA certificate '/etc/ssl/certs/ca-certificates.crt'\n", - "Resolving s3.amazonaws.com (s3.amazonaws.com)... 52.216.104.253\n", - "Connecting to s3.amazonaws.com (s3.amazonaws.com)|52.216.104.253|:443... connected.\n", - "HTTP request sent, awaiting response... 200 OK\n", - "Length: 456318 (446K) [text/plain]\n", - "Saving to: ‘/tmp/roberta-base-merges.txt’\n", - "\n", - "/tmp/roberta-base-m 100%[===================>] 445.62K --.-KB/s in 0.1s \n", - "\n", - "2020-12-04 09:25:01 (4.04 MB/s) - ‘/tmp/roberta-base-merges.txt’ saved [456318/456318]\n", - "\n" - ] - } - ], - "source": [ - "!wget \"https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json\" -O /tmp/roberta-base-vocab.json\n", - "!wget \"https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-merges.txt\" -O /tmp/roberta-base-merges.txt" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
\n", - " Mathias Bynens 'Z͑ͫ̓ͪ̂ͫ̽͏̴̙̤̞͉͚̯̞̠͍A̴̵̜̰͔ͫ͗͢L̠ͨͧͩ͘G̴̻͈͍͔̹̑͗̎̅͛́Ǫ̵̹̻̝̳͂̌̌͘!͖̬̰̙̗̿̋ͥͥ̂ͣ̐́́͜͞': Whenever you’re working on a piece of JavaScript code that deals with strings or regular expressions in some way, just add a unit test that contains a pile of poo (💩) in a string, 💩💩💩💩💩💩💩💩💩💩💩💩 and see if anything breaks. It’s a quick, fun, and easy way to see if your code supports astral symbols. Once you’ve found a Unicode-related bug in your code, all you need to do is apply the techniques discussed in this post to fix it.\n", - "
\n", - " \n", - " \n", - " " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "from tokenizers import ByteLevelBPETokenizer\n", - "\n", - "roberta_tokenizer = ByteLevelBPETokenizer.from_file(\"/tmp/roberta-base-vocab.json\", \"/tmp/roberta-base-merges.txt\")\n", - "roberta_visualizer = EncodingVisualizer(tokenizer=roberta_tokenizer, default_to_notebook=True)\n", - "roberta_visualizer(text, annotations=annotations)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/bindings/python/py_src/tokenizers/__init__.py b/bindings/python/py_src/tokenizers/__init__.py index 07e1e85be..003333d64 100644 --- a/bindings/python/py_src/tokenizers/__init__.py +++ b/bindings/python/py_src/tokenizers/__init__.py @@ -1,111 +1,20 @@ -"""Tokenizers — fast, batteries-included tokenization library. +"""Fast tokenizers built on the pipeline encode path. Start with `Tokenizer`.""" -Free-threaded Python (3.14t) note: - Wheels built against free-threaded CPython declare ``Py_MOD_GIL_NOT_USED`` - and use ``RwLock``-guarded interior mutability so component setters are - safe to call from multiple threads. Compound mutations - (``tokenizer.post_processor.special_tokens = …``) are still not atomic — - use a Python lock if you need the read-then-write to be serialized. - See ``docs/free-threading-audit.md`` for the full analysis. -""" - -from enum import Enum -from typing import List, Tuple, Union - - -Offsets = Tuple[int, int] - -TextInputSequence = str -"""A :obj:`str` that represents an input sequence """ - -PreTokenizedInputSequence = Union[List[str], Tuple[str]] -"""A pre-tokenized input sequence. Can be one of: - - - A :obj:`List` of :obj:`str` - - A :obj:`Tuple` of :obj:`str` -""" - -TextEncodeInput = Union[ - TextInputSequence, - Tuple[TextInputSequence, TextInputSequence], - List[TextInputSequence], -] -"""Represents a textual input for encoding. Can be either: - - - A single sequence: :data:`~tokenizers.TextInputSequence` - - A pair of sequences: - - - A :obj:`Tuple` of :data:`~tokenizers.TextInputSequence` - - Or a :obj:`List` of :data:`~tokenizers.TextInputSequence` of size 2 -""" - -PreTokenizedEncodeInput = Union[ - PreTokenizedInputSequence, - Tuple[PreTokenizedInputSequence, PreTokenizedInputSequence], - List[PreTokenizedInputSequence], -] -"""Represents a pre-tokenized input for encoding. Can be either: - - - A single sequence: :data:`~tokenizers.PreTokenizedInputSequence` - - A pair of sequences: - - - A :obj:`Tuple` of :data:`~tokenizers.PreTokenizedInputSequence` - - Or a :obj:`List` of :data:`~tokenizers.PreTokenizedInputSequence` of size 2 -""" - -InputSequence = Union[TextInputSequence, PreTokenizedInputSequence] -"""Represents all the possible types of input sequences for encoding. Can be: - - - When ``is_pretokenized=False``: :data:`~TextInputSequence` - - When ``is_pretokenized=True``: :data:`~PreTokenizedInputSequence` -""" - -EncodeInput = Union[TextEncodeInput, PreTokenizedEncodeInput] -"""Represents all the possible types of input for encoding. Can be: - - - When ``is_pretokenized=False``: :data:`~TextEncodeInput` - - When ``is_pretokenized=True``: :data:`~PreTokenizedEncodeInput` -""" - - -class OffsetReferential(Enum): - ORIGINAL = "original" - NORMALIZED = "normalized" - - -class OffsetType(Enum): - BYTE = "byte" - CHAR = "char" - - -class SplitDelimiterBehavior(Enum): - REMOVED = "removed" - ISOLATED = "isolated" - MERGED_WITH_PREVIOUS = "merged_with_previous" - MERGED_WITH_NEXT = "merged_with_next" - CONTIGUOUS = "contiguous" - - -from .tokenizers import ( +from ._native import ( AddedToken, - Encoding, - NormalizedString, - PreTokenizedString, - Regex, - Token, Tokenizer, - decoders, - models, - normalizers, - pre_tokenizers, - processors, - trainers, + TokenizersError, __version__, ) -from .implementations import ( - BertWordPieceTokenizer, - ByteLevelBPETokenizer, - CharBPETokenizer, - SentencePieceBPETokenizer, - SentencePieceUnigramTokenizer, -) +from . import models, normalizers, pre_tokenizers, trainers + +__all__ = [ + "AddedToken", + "Tokenizer", + "TokenizersError", + "__version__", + "models", + "normalizers", + "pre_tokenizers", + "trainers", +] diff --git a/bindings/python/py_src/tokenizers/__init__.pyi b/bindings/python/py_src/tokenizers/__init__.pyi index b72d90b91..71fd16c91 100644 --- a/bindings/python/py_src/tokenizers/__init__.pyi +++ b/bindings/python/py_src/tokenizers/__init__.pyi @@ -1,1330 +1,173 @@ +import numpy as np +import numpy.typing as npt + """ -Tokenizers Module +Fast tokenizers built on the pipeline encode path. Start with `Tokenizer`. """ -from collections.abc import Sequence -from typing import Any, Final, final - -from _typeshed import Incomplete - -from tokenizers.decoders import Decoder from tokenizers.models import Model from tokenizers.normalizers import Normalizer from tokenizers.pre_tokenizers import PreTokenizer -from tokenizers.processors import PostProcessor from tokenizers.trainers import Trainer - +from _typeshed import Incomplete +from collections.abc import Sequence +from os import PathLike +from typing import Any, Final, final __version__: Final[str] @final class AddedToken: """ - Represents a token that can be be added to a :class:`~tokenizers.Tokenizer`. - It can have special options that defines the way it should behave. - - Args: - content (:obj:`str`): The content of the token - - single_word (:obj:`bool`, defaults to :obj:`False`): - Defines whether this token should only match single words. If :obj:`True`, this - token will never match inside of a word. For example the token ``ing`` would match - on ``tokenizing`` if this option is :obj:`False`, but not if it is :obj:`True`. - The notion of "`inside of a word`" is defined by the word boundaries pattern in - regular expressions (ie. the token should start and end with word boundaries). - - lstrip (:obj:`bool`, defaults to :obj:`False`): - Defines whether this token should strip all potential whitespaces on its left side. - If :obj:`True`, this token will greedily match any whitespace on its left. For - example if we try to match the token ``[MASK]`` with ``lstrip=True``, in the text - ``"I saw a [MASK]"``, we would match on ``" [MASK]"``. (Note the space on the left). - - rstrip (:obj:`bool`, defaults to :obj:`False`): - Defines whether this token should strip all potential whitespaces on its right - side. If :obj:`True`, this token will greedily match any whitespace on its right. - It works just like :obj:`lstrip` but on the right. - - normalized (:obj:`bool`, defaults to :obj:`True` with :meth:`~tokenizers.Tokenizer.add_tokens` and :obj:`False` with :meth:`~tokenizers.Tokenizer.add_special_tokens`): - Defines whether this token should match against the normalized version of the input - text. For example, with the added token ``"yesterday"``, and a normalizer in charge of - lowercasing the text, the token could be extract from the input ``"I saw a lion - Yesterday"``. - special (:obj:`bool`, defaults to :obj:`False` with :meth:`~tokenizers.Tokenizer.add_tokens` and :obj:`False` with :meth:`~tokenizers.Tokenizer.add_special_tokens`): - Defines whether this token should be skipped when decoding. + A token added to the vocabulary after training, with options for how it + is matched in text: `single_word` only matches when it stands alone (not + inside a word); `lstrip`/`rstrip` also swallow the whitespace before/after + it; `normalized` matches against normalized instead of raw text (defaults + to the opposite of `special`); `special` marks template tokens like "" + that decoding should be able to skip. """ - def __eq__(self, /, other: object) -> bool: ... - def __ge__(self, /, other: object) -> bool: ... - def __getstate__(self, /) -> dict: ... - def __gt__(self, /, other: object) -> bool: ... - def __hash__(self, /) -> int: ... - def __le__(self, /, other: object) -> bool: ... - def __lt__(self, /, other: object) -> bool: ... - def __ne__(self, /, other: object) -> bool: ... - def __new__(cls, /, content: str | None = None, **kwargs) -> AddedToken: ... + def __new__(cls, /, content: str, *, single_word: bool = False, lstrip: bool = False, rstrip: bool = False, normalized: bool |None = None, special: bool = False) -> AddedToken: ... def __repr__(self, /) -> str: ... - def __setstate__(self, /, state: Any) -> None: ... - def __str__(self, /) -> str: ... @property - def content(self, /) -> str: - """ - Get the content of this :obj:`AddedToken` - """ - @content.setter - def content(self, /, content: str) -> None: - """ - Set the content of this :obj:`AddedToken` - """ + def content(self, /) -> str: ... @property - def lstrip(self, /) -> bool: - """ - Get the value of the :obj:`lstrip` option - """ + def lstrip(self, /) -> bool: ... @property - def normalized(self, /) -> bool: - """ - Get the value of the :obj:`normalized` option - """ + def normalized(self, /) -> bool: ... @property - def rstrip(self, /) -> bool: - """ - Get the value of the :obj:`rstrip` option - """ + def rstrip(self, /) -> bool: ... @property - def single_word(self, /) -> bool: - """ - Get the value of the :obj:`single_word` option - """ + def single_word(self, /) -> bool: ... @property - def special(self, /) -> bool: - """ - Get the value of the :obj:`special` option - """ - @special.setter - def special(self, /, special: bool) -> None: - """ - Set the value of the :obj:`special` option - """ - -@final -class Encoding: - """ - The :class:`~tokenizers.Encoding` represents the output of a :class:`~tokenizers.Tokenizer`. - - It holds all the information about the tokenized input, including the token IDs, - token strings, attention masks, offsets, and more. This is the main data structure - returned by :meth:`~tokenizers.Tokenizer.encode` and - :meth:`~tokenizers.Tokenizer.encode_batch`. - - Example:: - - >>> from tokenizers import Tokenizer - >>> tokenizer = Tokenizer.from_pretrained("bert-base-uncased") - >>> encoding = tokenizer.encode("Hello, world!") - >>> encoding.ids - [101, 7592, 1010, 2088, 999, 102] - >>> encoding.tokens - ['[CLS]', 'hello', ',', 'world', '!', '[SEP]'] - >>> encoding.offsets - [(0, 0), (0, 5), (5, 6), (7, 12), (12, 13), (0, 0)] - """ - def __getstate__(self, /) -> Any: ... - def __len__(self, /) -> int: ... - def __new__(cls, /) -> Encoding: ... - def __repr__(self, /) -> str: ... - def __setstate__(self, /, state: Any) -> None: ... - @property - def attention_mask(self, /) -> list[int]: - """ - The attention mask - - This indicates to the LM which tokens should be attended to, and which should not. - This is especially important when batching sequences, where we need to applying - padding. - - Returns: - :obj:`List[int]`: The attention mask - """ - def char_to_token(self, /, char_pos: int, sequence_index: int = 0) -> int | None: - """ - Get the token that contains the char at the given position in the input sequence. - - Args: - char_pos (:obj:`int`): - The position of a char in the input string - sequence_index (:obj:`int`, defaults to :obj:`0`): - The index of the sequence that contains the target char - - Returns: - :obj:`int`: The index of the token that contains this char in the encoded sequence - """ - def char_to_word(self, /, char_pos: int, sequence_index: int = 0) -> int | None: - """ - Get the word that contains the char at the given position in the input sequence. - - Args: - char_pos (:obj:`int`): - The position of a char in the input string - sequence_index (:obj:`int`, defaults to :obj:`0`): - The index of the sequence that contains the target char - - Returns: - :obj:`int`: The index of the word that contains this char in the input sequence - """ - @property - def ids(self, /) -> list[int]: - """ - The generated IDs - - The IDs are the main input to a Language Model. They are the token indices, - the numerical representations that a LM understands. - - Returns: - :obj:`List[int]`: The list of IDs - """ - @staticmethod - def merge(encodings: Sequence[Encoding], growing_offsets: bool = True) -> "Encoding": - """ - Merge the list of encodings into one final :class:`~tokenizers.Encoding` - - Args: - encodings (A :obj:`List` of :class:`~tokenizers.Encoding`): - The list of encodings that should be merged in one - - growing_offsets (:obj:`bool`, defaults to :obj:`True`): - Whether the offsets should accumulate while merging - - Returns: - :class:`~tokenizers.Encoding`: The resulting Encoding - """ - @property - def n_sequences(self, /) -> int: - """ - The number of sequences represented - - Returns: - :obj:`int`: The number of sequences in this :class:`~tokenizers.Encoding` - """ - @property - def offsets(self, /) -> list[tuple[int, int]]: - """ - The offsets associated to each token - - These offsets let's you slice the input string, and thus retrieve the original - part that led to producing the corresponding token. - - Returns: - A :obj:`List` of :obj:`Tuple[int, int]`: The list of offsets - """ - @property - def overflowing(self, /) -> list[Encoding]: - """ - A :obj:`List` of overflowing :class:`~tokenizers.Encoding` - - When using truncation, the :class:`~tokenizers.Tokenizer` takes care of splitting - the output into as many pieces as required to match the specified maximum length. - This field lets you retrieve all the subsequent pieces. - - When you use pairs of sequences, the overflowing pieces will contain enough - variations to cover all the possible combinations, while respecting the provided - maximum length. - """ - def pad(self, /, length: int, **kwargs) -> "None": - """ - Pad the :class:`~tokenizers.Encoding` at the given length - - Args: - length (:obj:`int`): - The desired length - - direction: (:obj:`str`, defaults to :obj:`right`): - The expected padding direction. Can be either :obj:`right` or :obj:`left` - - pad_id (:obj:`int`, defaults to :obj:`0`): - The ID corresponding to the padding token - - pad_type_id (:obj:`int`, defaults to :obj:`0`): - The type ID corresponding to the padding token - - pad_token (:obj:`str`, defaults to `[PAD]`): - The pad token to use - """ - @property - def sequence_ids(self, /) -> list[int | None]: - """ - The generated sequence indices. - - They represent the index of the input sequence associated to each token. - The sequence id can be None if the token is not related to any input sequence, - like for example with special tokens. - - Returns: - A :obj:`List` of :obj:`Optional[int]`: A list of optional sequence index. - """ - def set_sequence_id(self, /, sequence_id: int) -> None: - """ - Set the given sequence index - - Set the given sequence index for the whole range of tokens contained in this - :class:`~tokenizers.Encoding`. - """ - @property - def special_tokens_mask(self, /) -> list[int]: - """ - The special token mask - - This indicates which tokens are special tokens, and which are not. - - Returns: - :obj:`List[int]`: The special tokens mask - """ - def token_to_chars(self, /, token_index: int) -> tuple[int, int] | None: - """ - Get the offsets of the token at the given index. - - The returned offsets are related to the input sequence that contains the - token. In order to determine in which input sequence it belongs, you - must call :meth:`~tokenizers.Encoding.token_to_sequence()`. - - Args: - token_index (:obj:`int`): - The index of a token in the encoded sequence. - - Returns: - :obj:`Tuple[int, int]`: The token offsets :obj:`(first, last + 1)` - """ - def token_to_sequence(self, /, token_index: int) -> int | None: - """ - Get the index of the sequence represented by the given token. - - In the general use case, this method returns :obj:`0` for a single sequence or - the first sequence of a pair, and :obj:`1` for the second sequence of a pair - - Args: - token_index (:obj:`int`): - The index of a token in the encoded sequence. - - Returns: - :obj:`int`: The sequence id of the given token - """ - def token_to_word(self, /, token_index: int) -> int | None: - """ - Get the index of the word that contains the token in one of the input sequences. - - The returned word index is related to the input sequence that contains - the token. In order to determine in which input sequence it belongs, you - must call :meth:`~tokenizers.Encoding.token_to_sequence()`. - - Args: - token_index (:obj:`int`): - The index of a token in the encoded sequence. - - Returns: - :obj:`int`: The index of the word in the relevant input sequence. - """ - @property - def tokens(self, /) -> list[str]: - """ - The generated tokens - - They are the string representation of the IDs. - - Returns: - :obj:`List[str]`: The list of tokens - """ - def truncate(self, /, max_length: int, stride: int = 0, direction: str = "right") -> "None": - """ - Truncate the :class:`~tokenizers.Encoding` at the given length - - If this :class:`~tokenizers.Encoding` represents multiple sequences, when truncating - this information is lost. It will be considered as representing a single sequence. - - Args: - max_length (:obj:`int`): - The desired length - - stride (:obj:`int`, defaults to :obj:`0`): - The length of previous content to be included in each overflowing piece - - direction (:obj:`str`, defaults to :obj:`right`): - Truncate direction - """ - @property - def type_ids(self, /) -> list[int]: - """ - The generated type IDs - - Generally used for tasks like sequence classification or question answering, - these tokens let the LM know which input sequence corresponds to each tokens. - - Returns: - :obj:`List[int]`: The list of type ids - """ - @property - def word_ids(self, /) -> list[int | None]: - """ - The generated word indices. - - They represent the index of the word associated to each token. - When the input is pre-tokenized, they correspond to the ID of the given input label, - otherwise they correspond to the words indices as defined by the - :class:`~tokenizers.pre_tokenizers.PreTokenizer` that was used. - - For special tokens and such (any token that was generated from something that was - not part of the input), the output is :obj:`None` - - Returns: - A :obj:`List` of :obj:`Optional[int]`: A list of optional word index. - """ - def word_to_chars(self, /, word_index: int, sequence_index: int = 0) -> tuple[int, int] | None: - """ - Get the offsets of the word at the given index in one of the input sequences. - - Args: - word_index (:obj:`int`): - The index of a word in one of the input sequences. - sequence_index (:obj:`int`, defaults to :obj:`0`): - The index of the sequence that contains the target word - - Returns: - :obj:`Tuple[int, int]`: The range of characters (span) :obj:`(first, last + 1)` - """ - def word_to_tokens(self, /, word_index: int, sequence_index: int = 0) -> tuple[int, int] | None: - """ - Get the encoded tokens corresponding to the word at the given index - in one of the input sequences. - - Args: - word_index (:obj:`int`): - The index of a word in one of the input sequences. - sequence_index (:obj:`int`, defaults to :obj:`0`): - The index of the sequence that contains the target word - - Returns: - :obj:`Tuple[int, int]`: The range of tokens: :obj:`(first, last + 1)` - """ - @property - def words(self, /) -> list[int | None]: - """ - The generated word indices. - - .. warning:: - This is deprecated and will be removed in a future version. - Please use :obj:`~tokenizers.Encoding.word_ids` instead. - - They represent the index of the word associated to each token. - When the input is pre-tokenized, they correspond to the ID of the given input label, - otherwise they correspond to the words indices as defined by the - :class:`~tokenizers.pre_tokenizers.PreTokenizer` that was used. - - For special tokens and such (any token that was generated from something that was - not part of the input), the output is :obj:`None` - - Returns: - A :obj:`List` of :obj:`Optional[int]`: A list of optional word index. - """ - -@final -class NormalizedString: - """ - NormalizedString - - A NormalizedString takes care of modifying an "original" string, to obtain a "normalized" one. - While making all the requested modifications, it keeps track of the alignment information - between the two versions of the string. - - Args: - sequence: str: - The string sequence used to initialize this NormalizedString - """ - def __getitem__(self, /, range: int | tuple[int, int] | slice) -> NormalizedString | None: ... - def __new__(cls, /, sequence: str) -> NormalizedString: ... - def __repr__(self, /) -> str: ... - def __str__(self, /) -> str: ... - def append(self, /, s: str) -> None: - """ - Append the given sequence to the string - """ - def clear(self, /) -> None: - """ - Clears the string - """ - def filter(self, /, func: Any) -> None: - """ - Filter each character of the string using the given func - """ - def for_each(self, /, func: Any) -> None: - """ - Calls the given function for each character of the string - """ - def lowercase(self, /) -> None: - """ - Lowercase the string - """ - def lstrip(self, /) -> None: - """ - Strip the left of the string - """ - def map(self, /, func: Any) -> None: - """ - Calls the given function for each character of the string - - Replaces each character of the string using the returned value. Each - returned value **must** be a str of length 1 (ie a character). - """ - def nfc(self, /) -> None: - """ - Runs the NFC normalization - """ - def nfd(self, /) -> None: - """ - Runs the NFD normalization - """ - def nfkc(self, /) -> None: - """ - Runs the NFKC normalization - """ - def nfkd(self, /) -> None: - """ - Runs the NFKD normalization - """ - @property - def normalized(self, /) -> str: - """ - The normalized part of the string - """ - @property - def original(self, /) -> str: ... - def prepend(self, /, s: str) -> None: - """ - Prepend the given sequence to the string - """ - def replace(self, /, pattern: str | Regex, content: str) -> None: - """ - Replace the content of the given pattern with the provided content - - Args: - pattern: Pattern: - A pattern used to match the string. Usually a string or a Regex - - content: str: - The content to be used as replacement - """ - def rstrip(self, /) -> None: - """ - Strip the right of the string - """ - def slice(self, /, range: int | tuple[int, int] | slice) -> NormalizedString | None: - """ - Slice the string using the given range - """ - def split(self, /, pattern: str | Regex, behavior: Incomplete) -> list[NormalizedString]: - """ - Split the NormalizedString using the given pattern and the specified behavior - - Args: - pattern: Pattern: - A pattern used to split the string. Usually a string or a regex built with `tokenizers.Regex` - - behavior: SplitDelimiterBehavior: - The behavior to use when splitting. - Choices: "removed", "isolated", "merged_with_previous", "merged_with_next", - "contiguous" - - Returns: - A list of NormalizedString, representing each split - """ - def strip(self, /) -> None: - """ - Strip both ends of the string - """ - def uppercase(self, /) -> None: - """ - Uppercase the string - """ - -@final -class PreTokenizedString: - """ - PreTokenizedString - - Wrapper over a string, that provides a way to normalize, pre-tokenize, tokenize the - underlying string, while keeping track of the alignment information (offsets). - - The PreTokenizedString manages what we call `splits`. Each split represents a substring - which is a subpart of the original string, with the relevant offsets and tokens. - - When calling one of the methods used to modify the PreTokenizedString (namely one of - `split`, `normalize` or `tokenize), only the `splits` that don't have any associated - tokens will get modified. - - Args: - sequence: str: - The string sequence used to initialize this PreTokenizedString - """ - def __new__(cls, /, s: str) -> PreTokenizedString: ... - def get_splits( - self, /, offset_referential: Incomplete = ..., offset_type: Incomplete = ... - ) -> list[tuple[str, tuple[int, int], list[Token] | None]]: - """ - Get the splits currently managed by the PreTokenizedString - - Args: - offset_referential: :obj:`str` - Whether the returned splits should have offsets expressed relative - to the original string, or the normalized one. choices: "original", "normalized". - - offset_type: :obj:`str` - Whether the returned splits should have offsets expressed in bytes or chars. - When slicing an str, we usually want to use chars, which is the default value. - Now in some cases it might be interesting to get these offsets expressed in bytes, - so it is possible to change this here. - choices: "char", "bytes" - - Returns - A list of splits - """ - def normalize(self, /, func: Any) -> None: - """ - Normalize each split of the `PreTokenizedString` using the given `func` - - Args: - func: Callable[[NormalizedString], None]: - The function used to normalize each underlying split. This function - does not need to return anything, just calling the methods on the provided - NormalizedString allow its modification. - """ - def split(self, /, func: Any) -> None: - """ - Split the PreTokenizedString using the given `func` - - Args: - func: Callable[[index, NormalizedString], List[NormalizedString]]: - The function used to split each underlying split. - It is expected to return a list of `NormalizedString`, that represent the new - splits. If the given `NormalizedString` does not need any splitting, we can - just return it directly. - In order for the offsets to be tracked accurately, any returned `NormalizedString` - should come from calling either `.split` or `.slice` on the received one. - """ - def to_encoding(self, /, type_id: int = 0, word_idx: int | None = None) -> "Encoding": - """ - Return an Encoding generated from this PreTokenizedString - - Args: - type_id: int = 0: - The type_id to be used on the generated Encoding. - - word_idx: Optional[int] = None: - An optional word index to be used for each token of this Encoding. If provided, - all the word indices in the generated Encoding will use this value, instead - of the one automatically tracked during pre-tokenization. - - Returns: - An Encoding - """ - def tokenize(self, /, func: Any) -> None: - """ - Tokenize each split of the `PreTokenizedString` using the given `func` - - Args: - func: Callable[[str], List[Token]]: - The function used to tokenize each underlying split. This function must return - a list of Token generated from the input str. - """ - -@final -class Regex: - """ - Instantiate a new Regex with the given pattern - """ - def __new__(cls, /, s: str) -> Regex: ... - -@final -class Token: - def __new__(cls, /, id: int, value: str, offsets: tuple[int, int]) -> Token: - """ - Create a token from id, string value and byte offsets - """ - def as_tuple(self, /) -> tuple[int, str, tuple[int, int]]: ... - @property - def id(self, /) -> int: ... - @property - def offsets(self, /) -> tuple[int, int]: ... - @property - def value(self, /) -> str: ... + def special(self, /) -> bool: ... @final class Tokenizer: """ - A :obj:`Tokenizer` works as a pipeline. It processes some raw text as input - and outputs an :class:`~tokenizers.Encoding`. - - The pipeline is structured as follows: - - 1. The :class:`~tokenizers.normalizers.Normalizer` normalizes the raw input text. - 2. The :class:`~tokenizers.pre_tokenizers.PreTokenizer` splits the normalized text - into word-level tokens. - 3. The :class:`~tokenizers.models.Model` tokenizes each word into subword tokens - and maps them to IDs. - 4. The :class:`~tokenizers.processors.PostProcessor` applies any final - transformations (e.g., adding special tokens like ``[CLS]`` and ``[SEP]``). - - Args: - model (:class:`~tokenizers.models.Model`): - The core algorithm that this :obj:`Tokenizer` should be using. - - Example:: - - >>> from tokenizers import Tokenizer - >>> from tokenizers.models import BPE - >>> from tokenizers.normalizers import Lowercase - >>> from tokenizers.pre_tokenizers import Whitespace - >>> tokenizer = Tokenizer(BPE(unk_token="")) - >>> tokenizer.normalizer = Lowercase() - >>> tokenizer.pre_tokenizer = Whitespace() - >>> # Load a pre-built tokenizer from HuggingFace Hub - >>> tokenizer = Tokenizer.from_pretrained("bert-base-uncased") + A tokenizer: a model plus its optional normalizer and pre-tokenizer. + + Create one from a model (`Tokenizer(models.BPE())`), a file + (`Tokenizer.from_file`), or the Hub (`Tokenizer.from_pretrained`). + Changes — assigning components, training, adding tokens — apply to the + serializable definition; encoding runs a compiled pipeline that is rebuilt + automatically after any change. A definition the pipeline cannot run + raises `TokenizersError` at that point, with the reason. """ - def __getnewargs__(self, /) -> tuple: ... - def __getstate__(self, /) -> Any: ... - def __new__(cls, /, model: Model) -> Tokenizer: ... - def __repr__(self, /) -> str: ... - def __setstate__(self, /, state: Any) -> None: ... - def __str__(self, /) -> str: ... - def add_special_tokens(self, /, tokens: list) -> int: - """ - Add the given special tokens to the Tokenizer. - - If these tokens are already part of the vocabulary, it just let the Tokenizer know about - them. If they don't exist, the Tokenizer creates them, giving them a new id. - - These special tokens will never be processed by the model (ie won't be split into - multiple tokens), and they can be removed from the output when decoding. - - Args: - tokens (A :obj:`List` of :class:`~tokenizers.AddedToken` or :obj:`str`): - The list of special tokens we want to add to the vocabulary. Each token can either - be a string or an instance of :class:`~tokenizers.AddedToken` for more - customization. - - Returns: - :obj:`int`: The number of tokens that were created in the vocabulary - """ - def add_tokens(self, /, tokens: list) -> int: - """ - Add the given tokens to the vocabulary - - The given tokens are added only if they don't already exist in the vocabulary. - Each token then gets a new attributed id. - - Args: - tokens (A :obj:`List` of :class:`~tokenizers.AddedToken` or :obj:`str`): - The list of tokens we want to add to the vocabulary. Each token can be either a - string or an instance of :class:`~tokenizers.AddedToken` for more customization. - - Returns: - :obj:`int`: The number of tokens that were created in the vocabulary - """ - def async_decode_batch(self, /, sequences: Sequence[Sequence[int]], skip_special_tokens: bool = True) -> Any: - """ - Decode a batch of ids back to their corresponding string - - Args: - sequences (:obj:`List` of :obj:`List[int]`): - The batch of sequences we want to decode - - skip_special_tokens (:obj:`bool`, defaults to :obj:`True`): - Whether the special tokens should be removed from the decoded strings - - Returns: - :obj:`List[str]`: A list of decoded strings - """ - def async_encode( - self, /, sequence: Any, pair: Any | None = None, is_pretokenized: bool = False, add_special_tokens: bool = True - ) -> Any: - """ - Asynchronously encode the given input with character offsets. - - This is an async version of encode that can be awaited in async Python code. - - Example: - Here are some examples of the inputs that are accepted:: - - await async_encode("A single sequence") - - Args: - sequence (:obj:`~tokenizers.InputSequence`): - The main input sequence we want to encode. This sequence can be either raw - text or pre-tokenized, according to the ``is_pretokenized`` argument: - - - If ``is_pretokenized=False``: :class:`~tokenizers.TextInputSequence` - - If ``is_pretokenized=True``: :class:`~tokenizers.PreTokenizedInputSequence` - - pair (:obj:`~tokenizers.InputSequence`, `optional`): - An optional input sequence. The expected format is the same that for ``sequence``. - - is_pretokenized (:obj:`bool`, defaults to :obj:`False`): - Whether the input is already pre-tokenized - - add_special_tokens (:obj:`bool`, defaults to :obj:`True`): - Whether to add the special tokens - - Returns: - :class:`~tokenizers.Encoding`: The encoded result - """ - def async_encode_batch( - self, /, input: Sequence[Any], is_pretokenized: bool = False, add_special_tokens: bool = True - ) -> Any: - """ - Asynchronously encode the given batch of inputs with character offsets. - - This is an async version of encode_batch that can be awaited in async Python code. - - Example: - Here are some examples of the inputs that are accepted:: - - await async_encode_batch([ - "A single sequence", - ("A tuple with a sequence", "And its pair"), - [ "A", "pre", "tokenized", "sequence" ], - ([ "A", "pre", "tokenized", "sequence" ], "And its pair") - ]) - - Args: - input (A :obj:`List`/:obj:`Tuple` of :obj:`~tokenizers.EncodeInput`): - A list of single sequences or pair sequences to encode. Each sequence - can be either raw text or pre-tokenized, according to the ``is_pretokenized`` - argument: - - - If ``is_pretokenized=False``: :class:`~tokenizers.TextEncodeInput` - - If ``is_pretokenized=True``: :class:`~tokenizers.PreTokenizedEncodeInput` - - is_pretokenized (:obj:`bool`, defaults to :obj:`False`): - Whether the input is already pre-tokenized - - add_special_tokens (:obj:`bool`, defaults to :obj:`True`): - Whether to add the special tokens - - Returns: - A :obj:`List` of :class:`~tokenizers.Encoding`: The encoded batch - """ - def async_encode_batch_fast( - self, /, input: Sequence[Any], is_pretokenized: bool = False, add_special_tokens: bool = True - ) -> Any: - """ - Asynchronously encode the given batch of inputs without tracking character offsets. - - This is an async version of encode_batch_fast that can be awaited in async Python code. - - Example: - Here are some examples of the inputs that are accepted:: - - await async_encode_batch_fast([ - "A single sequence", - ("A tuple with a sequence", "And its pair"), - [ "A", "pre", "tokenized", "sequence" ], - ([ "A", "pre", "tokenized", "sequence" ], "And its pair") - ]) - - Args: - input (A :obj:`List`/:obj:`Tuple` of :obj:`~tokenizers.EncodeInput`): - A list of single sequences or pair sequences to encode. Each sequence - can be either raw text or pre-tokenized, according to the ``is_pretokenized`` - argument: - - - If ``is_pretokenized=False``: :class:`~tokenizers.TextEncodeInput` - - If ``is_pretokenized=True``: :class:`~tokenizers.PreTokenizedEncodeInput` - - is_pretokenized (:obj:`bool`, defaults to :obj:`False`): - Whether the input is already pre-tokenized - - add_special_tokens (:obj:`bool`, defaults to :obj:`True`): - Whether to add the special tokens - - Returns: - A :obj:`List` of :class:`~tokenizers.Encoding`: The encoded batch - """ - def decode(self, /, ids: Sequence[int], skip_special_tokens: bool = True) -> "str": - """ - Decode the given list of ids back to a string - - This is used to decode anything coming back from a Language Model - - Args: - ids (A :obj:`List/Tuple` of :obj:`int`): - The list of ids that we want to decode - - skip_special_tokens (:obj:`bool`, defaults to :obj:`True`): - Whether the special tokens should be removed from the decoded string - - Returns: - :obj:`str`: The decoded string - """ - def decode_batch(self, /, sequences: Sequence[Sequence[int]], skip_special_tokens: bool = True) -> "list[str]": - """ - Decode a batch of ids back to their corresponding string - - Args: - sequences (:obj:`List` of :obj:`List[int]`): - The batch of sequences we want to decode - - skip_special_tokens (:obj:`bool`, defaults to :obj:`True`): - Whether the special tokens should be removed from the decoded strings - - Returns: - :obj:`List[str]`: A list of decoded strings - """ - @property - def decoder(self, /) -> Any: - """ - The `optional` :class:`~tokenizers.decoders.Decoder` in use by the Tokenizer - """ - @decoder.setter - def decoder(self, /, decoder: Decoder | None) -> None: - """ - Set the :class:`~tokenizers.decoders.Decoder` - """ - def enable_padding(self, /, **kwargs) -> "None": - """ - Enable the padding - - Args: - direction (:obj:`str`, `optional`, defaults to :obj:`right`): - The direction in which to pad. Can be either ``right`` or ``left`` - - pad_to_multiple_of (:obj:`int`, `optional`): - If specified, the padding length should always snap to the next multiple of the - given value. For example if we were going to pad witha length of 250 but - ``pad_to_multiple_of=8`` then we will pad to 256. - - pad_id (:obj:`int`, defaults to 0): - The id to be used when padding - - pad_type_id (:obj:`int`, defaults to 0): - The type id to be used when padding - - pad_token (:obj:`str`, defaults to :obj:`[PAD]`): - The pad token to be used when padding - - length (:obj:`int`, `optional`): - If specified, the length at which to pad. If not specified we pad using the size of - the longest sequence in a batch. - """ - def enable_truncation(self, /, max_length: int, **kwargs) -> "None": + def __new__(cls, /, model: Model) -> Tokenizer: """ - Enable truncation - - Args: - max_length (:obj:`int`): - The max length at which to truncate - - stride (:obj:`int`, `optional`): - The length of the previous first sequence to be included in the overflowing - sequence - - strategy (:obj:`str`, `optional`, defaults to :obj:`longest_first`): - The strategy used to truncation. Can be one of ``longest_first``, ``only_first`` or - ``only_second``. - - direction (:obj:`str`, defaults to :obj:`right`): - Truncate direction - """ - def encode( - self, /, sequence: Any, pair: Any | None = None, is_pretokenized: bool = False, add_special_tokens: bool = True - ) -> "Encoding": - """ - Encode the given sequence and pair. This method can process raw text sequences - as well as already pre-tokenized sequences. - - Example: - Here are some examples of the inputs that are accepted:: - - encode("A single sequence")` - encode("A sequence", "And its pair")` - encode([ "A", "pre", "tokenized", "sequence" ], is_pretokenized=True)` - encode( - [ "A", "pre", "tokenized", "sequence" ], [ "And", "its", "pair" ], - is_pretokenized=True - ) - - Args: - sequence (:obj:`~tokenizers.InputSequence`): - The main input sequence we want to encode. This sequence can be either raw - text or pre-tokenized, according to the ``is_pretokenized`` argument: - - - If ``is_pretokenized=False``: :class:`~tokenizers.TextInputSequence` - - If ``is_pretokenized=True``: :class:`~tokenizers.PreTokenizedInputSequence` - - pair (:obj:`~tokenizers.InputSequence`, `optional`): - An optional input sequence. The expected format is the same that for ``sequence``. - - is_pretokenized (:obj:`bool`, defaults to :obj:`False`): - Whether the input is already pre-tokenized - - add_special_tokens (:obj:`bool`, defaults to :obj:`True`): - Whether to add the special tokens - - Returns: - :class:`~tokenizers.Encoding`: The encoded result + Create an untrained tokenizer from a model. """ - def encode_batch( - self, /, input: Sequence[Any], is_pretokenized: bool = False, add_special_tokens: bool = True - ) -> "list[Encoding]": + def __reduce__(self, /) -> tuple[Any, tuple[bytes]]: ... + def __repr__(self, /) -> str: ... + def add_special_tokens(self, /, tokens: Sequence[str |AddedToken]) -> int: """ - Encode the given batch of inputs. This method accept both raw text sequences - as well as already pre-tokenized sequences. The reason we use `PySequence` is - because it allows type checking with zero-cost (according to PyO3) as we don't - have to convert to check. - - Example: - Here are some examples of the inputs that are accepted:: - - encode_batch([ - "A single sequence", - ("A tuple with a sequence", "And its pair"), - [ "A", "pre", "tokenized", "sequence" ], - ([ "A", "pre", "tokenized", "sequence" ], "And its pair") - ]) - - Args: - input (A :obj:`List`/:obj:`Tuple` of :obj:`~tokenizers.EncodeInput`): - A list of single sequences or pair sequences to encode. Each sequence - can be either raw text or pre-tokenized, according to the ``is_pretokenized`` - argument: - - - If ``is_pretokenized=False``: :class:`~tokenizers.TextEncodeInput` - - If ``is_pretokenized=True``: :class:`~tokenizers.PreTokenizedEncodeInput` - - is_pretokenized (:obj:`bool`, defaults to :obj:`False`): - Whether the input is already pre-tokenized - - add_special_tokens (:obj:`bool`, defaults to :obj:`True`): - Whether to add the special tokens - - Returns: - A :obj:`List` of :class:`~tokenizers.Encoding`: The encoded batch + Add special tokens ("", "[CLS]", …) to the vocabulary. Same as + `add_tokens`, but every token is marked `special`. Returns how many + were actually new. """ - def encode_batch_fast( - self, /, input: Sequence[Any], is_pretokenized: bool = False, add_special_tokens: bool = True - ) -> "list[Encoding]": + def add_tokens(self, /, tokens: Sequence[str |AddedToken]) -> int: """ - Encode the given batch of inputs. This method is faster than `encode_batch` - because it doesn't keep track of offsets, they will be all zeros. - - Example: - Here are some examples of the inputs that are accepted:: - - encode_batch_fast([ - "A single sequence", - ("A tuple with a sequence", "And its pair"), - [ "A", "pre", "tokenized", "sequence" ], - ([ "A", "pre", "tokenized", "sequence" ], "And its pair") - ]) - - Args: - input (A :obj:`List`/:obj:`Tuple` of :obj:`~tokenizers.EncodeInput`): - A list of single sequences or pair sequences to encode. Each sequence - can be either raw text or pre-tokenized, according to the ``is_pretokenized`` - argument: - - - If ``is_pretokenized=False``: :class:`~tokenizers.TextEncodeInput` - - If ``is_pretokenized=True``: :class:`~tokenizers.PreTokenizedEncodeInput` - - is_pretokenized (:obj:`bool`, defaults to :obj:`False`): - Whether the input is already pre-tokenized - - add_special_tokens (:obj:`bool`, defaults to :obj:`True`): - Whether to add the special tokens - - Returns: - A :obj:`List` of :class:`~tokenizers.Encoding`: The encoded batch + Add tokens to the vocabulary and match them in the input text from now + on. Plain strings match with default options; pass `AddedToken` to + control matching. Returns how many were actually new. """ - @property - def encode_special_tokens(self, /) -> bool: + def decode(self, /, ids: Sequence[int], *, skip_special_tokens: bool = True) -> str: """ - Get the value of the `encode_special_tokens` attribute - - Returns: - :obj:`bool`: the tokenizer's encode_special_tokens attribute + Not implemented yet: decoding is not part of the encode pipeline. """ - @encode_special_tokens.setter - def encode_special_tokens(self, /, value: bool) -> None: + def encode(self, /, text: str, *, add_special_tokens: bool = True) -> "npt.NDArray[np.uint32]": """ - Modifies the tokenizer in order to use or not the special tokens - during encoding. - - Args: - value (:obj:`bool`): - Whether to use the special tokens or not + Encode `text` into token ids. + + Runs entirely outside the interpreter lock and returns a `numpy.uint32` + array backed by the Rust output buffer (no copy). """ - @staticmethod - def from_buffer(buffer: bytes) -> "Tokenizer": + def encode_batch(self, /, texts: Sequence[str], *, add_special_tokens: bool = True) -> "list[npt.NDArray[np.uint32]]": """ - Instantiate a new :class:`~tokenizers.Tokenizer` from the given buffer. - - Args: - buffer (:obj:`bytes`): - A buffer containing a previously serialized :class:`~tokenizers.Tokenizer` - - Returns: - :class:`~tokenizers.Tokenizer`: The new tokenizer + Encode a batch of texts, in parallel across Rust threads (respects + `TOKENIZERS_PARALLELISM`), without holding the interpreter lock. + Input strings are borrowed, not copied; each output is a `numpy.uint32` + array backed by its Rust buffer. """ @staticmethod - def from_file(path: str) -> "Tokenizer": + def from_buffer(buffer: Sequence[int]) -> "Tokenizer": """ - Instantiate a new :class:`~tokenizers.Tokenizer` from the file at the given path. - - Args: - path (:obj:`str`): - A path to a local JSON file representing a previously serialized - :class:`~tokenizers.Tokenizer` - - Returns: - :class:`~tokenizers.Tokenizer`: The new tokenizer + Load a tokenizer from the bytes of a `tokenizer.json` file. """ @staticmethod - def from_pretrained(identifier: str, revision: str = ..., token: str | None = None) -> "Tokenizer": + def from_file(path: str |PathLike[str]) -> "Tokenizer": """ - Instantiate a new :class:`~tokenizers.Tokenizer` from an existing file on the - Hugging Face Hub. - - Args: - identifier (:obj:`str`): - The identifier of a Model on the Hugging Face Hub, that contains - a tokenizer.json file - revision (:obj:`str`, defaults to `main`): - A branch or commit id - token (:obj:`str`, `optional`, defaults to `None`): - An optional auth token used to access private repositories on the - Hugging Face Hub - - Returns: - :class:`~tokenizers.Tokenizer`: The new tokenizer + Load a tokenizer from a `tokenizer.json` file. """ @staticmethod - def from_str(json: str) -> "Tokenizer": + def from_pretrained(identifier: str, *, revision: str = ..., token: str |None = None) -> "Tokenizer": """ - Instantiate a new :class:`~tokenizers.Tokenizer` from the given JSON string. - - Args: - json (:obj:`str`): - A valid JSON string representing a previously serialized - :class:`~tokenizers.Tokenizer` - - Returns: - :class:`~tokenizers.Tokenizer`: The new tokenizer + Download `tokenizer.json` from a model on the Hugging Face Hub (requires + the `huggingface_hub` package) and load it. """ - def get_added_tokens_decoder(self, /) -> "dict[int, AddedToken]": + def get_vocab(self, /, *, with_added_tokens: bool = True) -> dict[str, int]: """ - Get the underlying vocabulary - - Returns: - :obj:`Dict[int, AddedToken]`: The vocabulary + The whole vocabulary as a dict. This copies every entry; prefer + `token_to_id` for lookups. """ - def get_vocab(self, /, with_added_tokens: bool = True) -> "dict[str, int]": + def get_vocab_size(self, /, *, with_added_tokens: bool = True) -> int: """ - Get the underlying vocabulary - - Args: - with_added_tokens (:obj:`bool`, defaults to :obj:`True`): - Whether to include the added tokens - - Returns: - :obj:`Dict[str, int]`: The vocabulary - """ - def get_vocab_size(self, /, with_added_tokens: bool = True) -> "int": - """ - Get the size of the underlying vocabulary - - Args: - with_added_tokens (:obj:`bool`, defaults to :obj:`True`): - Whether to include the added tokens - - Returns: - :obj:`int`: The size of the vocabulary + Number of entries in the vocabulary. `with_added_tokens=False` counts + only what the model was trained with. """ - def id_to_token(self, /, id: int) -> "str | None": + def id_to_token(self, /, id: int) -> str |None: """ - Convert the given id to its corresponding token if it exists - - Args: - id (:obj:`int`): - The id to convert - - Returns: - :obj:`Optional[str]`: An optional token, :obj:`None` if out of vocabulary + The token behind `id`, or None if the id is out of range. """ @property - def model(self, /) -> Any: + def model(self, /) -> Model: """ - The :class:`~tokenizers.models.Model` in use by the Tokenizer + The model in use by this tokenizer (a copy: reassign to change it). """ @model.setter - def model(self, /, model: Model) -> None: - """ - Set the :class:`~tokenizers.models.Model` - """ - def no_padding(self, /) -> None: - """ - Disable padding - """ - def no_truncation(self, /) -> None: - """ - Disable truncation - """ + def model(self, /, model: Model) -> None: ... @property - def normalizer(self, /) -> Any: + def normalizer(self, /) -> Normalizer |None: """ - The `optional` :class:`~tokenizers.normalizers.Normalizer` in use by the Tokenizer + The optional normalizer in use by this tokenizer (a copy: reassign to + change it). """ @normalizer.setter - def normalizer(self, /, normalizer: Normalizer | None) -> None: - """ - Set the :class:`~tokenizers.normalizers.Normalizer` - """ - def num_special_tokens_to_add(self, /, is_pair: bool) -> int: - """ - Return the number of special tokens that would be added for single/pair sentences. - :param is_pair: Boolean indicating if the input would be a single sentence or a pair - :return: - """ - @property - def padding(self, /) -> dict | None: - """ - Get the current padding parameters - - `Cannot be set, use` :meth:`~tokenizers.Tokenizer.enable_padding` `instead` - - Returns: - (:obj:`dict`, `optional`): - A dict with the current padding parameters if padding is enabled - """ - def post_process( - self, /, encoding: Encoding, pair: Encoding | None = None, add_special_tokens: bool = True - ) -> Encoding: - """ - Apply all the post-processing steps to the given encodings. - - The various steps are: - - 1. Truncate according to the set truncation params (provided with - :meth:`~tokenizers.Tokenizer.enable_truncation`) - 2. Apply the :class:`~tokenizers.processors.PostProcessor` - 3. Pad according to the set padding params (provided with - :meth:`~tokenizers.Tokenizer.enable_padding`) - - Args: - encoding (:class:`~tokenizers.Encoding`): - The :class:`~tokenizers.Encoding` corresponding to the main sequence. - - pair (:class:`~tokenizers.Encoding`, `optional`): - An optional :class:`~tokenizers.Encoding` corresponding to the pair sequence. - - add_special_tokens (:obj:`bool`): - Whether to add the special tokens - - Returns: - :class:`~tokenizers.Encoding`: The final post-processed encoding - """ - @property - def post_processor(self, /) -> Any: - """ - The `optional` :class:`~tokenizers.processors.PostProcessor` in use by the Tokenizer - """ - @post_processor.setter - def post_processor(self, /, processor: PostProcessor | None) -> None: - """ - Set the :class:`~tokenizers.processors.PostProcessor` - """ + def normalizer(self, /, normalizer: Normalizer |None) -> None: ... @property - def pre_tokenizer(self, /) -> Any: + def pre_tokenizer(self, /) -> PreTokenizer |None: """ - The `optional` :class:`~tokenizers.pre_tokenizers.PreTokenizer` in use by the Tokenizer + The optional pre-tokenizer in use by this tokenizer (a copy: reassign + to change it). """ @pre_tokenizer.setter - def pre_tokenizer(self, /, pretok: PreTokenizer | None) -> None: + def pre_tokenizer(self, /, pre_tokenizer: PreTokenizer |None) -> None: ... + def save(self, /, path: str |PathLike[str], *, pretty: bool = True) -> None: """ - Set the :class:`~tokenizers.normalizers.Normalizer` + Save the tokenizer definition to a `tokenizer.json` file. """ - def save(self, /, path: str, pretty: bool = True) -> "None": - """ - Save the :class:`~tokenizers.Tokenizer` to the file at the given path. - - Args: - path (:obj:`str`): - A path to a file in which to save the serialized tokenizer. - - pretty (:obj:`bool`, defaults to :obj:`True`): - Whether the JSON file should be pretty formatted. + def to_str(self, /, *, pretty: bool = False) -> str: """ - def to_str(self, /, pretty: bool = False) -> "str": - """ - Gets a serialized string representing this :class:`~tokenizers.Tokenizer`. - - Args: - pretty (:obj:`bool`, defaults to :obj:`False`): - Whether the JSON string should be pretty formatted. - - Returns: - :obj:`str`: A string representing the serialized Tokenizer - """ - def token_to_id(self, /, token: str) -> "int | None": - """ - Convert the given token to its corresponding id if it exists - - Args: - token (:obj:`str`): - The token to convert - - Returns: - :obj:`Optional[int]`: An optional id, :obj:`None` if out of vocabulary + Serialize the tokenizer definition as a `tokenizer.json` string. """ - def train(self, /, files: Sequence[str], trainer: Trainer | None = None) -> None: + def token_to_id(self, /, token: str) -> int |None: """ - Train the Tokenizer using the given files. - - Reads the files line by line, while keeping all the whitespace, even new lines. - If you want to train from data store in-memory, you can check - :meth:`~tokenizers.Tokenizer.train_from_iterator` - - Args: - files (:obj:`List[str]`): - A list of path to the files that we should use for training - - trainer (:obj:`~tokenizers.trainers.Trainer`, `optional`): - An optional trainer that should be used to train our Model + The id of `token`, or None if it is not in the vocabulary. """ - def train_from_iterator(self, /, iterator: Any, trainer: Trainer | None = None, length: int | None = None) -> None: + def train(self, /, files: Sequence[str], *, trainer: Trainer |None = None) -> None: """ - Train the Tokenizer using the provided iterator. - - You can provide anything that is a Python Iterator - - * A list of sequences :obj:`List[str]` - * A generator that yields :obj:`str` or :obj:`List[str]` - * A Numpy array of strings - * ... - - Args: - iterator (:obj:`Iterator`): - Any iterator over strings or list of strings - - trainer (:obj:`~tokenizers.trainers.Trainer`, `optional`): - An optional trainer that should be used to train our Model - - length (:obj:`int`, `optional`): - The total number of sequences in the iterator. This is used to - provide meaningful progress tracking + Train the model's vocabulary on text files (one sequence per line). + Without a `trainer`, the model's default trainer is used. """ - @property - def truncation(self, /) -> dict | None: + def train_from_iterator(self, /, iterator: Any, *, trainer: Trainer |None = None) -> None: """ - Get the currently set truncation parameters - - `Cannot set, use` :meth:`~tokenizers.Tokenizer.enable_truncation` `instead` - - Returns: - (:obj:`dict`, `optional`): - A dict with the current truncation parameters if truncation is enabled + Train the model's vocabulary from any iterator of `str`. Without a + `trainer`, the model's default trainer is used. + + The interpreter lock is only re-acquired to refill an internal buffer + (256 sequences at a time); the training itself runs multi-threaded in + Rust with the lock released. """ def __getattr__(name: str) -> Incomplete: ... + +class TokenizersError(Exception): ... diff --git a/bindings/python/py_src/tokenizers/decoders.pyi b/bindings/python/py_src/tokenizers/decoders.pyi deleted file mode 100644 index 09e31e863..000000000 --- a/bindings/python/py_src/tokenizers/decoders.pyi +++ /dev/null @@ -1,372 +0,0 @@ -""" -Decoders Module -""" - -from collections.abc import Sequence as Sequence2 -from typing import Any, final - -from _typeshed import Incomplete - -from tokenizers import Regex, Tokenizer - -@final -class BPEDecoder(Decoder): - """ - BPEDecoder Decoder - - Args: - suffix (:obj:`str`, `optional`, defaults to :obj:``): - The suffix that was used to characterize an end-of-word. This suffix will - be replaced by whitespaces during the decoding - - Example:: - - >>> from tokenizers.decoders import BPEDecoder - >>> decoder = BPEDecoder() - >>> decoder.decode(["Hello", "world"]) - 'Hello world' - """ - def __new__(cls, /, suffix: str = ...) -> BPEDecoder: ... - @property - def suffix(self, /) -> str: ... - @suffix.setter - def suffix(self, /, suffix: str) -> None: ... - -@final -class ByteFallback(Decoder): - """ - ByteFallback Decoder - - ByteFallback is a decoder that handles tokens representing raw bytes in the - ``<0xNN>`` format (e.g., ``<0x61>`` for the byte ``0x61`` = ``'a'``). It converts - such tokens to their corresponding bytes and attempts to decode the resulting byte - sequence as UTF-8. This is used in LLaMA/SentencePiece models that use byte fallback - for unknown characters. Inconvertible byte tokens are replaced with the Unicode - replacement character (U+FFFD). - - Example:: - - >>> from tokenizers.decoders import ByteFallback, Fuse, Sequence - >>> decoder = Sequence([ByteFallback(), Fuse()]) - >>> decoder.decode(["<0x48>", "<0x65>", "<0x6C>", "<0x6C>", "<0x6F>"]) - 'Hello' - """ - def __new__(cls, /) -> ByteFallback: ... - -@final -class ByteLevel(Decoder): - """ - ByteLevel Decoder - - This decoder is to be used in tandem with the - :class:`~tokenizers.pre_tokenizers.ByteLevel` pre-tokenizer. It reverses the - byte-to-unicode mapping applied during pre-tokenization, converting the special - Unicode characters back into the original bytes to reconstruct the original string. - - Example:: - - >>> from tokenizers.decoders import ByteLevel - >>> decoder = ByteLevel() - >>> decoder.decode(["ĠHello", "Ġworld"]) - ' Hello world' - """ - def __new__(cls, /, **_kwargs) -> ByteLevel: ... - -@final -class CTC(Decoder): - """ - CTC Decoder - - Args: - pad_token (:obj:`str`, `optional`, defaults to :obj:``): - The pad token used by CTC to delimit a new token. - word_delimiter_token (:obj:`str`, `optional`, defaults to :obj:`|`): - The word delimiter token. It will be replaced by a - cleanup (:obj:`bool`, `optional`, defaults to :obj:`True`): - Whether to cleanup some tokenization artifacts. - Mainly spaces before punctuation, and some abbreviated english forms. - - Example:: - - >>> from tokenizers.decoders import CTC - >>> decoder = CTC() - >>> decoder.decode(["h", "e", "e", "", "l", "l", "o", "|", "w", "o", "r", "l", "d"]) - 'hello world' - """ - def __new__(cls, /, pad_token: str = ..., word_delimiter_token: str = ..., cleanup: bool = True) -> CTC: ... - @property - def cleanup(self, /) -> bool: ... - @cleanup.setter - def cleanup(self, /, cleanup: bool) -> None: ... - @property - def pad_token(self, /) -> str: ... - @pad_token.setter - def pad_token(self, /, pad_token: str) -> None: ... - @property - def word_delimiter_token(self, /) -> str: ... - @word_delimiter_token.setter - def word_delimiter_token(self, /, word_delimiter_token: str) -> None: ... - -@final -class DecodeStream: - """ - Provides incremental decoding of token IDs as they are generated, yielding - decoded text chunks as soon as they are available. - - Unlike batch decoding, streaming decode is designed for use with autoregressive - generation — tokens arrive one at a time and the decoder needs to handle - multi-byte sequences (e.g., UTF-8 characters split across token boundaries) and - byte-fallback tokens gracefully. - - The decoder internally buffers tokens until it can produce a valid UTF-8 string - chunk, then yields that chunk and advances its internal state. This means - individual calls to :meth:`~tokenizers.decoders.DecodeStream.step` may return - :obj:`None` when the current token completes a partial sequence that cannot yet - be decoded. - - Args: - skip_special_tokens (:obj:`bool`, defaults to :obj:`False`): - Whether to skip special tokens (e.g. ``[CLS]``, ``[SEP]``, ````) when - decoding. - - Example:: - - >>> from tokenizers import Tokenizer - >>> from tokenizers.decoders import DecodeStream - >>> tokenizer = Tokenizer.from_pretrained("gpt2") - >>> stream = DecodeStream(skip_special_tokens=True) - >>> # Simulate streaming token-by-token generation - >>> token_ids = tokenizer.encode("Hello, streaming world!").ids - >>> for token_id in token_ids: - ... chunk = stream.step(tokenizer, token_id) - ... if chunk is not None: - ... print(chunk, end="", flush=True) - """ - def __copy__(self, /) -> DecodeStream: ... - def __deepcopy__(self, /, _memo: dict) -> DecodeStream: ... - def __new__( - cls, /, ids: Sequence2[int] | None = None, skip_special_tokens: bool | None = False - ) -> DecodeStream: ... - def step(self, /, tokenizer: Tokenizer, id: Incomplete) -> str | None: - """ - Add the next token ID (or list of IDs) to the stream and return the next - decoded text chunk if one is available. - - Because some characters span multiple tokens (e.g. multi-byte UTF-8 - sequences or byte-fallback tokens), this method may return :obj:`None` - when the provided token does not yet complete a decodable unit. Callers - should simply continue feeding tokens until a non-:obj:`None` value is - returned. - - Args: - tokenizer (:class:`~tokenizers.Tokenizer`): - The tokenizer whose decoder pipeline will be used. - - id (:obj:`int` or :obj:`List[int]`): - The next token ID, or a list of token IDs to append to the stream. - - Returns: - :obj:`Optional[str]`: The next decoded text chunk if enough tokens have - accumulated, or :obj:`None` if more tokens are still needed. - """ - -class Decoder: - """ - Base class for all decoders - - This class is not supposed to be instantiated directly. Instead, any implementation of - a Decoder will return an instance of this class when instantiated. - """ - def __getstate__(self, /) -> Any: ... - def __repr__(self, /) -> str: ... - def __setstate__(self, /, state: Any) -> None: ... - def __str__(self, /) -> str: ... - @staticmethod - def custom(decoder: Any) -> Decoder: ... - def decode(self, /, tokens: Sequence2[str]) -> str: - """ - Decode the given list of tokens to a final string - - Args: - tokens (:obj:`List[str]`): - The list of tokens to decode - - Returns: - :obj:`str`: The decoded string - """ - -@final -class Fuse(Decoder): - """ - Fuse Decoder - - Fuse simply concatenates every token into a single string without any separator. - This is typically the last step in a decoder chain when other decoders need to - operate on individual tokens before they are joined together. - - Example:: - - >>> from tokenizers.decoders import Fuse - >>> decoder = Fuse() - >>> decoder.decode(["Hello", ",", " ", "world", "!"]) - 'Hello, world!' - """ - def __new__(cls, /) -> Fuse: ... - -@final -class Metaspace(Decoder): - """ - Metaspace Decoder - - Args: - replacement (:obj:`str`, `optional`, defaults to :obj:`▁`): - The replacement character. Must be exactly one character. By default we - use the `▁` (U+2581) meta symbol (Same as in SentencePiece). - - prepend_scheme (:obj:`str`, `optional`, defaults to :obj:`"always"`): - Whether to add a space to the first word if there isn't already one. This - lets us treat `hello` exactly like `say hello`. - Choices: "always", "never", "first". First means the space is only added on the first - token (relevant when special tokens are used or other pre_tokenizer are used). - - Example:: - - >>> from tokenizers.decoders import Metaspace - >>> decoder = Metaspace() - >>> decoder.decode(["▁Hello", "▁my", "▁friend"]) - 'Hello my friend' - """ - def __new__(cls, /, replacement: str = "▁", prepend_scheme: str = ..., split: bool = True) -> Metaspace: ... - @property - def prepend_scheme(self, /) -> str: ... - @prepend_scheme.setter - def prepend_scheme(self, /, prepend_scheme: str) -> None: ... - @property - def replacement(self, /) -> str: ... - @replacement.setter - def replacement(self, /, replacement: str) -> None: ... - @property - def split(self, /) -> bool: ... - @split.setter - def split(self, /, split: bool) -> None: ... - -@final -class Replace(Decoder): - """ - Replace Decoder - - This decoder is to be used in tandem with the - :class:`~tokenizers.normalizers.Replace` normalizer or a similar replace operation. - It reverses a string replacement by substituting the replacement content back - with the original pattern. - - Args: - pattern (:obj:`str` or :class:`~tokenizers.Regex`): - The pattern that was used as the replacement target during encoding. - - content (:obj:`str`): - The string to replace each match of the pattern with during decoding. - - Example:: - - >>> from tokenizers.decoders import Replace - >>> decoder = Replace("▁", " ") - >>> decoder.decode(["▁Hello", "▁world"]) - ' Hello world' - """ - def __new__(cls, /, pattern: str | Regex, content: str) -> Replace: ... - -@final -class Sequence(Decoder): - """ - Sequence Decoder - - Chains multiple decoders together, applying them in order. Each decoder in the - sequence processes the output of the previous one, allowing complex decoding - pipelines to be built from simpler components. - - Args: - decoders (:obj:`List[Decoder]`): - The list of decoders to chain together. - - Example:: - - >>> from tokenizers.decoders import ByteFallback, Fuse, Metaspace, Sequence - >>> decoder = Sequence([ByteFallback(), Fuse(), Metaspace()]) - >>> decoder.decode(["▁Hello", "▁world"]) - 'Hello world' - """ - def __getnewargs__(self, /) -> tuple: ... - def __new__(cls, /, decoders_py: list) -> Sequence: ... - -@final -class Strip(Decoder): - """ - Strip Decoder - - Strips a given number of occurrences of a character from the left and/or right - side of each token. This is useful for removing padding characters or special - prefix/suffix markers added during tokenization. - - Args: - content (:obj:`str`, defaults to :obj:`" "`): - The character to strip from each token. - - left (:obj:`int`, defaults to :obj:`0`): - The number of occurrences of :obj:`content` to remove from the left - side of each token. - - right (:obj:`int`, defaults to :obj:`0`): - The number of occurrences of :obj:`content` to remove from the right - side of each token. - - Example:: - - >>> from tokenizers.decoders import Strip - >>> decoder = Strip(content="▁", left=1) - >>> decoder.decode(["▁Hello", "▁world"]) - 'Hello world' - """ - def __new__(cls, /, content: str = " ", left: int = 0, right: int = 0) -> Strip: ... - @property - def content(self, /) -> str: ... - @content.setter - def content(self, /, content: str) -> None: ... - @property - def start(self, /) -> int: ... - @start.setter - def start(self, /, start: int) -> None: ... - @property - def stop(self, /) -> int: ... - @stop.setter - def stop(self, /, stop: int) -> None: ... - -@final -class WordPiece(Decoder): - """ - WordPiece Decoder - - Args: - prefix (:obj:`str`, `optional`, defaults to :obj:`##`): - The prefix to use for subwords that are not a beginning-of-word - - cleanup (:obj:`bool`, `optional`, defaults to :obj:`True`): - Whether to cleanup some tokenization artifacts. Mainly spaces before punctuation, - and some abbreviated english forms. - - Example:: - - >>> from tokenizers.decoders import WordPiece - >>> decoder = WordPiece() - >>> decoder.decode(["Hello", ",", "##world", "!"]) - 'Hello, world!' - """ - def __new__(cls, /, prefix: str = ..., cleanup: bool = True) -> WordPiece: ... - @property - def cleanup(self, /) -> bool: ... - @cleanup.setter - def cleanup(self, /, cleanup: bool) -> None: ... - @property - def prefix(self, /) -> str: ... - @prefix.setter - def prefix(self, /, prefix: str) -> None: ... diff --git a/bindings/python/py_src/tokenizers/decoders/__init__.py b/bindings/python/py_src/tokenizers/decoders/__init__.py deleted file mode 100644 index 12ada5dbd..000000000 --- a/bindings/python/py_src/tokenizers/decoders/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -from .. import decoders - - -Decoder = decoders.Decoder -ByteLevel = decoders.ByteLevel -Replace = decoders.Replace -WordPiece = decoders.WordPiece -ByteFallback = decoders.ByteFallback -Fuse = decoders.Fuse -Strip = decoders.Strip -Metaspace = decoders.Metaspace -BPEDecoder = decoders.BPEDecoder -CTC = decoders.CTC -Sequence = decoders.Sequence -DecodeStream = decoders.DecodeStream diff --git a/bindings/python/py_src/tokenizers/implementations/__init__.py b/bindings/python/py_src/tokenizers/implementations/__init__.py deleted file mode 100644 index 7e775892d..000000000 --- a/bindings/python/py_src/tokenizers/implementations/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from .base_tokenizer import BaseTokenizer -from .bert_wordpiece import BertWordPieceTokenizer -from .byte_level_bpe import ByteLevelBPETokenizer -from .char_level_bpe import CharBPETokenizer -from .sentencepiece_bpe import SentencePieceBPETokenizer -from .sentencepiece_unigram import SentencePieceUnigramTokenizer diff --git a/bindings/python/py_src/tokenizers/implementations/base_tokenizer.py b/bindings/python/py_src/tokenizers/implementations/base_tokenizer.py deleted file mode 100644 index b01530399..000000000 --- a/bindings/python/py_src/tokenizers/implementations/base_tokenizer.py +++ /dev/null @@ -1,477 +0,0 @@ -from typing import Dict, List, Optional, Tuple, Union - -from tokenizers import AddedToken, EncodeInput, Encoding, InputSequence, Tokenizer -from tokenizers.decoders import Decoder -from tokenizers.models import Model -from tokenizers.normalizers import Normalizer -from tokenizers.pre_tokenizers import PreTokenizer -from tokenizers.processors import PostProcessor - - -Offsets = Tuple[int, int] - - -class BaseTokenizer: - def __init__(self, tokenizer: Tokenizer, parameters=None): - self._tokenizer = tokenizer - self._parameters = parameters if parameters is not None else {} - - def __repr__(self): - return "Tokenizer(vocabulary_size={}, {})".format( - self._tokenizer.get_vocab_size(), - ", ".join(k + "=" + str(v) for k, v in self._parameters.items()), - ) - - def num_special_tokens_to_add(self, is_pair: bool) -> int: - """ - Return the number of special tokens that would be added for single/pair sentences. - :param is_pair: Boolean indicating if the input would be a single sentence or a pair - :return: - """ - return self._tokenizer.num_special_tokens_to_add(is_pair) - - def get_vocab(self, with_added_tokens: bool = True) -> Dict[str, int]: - """Returns the vocabulary - - Args: - with_added_tokens: boolean: - Whether to include the added tokens in the vocabulary - - Returns: - The vocabulary - """ - return self._tokenizer.get_vocab(with_added_tokens=with_added_tokens) - - def get_added_tokens_decoder(self) -> Dict[int, AddedToken]: - """Returns the added reverse vocabulary - - Returns: - The added vocabulary mapping ints to AddedTokens - """ - return self._tokenizer.get_added_tokens_decoder() - - def get_vocab_size(self, with_added_tokens: bool = True) -> int: - """Return the size of vocabulary, with or without added tokens. - - Args: - with_added_tokens: (`optional`) bool: - Whether to count in added special tokens or not - - Returns: - Size of vocabulary - """ - return self._tokenizer.get_vocab_size(with_added_tokens=with_added_tokens) - - def enable_padding( - self, - direction: Optional[str] = "right", - pad_to_multiple_of: Optional[int] = None, - pad_id: Optional[int] = 0, - pad_type_id: Optional[int] = 0, - pad_token: Optional[str] = "[PAD]", - length: Optional[int] = None, - ): - """Change the padding strategy - - Args: - direction: (`optional`) str: - Can be one of: `right` or `left` - - pad_to_multiple_of: (`optional`) unsigned int: - If specified, the padding length should always snap to the next multiple of - the given value. For example if we were going to pad with a length of 250 but - `pad_to_multiple_of=8` then we will pad to 256. - - pad_id: (`optional`) unsigned int: - The indice to be used when padding - - pad_type_id: (`optional`) unsigned int: - The type indice to be used when padding - - pad_token: (`optional`) str: - The pad token to be used when padding - - length: (`optional`) unsigned int: - If specified, the length at which to pad. If not specified - we pad using the size of the longest sequence in a batch - """ - return self._tokenizer.enable_padding( - direction=direction, - pad_to_multiple_of=pad_to_multiple_of, - pad_id=pad_id, - pad_type_id=pad_type_id, - pad_token=pad_token, - length=length, - ) - - def no_padding(self): - """Disable padding""" - return self._tokenizer.no_padding() - - @property - def padding(self) -> Optional[dict]: - """Get the current padding parameters - - Returns: - None if padding is disabled, a dict with the currently set parameters - if the padding is enabled. - """ - return self._tokenizer.padding - - def enable_truncation(self, max_length: int, stride: Optional[int] = 0, strategy: Optional[str] = "longest_first"): - """Change the truncation options - - Args: - max_length: unsigned int: - The maximum length at which to truncate - - stride: (`optional`) unsigned int: - The length of the previous first sequence to be included - in the overflowing sequence - - strategy: (`optional`) str: - Can be one of `longest_first`, `only_first` or `only_second` - """ - return self._tokenizer.enable_truncation(max_length, stride=stride, strategy=strategy) - - def no_truncation(self): - """Disable truncation""" - return self._tokenizer.no_truncation() - - @property - def truncation(self) -> Optional[dict]: - """Get the current truncation parameters - - Returns: - None if truncation is disabled, a dict with the current truncation parameters if - truncation is enabled - """ - return self._tokenizer.truncation - - def add_tokens(self, tokens: List[Union[str, AddedToken]]) -> int: - """Add the given tokens to the vocabulary - - Args: - tokens: List[Union[str, AddedToken]]: - A list of tokens to add to the vocabulary. Each token can either be - a string, or an instance of AddedToken - - Returns: - The number of tokens that were added to the vocabulary - """ - return self._tokenizer.add_tokens(tokens) - - def add_special_tokens(self, special_tokens: List[Union[str, AddedToken]]) -> int: - """Add the given special tokens to the vocabulary, and treat them as special tokens. - - The special tokens will never be processed by the model, and will be - removed while decoding. - - Args: - tokens: List[Union[str, AddedToken]]: - A list of special tokens to add to the vocabulary. Each token can either be - a string, or an instance of AddedToken - - Returns: - The number of tokens that were added to the vocabulary - """ - return self._tokenizer.add_special_tokens(special_tokens) - - def normalize(self, sequence: str) -> str: - """Normalize the given sequence - - Args: - sequence: str: - The sequence to normalize - - Returns: - The normalized string - """ - return self._tokenizer.normalizer.normalize_str(sequence) - - def encode( - self, - sequence: InputSequence, - pair: Optional[InputSequence] = None, - is_pretokenized: bool = False, - add_special_tokens: bool = True, - ) -> Encoding: - """Encode the given sequence and pair. This method can process raw text sequences as well - as already pre-tokenized sequences. - - Args: - sequence: InputSequence: - The sequence we want to encode. This sequence can be either raw text or - pre-tokenized, according to the `is_pretokenized` argument: - - - If `is_pretokenized=False`: `InputSequence` is expected to be `str` - - If `is_pretokenized=True`: `InputSequence` is expected to be - `Union[List[str], Tuple[str]]` - - is_pretokenized: bool: - Whether the input is already pre-tokenized. - - add_special_tokens: bool: - Whether to add the special tokens while encoding. - - Returns: - An Encoding - """ - if sequence is None: - raise ValueError("encode: `sequence` can't be `None`") - - return self._tokenizer.encode(sequence, pair, is_pretokenized, add_special_tokens) - - def encode_batch( - self, - inputs: List[EncodeInput], - is_pretokenized: bool = False, - add_special_tokens: bool = True, - ) -> List[Encoding]: - """Encode the given inputs. This method accept both raw text sequences as well as already - pre-tokenized sequences. - - Args: - inputs: List[EncodeInput]: - A list of single sequences or pair sequences to encode. Each `EncodeInput` is - expected to be of the following form: - `Union[InputSequence, Tuple[InputSequence, InputSequence]]` - - Each `InputSequence` can either be raw text or pre-tokenized, - according to the `is_pretokenized` argument: - - - If `is_pretokenized=False`: `InputSequence` is expected to be `str` - - If `is_pretokenized=True`: `InputSequence` is expected to be - `Union[List[str], Tuple[str]]` - - is_pretokenized: bool: - Whether the input is already pre-tokenized. - - add_special_tokens: bool: - Whether to add the special tokens while encoding. - - Returns: - A list of Encoding - """ - - if inputs is None: - raise ValueError("encode_batch: `inputs` can't be `None`") - - return self._tokenizer.encode_batch(inputs, is_pretokenized, add_special_tokens) - - async def async_encode_batch( - self, - inputs: List[EncodeInput], - is_pretokenized: bool = False, - add_special_tokens: bool = True, - ) -> List[Encoding]: - """Asynchronously encode a batch (tracks character offsets). - - Args: - inputs: A list of single or pair sequences to encode. - is_pretokenized: Whether inputs are already pre-tokenized. - add_special_tokens: Whether to add special tokens. - - Returns: - A list of Encoding. - """ - if inputs is None: - raise ValueError("async_encode_batch: `inputs` can't be `None`") - # Exposed by the Rust bindings via pyo3_async_runtimes::tokio::future_into_py - return await self._tokenizer.async_encode_batch(inputs, is_pretokenized, add_special_tokens) - - async def async_encode_batch_fast( - self, - inputs: List[EncodeInput], - is_pretokenized: bool = False, - add_special_tokens: bool = True, - ) -> List[Encoding]: - """Asynchronously encode a batch (no character offsets, faster). - - Args: - inputs: A list of single or pair sequences to encode. - is_pretokenized: Whether inputs are already pre-tokenized. - add_special_tokens: Whether to add special tokens. - - Returns: - A list of Encoding. - """ - if inputs is None: - raise ValueError("async_encode_batch_fast: `inputs` can't be `None`") - return await self._tokenizer.async_encode_batch_fast(inputs, is_pretokenized, add_special_tokens) - - def decode(self, ids: List[int], skip_special_tokens: Optional[bool] = True) -> str: - """Decode the given list of ids to a string sequence - - Args: - ids: List[unsigned int]: - A list of ids to be decoded - - skip_special_tokens: (`optional`) boolean: - Whether to remove all the special tokens from the output string - - Returns: - The decoded string - """ - if ids is None: - raise ValueError("None input is not valid. Should be a list of integers.") - - return self._tokenizer.decode(ids, skip_special_tokens=skip_special_tokens) - - def decode_batch(self, sequences: List[List[int]], skip_special_tokens: Optional[bool] = True) -> str: - """Decode the list of sequences to a list of string sequences - - Args: - sequences: List[List[unsigned int]]: - A list of sequence of ids to be decoded - - skip_special_tokens: (`optional`) boolean: - Whether to remove all the special tokens from the output strings - - Returns: - A list of decoded strings - """ - if sequences is None: - raise ValueError("None input is not valid. Should be list of list of integers.") - - return self._tokenizer.decode_batch(sequences, skip_special_tokens=skip_special_tokens) - - async def async_decode_batch( - self, - sequences: List[List[int]], - skip_special_tokens: bool = True, - ) -> List[str]: - """Asynchronously decode a batch of sequences. - - Args: - sequences: A list of sequences of ids to decode. - skip_special_tokens: Whether to remove special tokens from output. - - Returns: - A list of decoded strings. - """ - if sequences is None: - raise ValueError("async_decode_batch: `sequences` can't be `None`") - return await self._tokenizer.async_decode_batch(sequences, skip_special_tokens) - - def token_to_id(self, token: str) -> Optional[int]: - """Convert the given token to its corresponding id - - Args: - token: str: - The token to convert - - Returns: - The corresponding id if it exists, None otherwise - """ - return self._tokenizer.token_to_id(token) - - def id_to_token(self, id: int) -> Optional[str]: - """Convert the given token id to its corresponding string - - Args: - token: id: - The token id to convert - - Returns: - The corresponding string if it exists, None otherwise - """ - return self._tokenizer.id_to_token(id) - - def save_model(self, directory: str, prefix: Optional[str] = None): - """Save the current model to the given directory - - Args: - directory: str: - A path to the destination directory - - prefix: (Optional) str: - An optional prefix, used to prefix each file name - """ - return self._tokenizer.model.save(directory, prefix=prefix) - - def save(self, path: str, pretty: bool = True): - """Save the current Tokenizer at the given path - - Args: - path: str: - A path to the destination Tokenizer file - """ - return self._tokenizer.save(path, pretty) - - def to_str(self, pretty: bool = False): - """Get a serialized JSON version of the Tokenizer as a str - - Args: - pretty: bool: - Whether the JSON string should be prettified - - Returns: - str - """ - return self._tokenizer.to_str(pretty) - - def post_process( - self, encoding: Encoding, pair: Optional[Encoding] = None, add_special_tokens: bool = True - ) -> Encoding: - """Apply all the post-processing steps to the given encodings. - - The various steps are: - 1. Truncate according to global params (provided to `enable_truncation`) - 2. Apply the PostProcessor - 3. Pad according to global params. (provided to `enable_padding`) - - Args: - encoding: Encoding: - The main Encoding to post process - - pair: Optional[Encoding]: - An optional pair Encoding - - add_special_tokens: bool: - Whether to add special tokens - - Returns: - The resulting Encoding - """ - return self._tokenizer.post_process(encoding, pair, add_special_tokens) - - @property - def model(self) -> Model: - return self._tokenizer.model - - @model.setter - def model(self, model: Model): - self._tokenizer.model = model - - @property - def normalizer(self) -> Normalizer: - return self._tokenizer.normalizer - - @normalizer.setter - def normalizer(self, normalizer: Normalizer): - self._tokenizer.normalizer = normalizer - - @property - def pre_tokenizer(self) -> PreTokenizer: - return self._tokenizer.pre_tokenizer - - @pre_tokenizer.setter - def pre_tokenizer(self, pre_tokenizer: PreTokenizer): - self._tokenizer.pre_tokenizer = pre_tokenizer - - @property - def post_processor(self) -> PostProcessor: - return self._tokenizer.post_processor - - @post_processor.setter - def post_processor(self, post_processor: PostProcessor): - self._tokenizer.post_processor = post_processor - - @property - def decoder(self) -> Decoder: - return self._tokenizer.decoder - - @decoder.setter - def decoder(self, decoder: Decoder): - self._tokenizer.decoder = decoder diff --git a/bindings/python/py_src/tokenizers/implementations/bert_wordpiece.py b/bindings/python/py_src/tokenizers/implementations/bert_wordpiece.py deleted file mode 100644 index 1f34e3ca8..000000000 --- a/bindings/python/py_src/tokenizers/implementations/bert_wordpiece.py +++ /dev/null @@ -1,151 +0,0 @@ -from typing import Dict, Iterator, List, Optional, Union - -from tokenizers import AddedToken, Tokenizer, decoders, trainers -from tokenizers.models import WordPiece -from tokenizers.normalizers import BertNormalizer -from tokenizers.pre_tokenizers import BertPreTokenizer -from tokenizers.processors import BertProcessing - -from .base_tokenizer import BaseTokenizer - - -class BertWordPieceTokenizer(BaseTokenizer): - """Bert WordPiece Tokenizer""" - - def __init__( - self, - vocab: Optional[Union[str, Dict[str, int]]] = None, - unk_token: Union[str, AddedToken] = "[UNK]", - sep_token: Union[str, AddedToken] = "[SEP]", - cls_token: Union[str, AddedToken] = "[CLS]", - pad_token: Union[str, AddedToken] = "[PAD]", - mask_token: Union[str, AddedToken] = "[MASK]", - clean_text: bool = True, - handle_chinese_chars: bool = True, - strip_accents: Optional[bool] = None, - lowercase: bool = True, - wordpieces_prefix: str = "##", - ): - if vocab is not None: - tokenizer = Tokenizer(WordPiece(vocab, unk_token=str(unk_token))) - else: - tokenizer = Tokenizer(WordPiece(unk_token=str(unk_token))) - - # Let the tokenizer know about special tokens if they are part of the vocab - if tokenizer.token_to_id(str(unk_token)) is not None: - tokenizer.add_special_tokens([str(unk_token)]) - if tokenizer.token_to_id(str(sep_token)) is not None: - tokenizer.add_special_tokens([str(sep_token)]) - if tokenizer.token_to_id(str(cls_token)) is not None: - tokenizer.add_special_tokens([str(cls_token)]) - if tokenizer.token_to_id(str(pad_token)) is not None: - tokenizer.add_special_tokens([str(pad_token)]) - if tokenizer.token_to_id(str(mask_token)) is not None: - tokenizer.add_special_tokens([str(mask_token)]) - - tokenizer.normalizer = BertNormalizer( - clean_text=clean_text, - handle_chinese_chars=handle_chinese_chars, - strip_accents=strip_accents, - lowercase=lowercase, - ) - tokenizer.pre_tokenizer = BertPreTokenizer() - - if vocab is not None: - sep_token_id = tokenizer.token_to_id(str(sep_token)) - if sep_token_id is None: - raise TypeError("sep_token not found in the vocabulary") - cls_token_id = tokenizer.token_to_id(str(cls_token)) - if cls_token_id is None: - raise TypeError("cls_token not found in the vocabulary") - - tokenizer.post_processor = BertProcessing((str(sep_token), sep_token_id), (str(cls_token), cls_token_id)) - tokenizer.decoder = decoders.WordPiece(prefix=wordpieces_prefix) - - parameters = { - "model": "BertWordPiece", - "unk_token": unk_token, - "sep_token": sep_token, - "cls_token": cls_token, - "pad_token": pad_token, - "mask_token": mask_token, - "clean_text": clean_text, - "handle_chinese_chars": handle_chinese_chars, - "strip_accents": strip_accents, - "lowercase": lowercase, - "wordpieces_prefix": wordpieces_prefix, - } - - super().__init__(tokenizer, parameters) - - @staticmethod - def from_file(vocab: str, **kwargs): - vocab = WordPiece.read_file(vocab) - return BertWordPieceTokenizer(vocab, **kwargs) - - def train( - self, - files: Union[str, List[str]], - vocab_size: int = 30000, - min_frequency: int = 2, - limit_alphabet: int = 1000, - initial_alphabet: List[str] = [], - special_tokens: List[Union[str, AddedToken]] = [ - "[PAD]", - "[UNK]", - "[CLS]", - "[SEP]", - "[MASK]", - ], - show_progress: bool = True, - wordpieces_prefix: str = "##", - ): - """Train the model using the given files""" - - trainer = trainers.WordPieceTrainer( - vocab_size=vocab_size, - min_frequency=min_frequency, - limit_alphabet=limit_alphabet, - initial_alphabet=initial_alphabet, - special_tokens=special_tokens, - show_progress=show_progress, - continuing_subword_prefix=wordpieces_prefix, - ) - if isinstance(files, str): - files = [files] - self._tokenizer.train(files, trainer=trainer) - - def train_from_iterator( - self, - iterator: Union[Iterator[str], Iterator[Iterator[str]]], - vocab_size: int = 30000, - min_frequency: int = 2, - limit_alphabet: int = 1000, - initial_alphabet: List[str] = [], - special_tokens: List[Union[str, AddedToken]] = [ - "[PAD]", - "[UNK]", - "[CLS]", - "[SEP]", - "[MASK]", - ], - show_progress: bool = True, - wordpieces_prefix: str = "##", - length: Optional[int] = None, - ): - """Train the model using the given iterator""" - - trainer = trainers.WordPieceTrainer( - vocab_size=vocab_size, - min_frequency=min_frequency, - limit_alphabet=limit_alphabet, - initial_alphabet=initial_alphabet, - special_tokens=special_tokens, - show_progress=show_progress, - continuing_subword_prefix=wordpieces_prefix, - ) - self._tokenizer.train_from_iterator( - iterator, - trainer=trainer, - length=length, - ) diff --git a/bindings/python/py_src/tokenizers/implementations/byte_level_bpe.py b/bindings/python/py_src/tokenizers/implementations/byte_level_bpe.py deleted file mode 100644 index f65f05e1d..000000000 --- a/bindings/python/py_src/tokenizers/implementations/byte_level_bpe.py +++ /dev/null @@ -1,122 +0,0 @@ -from typing import Dict, Iterator, List, Optional, Tuple, Union - -from tokenizers import AddedToken, Tokenizer, decoders, pre_tokenizers, processors, trainers -from tokenizers.models import BPE -from tokenizers.normalizers import Lowercase, Sequence, unicode_normalizer_from_str - -from .base_tokenizer import BaseTokenizer - - -class ByteLevelBPETokenizer(BaseTokenizer): - """ByteLevelBPETokenizer - - Represents a Byte-level BPE as introduced by OpenAI with their GPT-2 model - """ - - def __init__( - self, - vocab: Optional[Union[str, Dict[str, int]]] = None, - merges: Optional[Union[str, List[Tuple[str, str]]]] = None, - add_prefix_space: bool = False, - lowercase: bool = False, - dropout: Optional[float] = None, - unicode_normalizer: Optional[str] = None, - continuing_subword_prefix: Optional[str] = None, - end_of_word_suffix: Optional[str] = None, - trim_offsets: bool = False, - ): - if vocab is not None and merges is not None: - tokenizer = Tokenizer( - BPE( - vocab, - merges, - dropout=dropout, - continuing_subword_prefix=continuing_subword_prefix or "", - end_of_word_suffix=end_of_word_suffix or "", - ) - ) - else: - tokenizer = Tokenizer(BPE()) - - # Check for Unicode normalization first (before everything else) - normalizers = [] - - if unicode_normalizer: - normalizers += [unicode_normalizer_from_str(unicode_normalizer)] - - if lowercase: - normalizers += [Lowercase()] - - # Create the normalizer structure - if len(normalizers) > 0: - if len(normalizers) > 1: - tokenizer.normalizer = Sequence(normalizers) - else: - tokenizer.normalizer = normalizers[0] - - tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=add_prefix_space) - tokenizer.decoder = decoders.ByteLevel() - tokenizer.post_processor = processors.ByteLevel(trim_offsets=trim_offsets) - - parameters = { - "model": "ByteLevelBPE", - "add_prefix_space": add_prefix_space, - "lowercase": lowercase, - "dropout": dropout, - "unicode_normalizer": unicode_normalizer, - "continuing_subword_prefix": continuing_subword_prefix, - "end_of_word_suffix": end_of_word_suffix, - "trim_offsets": trim_offsets, - } - - super().__init__(tokenizer, parameters) - - @staticmethod - def from_file(vocab_filename: str, merges_filename: str, **kwargs): - vocab, merges = BPE.read_file(vocab_filename, merges_filename) - return ByteLevelBPETokenizer(vocab, merges, **kwargs) - - def train( - self, - files: Union[str, List[str]], - vocab_size: int = 30000, - min_frequency: int = 2, - show_progress: bool = True, - special_tokens: List[Union[str, AddedToken]] = [], - ): - """Train the model using the given files""" - - trainer = trainers.BpeTrainer( - vocab_size=vocab_size, - min_frequency=min_frequency, - show_progress=show_progress, - special_tokens=special_tokens, - initial_alphabet=pre_tokenizers.ByteLevel.alphabet(), - ) - if isinstance(files, str): - files = [files] - self._tokenizer.train(files, trainer=trainer) - - def train_from_iterator( - self, - iterator: Union[Iterator[str], Iterator[Iterator[str]]], - vocab_size: int = 30000, - min_frequency: int = 2, - show_progress: bool = True, - special_tokens: List[Union[str, AddedToken]] = [], - length: Optional[int] = None, - ): - """Train the model using the given iterator""" - - trainer = trainers.BpeTrainer( - vocab_size=vocab_size, - min_frequency=min_frequency, - show_progress=show_progress, - special_tokens=special_tokens, - initial_alphabet=pre_tokenizers.ByteLevel.alphabet(), - ) - self._tokenizer.train_from_iterator( - iterator, - trainer=trainer, - length=length, - ) diff --git a/bindings/python/py_src/tokenizers/implementations/char_level_bpe.py b/bindings/python/py_src/tokenizers/implementations/char_level_bpe.py deleted file mode 100644 index 62b5bcdf0..000000000 --- a/bindings/python/py_src/tokenizers/implementations/char_level_bpe.py +++ /dev/null @@ -1,150 +0,0 @@ -from typing import Dict, Iterator, List, Optional, Tuple, Union - -from .. import AddedToken, Tokenizer, decoders, pre_tokenizers, trainers -from ..models import BPE -from ..normalizers import BertNormalizer, Lowercase, Sequence, unicode_normalizer_from_str -from .base_tokenizer import BaseTokenizer - - -class CharBPETokenizer(BaseTokenizer): - """Original BPE Tokenizer - - Represents the BPE algorithm, as introduced by Rico Sennrich - (https://arxiv.org/abs/1508.07909) - - The defaults settings corresponds to OpenAI GPT BPE tokenizers and differs from the original - Sennrich subword-nmt implementation by the following options that you can deactivate: - - adding a normalizer to clean up the text (deactivate with `bert_normalizer=False`) by: - * removing any control characters and replacing all whitespaces by the classic one. - * handle chinese chars by putting spaces around them. - * strip all accents. - - spitting on punctuation in addition to whitespaces (deactivate it with - `split_on_whitespace_only=True`) - """ - - def __init__( - self, - vocab: Optional[Union[str, Dict[str, int]]] = None, - merges: Optional[Union[str, List[Tuple[str, str]]]] = None, - unk_token: Union[str, AddedToken] = "", - suffix: str = "", - dropout: Optional[float] = None, - lowercase: bool = False, - unicode_normalizer: Optional[str] = None, - bert_normalizer: bool = True, - split_on_whitespace_only: bool = False, - ): - if vocab is not None and merges is not None: - tokenizer = Tokenizer( - BPE( - vocab, - merges, - dropout=dropout, - unk_token=str(unk_token), - end_of_word_suffix=suffix, - ) - ) - else: - tokenizer = Tokenizer(BPE(unk_token=str(unk_token), dropout=dropout, end_of_word_suffix=suffix)) - - if tokenizer.token_to_id(str(unk_token)) is not None: - tokenizer.add_special_tokens([str(unk_token)]) - - # Check for Unicode normalization first (before everything else) - normalizers = [] - - if unicode_normalizer: - normalizers += [unicode_normalizer_from_str(unicode_normalizer)] - - if bert_normalizer: - normalizers += [BertNormalizer(lowercase=False)] - - if lowercase: - normalizers += [Lowercase()] - - # Create the normalizer structure - if len(normalizers) > 0: - if len(normalizers) > 1: - tokenizer.normalizer = Sequence(normalizers) - else: - tokenizer.normalizer = normalizers[0] - - if split_on_whitespace_only: - tokenizer.pre_tokenizer = pre_tokenizers.WhitespaceSplit() - else: - tokenizer.pre_tokenizer = pre_tokenizers.BertPreTokenizer() - - tokenizer.decoder = decoders.BPEDecoder(suffix=suffix) - - parameters = { - "model": "BPE", - "unk_token": unk_token, - "suffix": suffix, - "dropout": dropout, - "lowercase": lowercase, - "unicode_normalizer": unicode_normalizer, - "bert_normalizer": bert_normalizer, - "split_on_whitespace_only": split_on_whitespace_only, - } - - super().__init__(tokenizer, parameters) - - @staticmethod - def from_file(vocab_filename: str, merges_filename: str, **kwargs): - vocab, merges = BPE.read_file(vocab_filename, merges_filename) - return CharBPETokenizer(vocab, merges, **kwargs) - - def train( - self, - files: Union[str, List[str]], - vocab_size: int = 30000, - min_frequency: int = 2, - special_tokens: List[Union[str, AddedToken]] = [""], - limit_alphabet: int = 1000, - initial_alphabet: List[str] = [], - suffix: Optional[str] = "", - show_progress: bool = True, - ): - """Train the model using the given files""" - - trainer = trainers.BpeTrainer( - vocab_size=vocab_size, - min_frequency=min_frequency, - special_tokens=special_tokens, - limit_alphabet=limit_alphabet, - initial_alphabet=initial_alphabet, - end_of_word_suffix=suffix, - show_progress=show_progress, - ) - if isinstance(files, str): - files = [files] - self._tokenizer.train(files, trainer=trainer) - - def train_from_iterator( - self, - iterator: Union[Iterator[str], Iterator[Iterator[str]]], - vocab_size: int = 30000, - min_frequency: int = 2, - special_tokens: List[Union[str, AddedToken]] = [""], - limit_alphabet: int = 1000, - initial_alphabet: List[str] = [], - suffix: Optional[str] = "", - show_progress: bool = True, - length: Optional[int] = None, - ): - """Train the model using the given iterator""" - - trainer = trainers.BpeTrainer( - vocab_size=vocab_size, - min_frequency=min_frequency, - special_tokens=special_tokens, - limit_alphabet=limit_alphabet, - initial_alphabet=initial_alphabet, - end_of_word_suffix=suffix, - show_progress=show_progress, - ) - self._tokenizer.train_from_iterator( - iterator, - trainer=trainer, - length=length, - ) diff --git a/bindings/python/py_src/tokenizers/implementations/sentencepiece_bpe.py b/bindings/python/py_src/tokenizers/implementations/sentencepiece_bpe.py deleted file mode 100644 index 26200489a..000000000 --- a/bindings/python/py_src/tokenizers/implementations/sentencepiece_bpe.py +++ /dev/null @@ -1,103 +0,0 @@ -from typing import Dict, Iterator, List, Optional, Tuple, Union - -from tokenizers import AddedToken, Tokenizer, decoders, pre_tokenizers, trainers -from tokenizers.models import BPE -from tokenizers.normalizers import NFKC - -from .base_tokenizer import BaseTokenizer - - -class SentencePieceBPETokenizer(BaseTokenizer): - """SentencePiece BPE Tokenizer - - Represents the BPE algorithm, with the pretokenization used by SentencePiece - """ - - def __init__( - self, - vocab: Optional[Union[str, Dict[str, int]]] = None, - merges: Optional[Union[str, List[Tuple[str, str]]]] = None, - unk_token: Union[str, AddedToken] = "", - replacement: str = "▁", - add_prefix_space: bool = True, - dropout: Optional[float] = None, - fuse_unk: Optional[bool] = False, - ): - if vocab is not None and merges is not None: - tokenizer = Tokenizer(BPE(vocab, merges, dropout=dropout, unk_token=unk_token, fuse_unk=fuse_unk)) - else: - tokenizer = Tokenizer(BPE(dropout=dropout, unk_token=unk_token, fuse_unk=fuse_unk)) - - if tokenizer.token_to_id(str(unk_token)) is not None: - tokenizer.add_special_tokens([str(unk_token)]) - - tokenizer.normalizer = NFKC() - prepend_scheme = "always" if add_prefix_space else "never" - tokenizer.pre_tokenizer = pre_tokenizers.Metaspace(replacement=replacement, prepend_scheme=prepend_scheme) - tokenizer.decoder = decoders.Metaspace(replacement=replacement, prepend_scheme=prepend_scheme) - - parameters = { - "model": "SentencePieceBPE", - "unk_token": unk_token, - "replacement": replacement, - "add_prefix_space": add_prefix_space, - "dropout": dropout, - } - - super().__init__(tokenizer, parameters) - - @staticmethod - def from_file(vocab_filename: str, merges_filename: str, **kwargs): - vocab, merges = BPE.read_file(vocab_filename, merges_filename) - return SentencePieceBPETokenizer(vocab, merges, **kwargs) - - def train( - self, - files: Union[str, List[str]], - vocab_size: int = 30000, - min_frequency: int = 2, - special_tokens: List[Union[str, AddedToken]] = [""], - limit_alphabet: int = 1000, - initial_alphabet: List[str] = [], - show_progress: bool = True, - ): - """Train the model using the given files""" - - trainer = trainers.BpeTrainer( - vocab_size=vocab_size, - min_frequency=min_frequency, - special_tokens=special_tokens, - limit_alphabet=limit_alphabet, - initial_alphabet=initial_alphabet, - show_progress=show_progress, - ) - if isinstance(files, str): - files = [files] - self._tokenizer.train(files, trainer=trainer) - - def train_from_iterator( - self, - iterator: Union[Iterator[str], Iterator[Iterator[str]]], - vocab_size: int = 30000, - min_frequency: int = 2, - special_tokens: List[Union[str, AddedToken]] = [""], - limit_alphabet: int = 1000, - initial_alphabet: List[str] = [], - show_progress: bool = True, - length: Optional[int] = None, - ): - """Train the model using the given iterator""" - - trainer = trainers.BpeTrainer( - vocab_size=vocab_size, - min_frequency=min_frequency, - special_tokens=special_tokens, - limit_alphabet=limit_alphabet, - initial_alphabet=initial_alphabet, - show_progress=show_progress, - ) - self._tokenizer.train_from_iterator( - iterator, - trainer=trainer, - length=length, - ) diff --git a/bindings/python/py_src/tokenizers/implementations/sentencepiece_unigram.py b/bindings/python/py_src/tokenizers/implementations/sentencepiece_unigram.py deleted file mode 100644 index 5e945a433..000000000 --- a/bindings/python/py_src/tokenizers/implementations/sentencepiece_unigram.py +++ /dev/null @@ -1,196 +0,0 @@ -import json -import os -from typing import Iterator, List, Optional, Union, Tuple - -from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers, trainers -from tokenizers.models import Unigram - -from .base_tokenizer import BaseTokenizer - - -class SentencePieceUnigramTokenizer(BaseTokenizer): - """SentencePiece Unigram Tokenizer - - Represents the Unigram algorithm, with the pretokenization used by SentencePiece - """ - - def __init__( - self, - vocab: Optional[List[Tuple[str, float]]] = None, - replacement: str = "▁", - add_prefix_space: bool = True, - ): - if vocab is not None: - # Let Unigram(..) fail if only one of them is None - tokenizer = Tokenizer(Unigram(vocab)) - else: - tokenizer = Tokenizer(Unigram()) - - tokenizer.normalizer = normalizers.Sequence( - [normalizers.Nmt(), normalizers.NFKC(), normalizers.Replace(Regex(" {2,}"), " ")] - ) - prepend_scheme = "always" if add_prefix_space else "never" - tokenizer.pre_tokenizer = pre_tokenizers.Metaspace(replacement=replacement, prepend_scheme=prepend_scheme) - tokenizer.decoder = decoders.Metaspace(replacement=replacement, prepend_scheme=prepend_scheme) - - parameters = { - "model": "SentencePieceUnigram", - "replacement": replacement, - "add_prefix_space": add_prefix_space, - } - - super().__init__(tokenizer, parameters) - - def train( - self, - files: Union[str, List[str]], - vocab_size: int = 8000, - show_progress: bool = True, - special_tokens: Optional[List[Union[str, AddedToken]]] = None, - initial_alphabet: Optional[List[str]] = None, - unk_token: Optional[str] = None, - ): - """ - Train the model using the given files - - Args: - files (:obj:`List[str]`): - A list of path to the files that we should use for training - vocab_size (:obj:`int`): - The size of the final vocabulary, including all tokens and alphabet. - show_progress (:obj:`bool`): - Whether to show progress bars while training. - special_tokens (:obj:`List[Union[str, AddedToken]]`, `optional`): - A list of special tokens the model should know of. - initial_alphabet (:obj:`List[str]`, `optional`): - A list of characters to include in the initial alphabet, even - if not seen in the training dataset. - If the strings contain more than one character, only the first one - is kept. - unk_token (:obj:`str`, `optional`): - The unknown token to be used by the model. - """ - - if special_tokens is None: - special_tokens = [] - - if initial_alphabet is None: - initial_alphabet = [] - - trainer = trainers.UnigramTrainer( - vocab_size=vocab_size, - special_tokens=special_tokens, - show_progress=show_progress, - initial_alphabet=initial_alphabet, - unk_token=unk_token, - ) - - if isinstance(files, str): - files = [files] - self._tokenizer.train(files, trainer=trainer) - - def train_from_iterator( - self, - iterator: Union[Iterator[str], Iterator[Iterator[str]]], - vocab_size: int = 8000, - show_progress: bool = True, - special_tokens: Optional[List[Union[str, AddedToken]]] = None, - initial_alphabet: Optional[List[str]] = None, - unk_token: Optional[str] = None, - length: Optional[int] = None, - ): - """ - Train the model using the given iterator - - Args: - iterator (:obj:`Union[Iterator[str], Iterator[Iterator[str]]]`): - Any iterator over strings or list of strings - vocab_size (:obj:`int`): - The size of the final vocabulary, including all tokens and alphabet. - show_progress (:obj:`bool`): - Whether to show progress bars while training. - special_tokens (:obj:`List[Union[str, AddedToken]]`, `optional`): - A list of special tokens the model should know of. - initial_alphabet (:obj:`List[str]`, `optional`): - A list of characters to include in the initial alphabet, even - if not seen in the training dataset. - If the strings contain more than one character, only the first one - is kept. - unk_token (:obj:`str`, `optional`): - The unknown token to be used by the model. - length (:obj:`int`, `optional`): - The total number of sequences in the iterator. This is used to - provide meaningful progress tracking - """ - - if special_tokens is None: - special_tokens = [] - - if initial_alphabet is None: - initial_alphabet = [] - - trainer = trainers.UnigramTrainer( - vocab_size=vocab_size, - special_tokens=special_tokens, - show_progress=show_progress, - initial_alphabet=initial_alphabet, - unk_token=unk_token, - ) - - self._tokenizer.train_from_iterator( - iterator, - trainer=trainer, - length=length, - ) - - @staticmethod - def from_spm(filename: str): - try: - import sys - - sys.path.append(".") - - import sentencepiece_model_pb2 as model # type: ignore[import] - except Exception: - raise Exception( - "You don't seem to have the required protobuf file, in order to use this function you need to run `pip install protobuf` and `wget https://raw.githubusercontent.com/google/sentencepiece/master/python/src/sentencepiece/sentencepiece_model_pb2.py` for us to be able to read the intrinsics of your spm_file. `pip install sentencepiece` is not required." - ) - - m = model.ModelProto() - m.ParseFromString(open(filename, "rb").read()) - - precompiled_charsmap = m.normalizer_spec.precompiled_charsmap - vocab = [(piece.piece, piece.score) for piece in m.pieces] - unk_id = m.trainer_spec.unk_id - model_type = m.trainer_spec.model_type - byte_fallback = m.trainer_spec.byte_fallback - if model_type != 1: - raise Exception( - "You're trying to run a `Unigram` model but you're file was trained with a different algorithm" - ) - - replacement = "▁" - add_prefix_space = True - - tokenizer = Tokenizer(Unigram(vocab, unk_id, byte_fallback)) - - if precompiled_charsmap: - tokenizer.normalizer = normalizers.Sequence( - [ - normalizers.Precompiled(precompiled_charsmap), - normalizers.Replace(Regex(" {2,}"), " "), - ] - ) - else: - tokenizer.normalizer = normalizers.Sequence([normalizers.Replace(Regex(" {2,}"), " ")]) - prepend_scheme = "always" if add_prefix_space else "never" - tokenizer.pre_tokenizer = pre_tokenizers.Metaspace(replacement=replacement, prepend_scheme=prepend_scheme) - tokenizer.decoder = decoders.Metaspace(replacement=replacement, prepend_scheme=prepend_scheme) - - parameters = { - "model": "SentencePieceUnigram", - } - - obj = BaseTokenizer.__new__(SentencePieceUnigramTokenizer, tokenizer, parameters) # type: ignore[arg-type] - BaseTokenizer.__init__(obj, tokenizer, parameters) - return obj diff --git a/bindings/python/py_src/tokenizers/models.pyi b/bindings/python/py_src/tokenizers/models.pyi deleted file mode 100644 index a4c8066f4..000000000 --- a/bindings/python/py_src/tokenizers/models.pyi +++ /dev/null @@ -1,421 +0,0 @@ -""" -Models Module -""" - -from collections.abc import Sequence -from typing import Any, final - -from tokenizers import Token - -@final -class BPE(Model): - """ - An implementation of the BPE (Byte-Pair Encoding) algorithm - - Args: - vocab (:obj:`Dict[str, int]`, `optional`): - A dictionary of string keys and their ids :obj:`{"am": 0,...}` - - merges (:obj:`List[Tuple[str, str]]`, `optional`): - A list of pairs of tokens (:obj:`Tuple[str, str]`) :obj:`[("a", "b"),...]` - - cache_capacity (:obj:`int`, `optional`): - The number of words that the BPE cache can contain. The cache allows - to speed-up the process by keeping the result of the merge operations - for a number of words. - - dropout (:obj:`float`, `optional`): - A float between 0 and 1 that represents the BPE dropout to use. - - unk_token (:obj:`str`, `optional`): - The unknown token to be used by the model. - - continuing_subword_prefix (:obj:`str`, `optional`): - The prefix to attach to subword units that don't represent a beginning of word. - - end_of_word_suffix (:obj:`str`, `optional`): - The suffix to attach to subword units that represent an end of word. - - fuse_unk (:obj:`bool`, `optional`): - Whether to fuse any subsequent unknown tokens into a single one - - byte_fallback (:obj:`bool`, `optional`): - Whether to use spm byte-fallback trick (defaults to False) - - ignore_merges (:obj:`bool`, `optional`): - Whether or not to match tokens with the vocab before using merges. - - Example:: - - >>> from tokenizers.models import BPE - >>> # Build an empty model (to be trained) - >>> model = BPE(unk_token="") - >>> # Load from vocabulary and merges files - >>> model = BPE.from_file("vocab.json", "merges.txt") - """ - def __new__( - cls, - /, - vocab: dict[str, int] | str | None = None, - merges: Sequence[tuple[str, str]] | str | None = None, - **kwargs, - ) -> BPE: ... - def _clear_cache(self, /) -> "None": - """ - Clears the internal cache - """ - def _resize_cache(self, /, capacity: int) -> "None": - """ - Resize the internal cache - """ - @property - def byte_fallback(self, /) -> bool: ... - @byte_fallback.setter - def byte_fallback(self, /, byte_fallback: bool) -> None: ... - @property - def continuing_subword_prefix(self, /) -> str | None: ... - @continuing_subword_prefix.setter - def continuing_subword_prefix(self, /, continuing_subword_prefix: str | None) -> None: ... - @property - def dropout(self, /) -> float | None: ... - @dropout.setter - def dropout(self, /, dropout: float | None) -> None: ... - @property - def end_of_word_suffix(self, /) -> str | None: ... - @end_of_word_suffix.setter - def end_of_word_suffix(self, /, end_of_word_suffix: str | None) -> None: ... - @classmethod - def from_file(cls, /, vocab: str, merges: str, **kwargs) -> "BPE": - """ - Instantiate a BPE model from the given files. - - This method is roughly equivalent to doing:: - - vocab, merges = BPE.read_file(vocab_filename, merges_filename) - bpe = BPE(vocab, merges) - - If you don't need to keep the :obj:`vocab, merges` values lying around, - this method is more optimized than manually calling - :meth:`~tokenizers.models.BPE.read_file` to initialize a :class:`~tokenizers.models.BPE` - - Args: - vocab (:obj:`str`): - The path to a :obj:`vocab.json` file - - merges (:obj:`str`): - The path to a :obj:`merges.txt` file - - Returns: - :class:`~tokenizers.models.BPE`: An instance of BPE loaded from these files - """ - @property - def fuse_unk(self, /) -> bool: ... - @fuse_unk.setter - def fuse_unk(self, /, fuse_unk: bool) -> None: ... - @property - def ignore_merges(self, /) -> bool: ... - @ignore_merges.setter - def ignore_merges(self, /, ignore_merges: bool) -> None: ... - @staticmethod - def read_file(vocab: str, merges: str) -> tuple[dict[str, int], list[tuple[str, str]]]: - """ - Read a :obj:`vocab.json` and a :obj:`merges.txt` files - - This method provides a way to read and parse the content of these files, - returning the relevant data structures. If you want to instantiate some BPE models - from memory, this method gives you the expected input from the standard files. - - Args: - vocab (:obj:`str`): - The path to a :obj:`vocab.json` file - - merges (:obj:`str`): - The path to a :obj:`merges.txt` file - - Returns: - A :obj:`Tuple` with the vocab and the merges: - The vocabulary and merges loaded into memory - """ - @property - def unk_token(self, /) -> str | None: ... - @unk_token.setter - def unk_token(self, /, unk_token: str | None) -> None: ... - -class Model: - """ - Base class for all models - - The model represents the actual tokenization algorithm. This is the part that - will contain and manage the learned vocabulary. - - This class cannot be constructed directly. Please use one of the concrete models. - """ - def __getstate__(self, /) -> Any: ... - def __new__(cls, /) -> "Model": ... - def __repr__(self, /) -> str: ... - def __setstate__(self, /, state: Any) -> None: ... - def __str__(self, /) -> str: ... - def get_trainer(self, /) -> Any: - """ - Get the associated :class:`~tokenizers.trainers.Trainer` - - Retrieve the :class:`~tokenizers.trainers.Trainer` associated to this - :class:`~tokenizers.models.Model`. - - Returns: - :class:`~tokenizers.trainers.Trainer`: The Trainer used to train this model - """ - def id_to_token(self, /, id: int) -> str | None: - """ - Get the token associated to an ID - - Args: - id (:obj:`int`): - An ID to convert to a token - - Returns: - :obj:`str`: The token associated to the ID - """ - def save(self, /, folder: str, prefix: str | None = None, name: str | None = None) -> "list[str]": - """ - Save the current model - - Save the current model in the given folder, using the given prefix for the various - files that will get created. - Any file with the same name that already exists in this folder will be overwritten. - - Args: - folder (:obj:`str`): - The path to the target folder in which to save the various files - - prefix (:obj:`str`, `optional`): - An optional prefix, used to prefix each file name - - Returns: - :obj:`List[str]`: The list of saved files - """ - def token_to_id(self, /, token: str) -> int | None: - """ - Get the ID associated to a token - - Args: - token (:obj:`str`): - A token to convert to an ID - - Returns: - :obj:`int`: The ID associated to the token - """ - def tokenize(self, /, sequence: str) -> list[Token]: - """ - Tokenize a sequence - - Args: - sequence (:obj:`str`): - A sequence to tokenize - - Returns: - A :obj:`List` of :class:`~tokenizers.Token`: The generated tokens - """ - -@final -class Unigram(Model): - """ - An implementation of the Unigram algorithm - - The Unigram algorithm is a subword tokenization algorithm based on unigram language - models, as used in SentencePiece. It learns a vocabulary by starting with a large - initial vocabulary and iteratively pruning it using the EM algorithm. - - Args: - vocab (:obj:`List[Tuple[str, float]]`, `optional`): - A list of vocabulary items and their log-probability scores, - e.g. ``[("am", -0.2442), ...]``. If not provided, an empty model is created. - - unk_id (:obj:`int`, `optional`): - The index of the unknown token in the vocabulary list. - - byte_fallback (:obj:`bool`, `optional`, defaults to :obj:`False`): - Whether to use SentencePiece byte fallback for characters not in the vocabulary. - - alpha (:obj:`float`, `optional`): - A float between 0 and 1 that represents the smoothing parameter (temperature) to use. - - nbest_size (:obj:`int`, `optional`): - An integer greater than 0 that represents the maximum number of best paths to consider. - If not set, it samples from the full lattice (i.e. all valid subword segmentations). - - Example:: - - >>> from tokenizers.models import Unigram - >>> # Build an empty model (to be trained) - >>> model = Unigram() - >>> # Build from a vocabulary list - >>> vocab = [("", 0.0), ("hello", -1.0), ("world", -1.5)] - >>> model = Unigram(vocab=vocab, unk_id=0) - """ - def __new__( - cls, - /, - vocab: Sequence[tuple[str, float]] | None = None, - unk_id: int | None = None, - byte_fallback: bool | None = None, - alpha: float | None = None, - nbest_size: int | None = None, - ) -> Unigram: ... - def _clear_cache(self, /) -> "None": - """ - Clears the internal cache - """ - def _resize_cache(self, /, capacity: int) -> "None": - """ - Resize the internal cache - """ - @property - def alpha(self, /) -> float | None: ... - @alpha.setter - def alpha(self, /, alpha: float | None) -> None: ... - @property - def nbest_size(self, /) -> int | None: ... - @nbest_size.setter - def nbest_size(self, /, nbest_size: int | None) -> None: ... - -@final -class WordLevel(Model): - """ - An implementation of the WordLevel algorithm - - Most simple tokenizer model based on mapping tokens to their corresponding id. - - Args: - vocab (:obj:`str`, `optional`): - A dictionary of string keys and their ids :obj:`{"am": 0,...}` - - unk_token (:obj:`str`, `optional`): - The unknown token to be used by the model. - - Example:: - - >>> from tokenizers.models import WordLevel - >>> # Build from a vocabulary dictionary - >>> vocab = {"hello": 0, "world": 1, "": 2} - >>> model = WordLevel(vocab=vocab, unk_token="") - >>> # Load from file - >>> model = WordLevel.from_file("vocab.json", unk_token="") - """ - def __new__(cls, /, vocab: dict[str, int] | str | None = None, unk_token: str | None = None) -> WordLevel: ... - @classmethod - def from_file(cls, /, vocab: str, unk_token: str | None = None) -> "WordLevel": - """ - Instantiate a WordLevel model from the given file - - This method is roughly equivalent to doing:: - - vocab = WordLevel.read_file(vocab_filename) - wordlevel = WordLevel(vocab) - - If you don't need to keep the :obj:`vocab` values lying around, this method is - more optimized than manually calling :meth:`~tokenizers.models.WordLevel.read_file` to - initialize a :class:`~tokenizers.models.WordLevel` - - Args: - vocab (:obj:`str`): - The path to a :obj:`vocab.json` file - - Returns: - :class:`~tokenizers.models.WordLevel`: An instance of WordLevel loaded from file - """ - @staticmethod - def read_file(vocab: str) -> dict[str, int]: - """ - Read a :obj:`vocab.json` - - This method provides a way to read and parse the content of a vocabulary file, - returning the relevant data structures. If you want to instantiate some WordLevel models - from memory, this method gives you the expected input from the standard files. - - Args: - vocab (:obj:`str`): - The path to a :obj:`vocab.json` file - - Returns: - :obj:`Dict[str, int]`: The vocabulary as a :obj:`dict` - """ - @property - def unk_token(self, /) -> str: ... - @unk_token.setter - def unk_token(self, /, unk_token: str) -> None: ... - -@final -class WordPiece(Model): - """ - An implementation of the WordPiece algorithm - - Args: - vocab (:obj:`Dict[str, int]`, `optional`): - A dictionary of string keys and their ids :obj:`{"am": 0,...}` - - unk_token (:obj:`str`, `optional`): - The unknown token to be used by the model. - - max_input_chars_per_word (:obj:`int`, `optional`): - The maximum number of characters to authorize in a single word. - - Example:: - - >>> from tokenizers.models import WordPiece - >>> # Build an empty model (to be trained) - >>> model = WordPiece(unk_token="[UNK]") - >>> # Load from a vocabulary file - >>> model = WordPiece.from_file("vocab.txt") - """ - def __new__(cls, /, vocab: dict[str, int] | str | None = None, **kwargs) -> WordPiece: ... - @property - def continuing_subword_prefix(self, /) -> str: ... - @continuing_subword_prefix.setter - def continuing_subword_prefix(self, /, continuing_subword_prefix: str) -> None: ... - @classmethod - def from_file(cls, /, vocab: str, **kwargs) -> "WordPiece": - """ - Instantiate a WordPiece model from the given file - - This method is roughly equivalent to doing:: - - vocab = WordPiece.read_file(vocab_filename) - wordpiece = WordPiece(vocab) - - If you don't need to keep the :obj:`vocab` values lying around, this method is - more optimized than manually calling :meth:`~tokenizers.models.WordPiece.read_file` to - initialize a :class:`~tokenizers.models.WordPiece` - - Args: - vocab (:obj:`str`): - The path to a :obj:`vocab.txt` file - - Returns: - :class:`~tokenizers.models.WordPiece`: An instance of WordPiece loaded from file - """ - @property - def max_input_chars_per_word(self, /) -> int: ... - @max_input_chars_per_word.setter - def max_input_chars_per_word(self, /, max: int) -> None: ... - @staticmethod - def read_file(vocab: str) -> dict[str, int]: - """ - Read a :obj:`vocab.txt` file - - This method provides a way to read and parse the content of a standard `vocab.txt` - file as used by the WordPiece Model, returning the relevant data structures. If you - want to instantiate some WordPiece models from memory, this method gives you the - expected input from the standard files. - - Args: - vocab (:obj:`str`): - The path to a :obj:`vocab.txt` file - - Returns: - :obj:`Dict[str, int]`: The vocabulary as a :obj:`dict` - """ - @property - def unk_token(self, /) -> str: ... - @unk_token.setter - def unk_token(self, /, unk_token: str) -> None: ... diff --git a/bindings/python/py_src/tokenizers/models/__init__.py b/bindings/python/py_src/tokenizers/models/__init__.py index 5adfc8e25..6ae6ab347 100644 --- a/bindings/python/py_src/tokenizers/models/__init__.py +++ b/bindings/python/py_src/tokenizers/models/__init__.py @@ -1,9 +1,11 @@ -# Generated content DO NOT EDIT +"""The algorithms that turn pre-tokenized pieces into token ids.""" -from .. import models +from .._native import models as _models -BPE = models.BPE -Model = models.Model -Unigram = models.Unigram -WordLevel = models.WordLevel -WordPiece = models.WordPiece +Model = _models.Model +BPE = _models.BPE +WordPiece = _models.WordPiece +WordLevel = _models.WordLevel +Unigram = _models.Unigram + +__all__ = ["Model", "BPE", "WordPiece", "WordLevel", "Unigram"] diff --git a/bindings/python-pipeline/py_src/tokenizers_pipeline/models/__init__.pyi b/bindings/python/py_src/tokenizers/models/__init__.pyi similarity index 100% rename from bindings/python-pipeline/py_src/tokenizers_pipeline/models/__init__.pyi rename to bindings/python/py_src/tokenizers/models/__init__.pyi diff --git a/bindings/python/py_src/tokenizers/normalizers.pyi b/bindings/python/py_src/tokenizers/normalizers.pyi deleted file mode 100644 index 78a9f08f3..000000000 --- a/bindings/python/py_src/tokenizers/normalizers.pyi +++ /dev/null @@ -1,399 +0,0 @@ -""" -Normalizers Module -""" - -from collections.abc import Sequence as Sequence2 -from typing import Any, final - -from tokenizers import NormalizedString, Regex - -@final -class BertNormalizer(Normalizer): - """ - BertNormalizer - - Takes care of normalizing raw text before giving it to a Bert model. - This includes cleaning the text, handling accents, chinese chars and lowercasing - - Args: - clean_text (:obj:`bool`, `optional`, defaults to :obj:`True`): - Whether to clean the text, by removing any control characters - and replacing all whitespaces by the classic one. - - handle_chinese_chars (:obj:`bool`, `optional`, defaults to :obj:`True`): - Whether to handle chinese chars by putting spaces around them. - - strip_accents (:obj:`bool`, `optional`): - Whether to strip all accents. If this option is not specified (ie == None), - then it will be determined by the value for `lowercase` (as in the original Bert). - - lowercase (:obj:`bool`, `optional`, defaults to :obj:`True`): - Whether to lowercase. - - Example:: - - >>> from tokenizers.normalizers import BertNormalizer - >>> normalizer = BertNormalizer(lowercase=True) - >>> normalizer.normalize_str("Héllo WORLD") - 'hello world' - """ - def __new__( - cls, - /, - clean_text: bool = True, - handle_chinese_chars: bool = True, - strip_accents: bool | None = None, - lowercase: bool = True, - ) -> BertNormalizer: ... - @property - def clean_text(self, /) -> bool: ... - @clean_text.setter - def clean_text(self, /, clean_text: bool) -> None: ... - @property - def handle_chinese_chars(self, /) -> bool: ... - @handle_chinese_chars.setter - def handle_chinese_chars(self, /, handle_chinese_chars: bool) -> None: ... - @property - def lowercase(self, /) -> bool: ... - @lowercase.setter - def lowercase(self, /, lowercase: bool) -> None: ... - @property - def strip_accents(self, /) -> bool | None: ... - @strip_accents.setter - def strip_accents(self, /, strip_accents: bool | None) -> None: ... - -@final -class ByteLevel(Normalizer): - """ - Bytelevel Normalizer - - Converts all bytes in the input to their Unicode representation using the GPT-2 - byte-to-unicode mapping. Every byte value (0–255) is mapped to a unique visible - character so that any arbitrary binary input can be tokenized without needing a - special unknown token. - - This normalizer is used together with the - :class:`~tokenizers.pre_tokenizers.ByteLevel` pre-tokenizer and - :class:`~tokenizers.decoders.ByteLevel` decoder. - - Example:: - - >>> from tokenizers.normalizers import ByteLevel - >>> normalizer = ByteLevel() - >>> normalizer.normalize_str("hello\nworld") - 'helloĊworld' - """ - def __new__(cls, /) -> ByteLevel: ... - -@final -class Lowercase(Normalizer): - """ - Lowercase Normalizer - - Converts all text to lowercase using Unicode-aware lowercasing. This is equivalent - to calling :meth:`str.lower` on the input. - - Example:: - - >>> from tokenizers.normalizers import Lowercase - >>> normalizer = Lowercase() - >>> normalizer.normalize_str("Hello World") - 'hello world' - """ - def __new__(cls, /) -> Lowercase: ... - -@final -class NFC(Normalizer): - """ - NFC Unicode Normalizer - - Applies Unicode NFC (Canonical Decomposition, followed by Canonical Composition) - normalization. First decomposes characters, then recomposes them using canonical - composition rules. This produces the canonical composed form. - - Example:: - - >>> from tokenizers.normalizers import NFC - >>> normalizer = NFC() - >>> normalizer.normalize_str("e\u0301") # 'e' + combining accent - 'é' - """ - def __new__(cls, /) -> NFC: ... - -@final -class NFD(Normalizer): - """ - NFD Unicode Normalizer - - Applies Unicode NFD (Canonical Decomposition) normalization. Decomposes characters into - their canonical components. For example, accented characters like ``é`` (U+00E9) are - decomposed into ``e`` (U+0065) + combining accent (U+0301). - - This is often used as a first step before stripping accents with - :class:`~tokenizers.normalizers.StripAccents`. - - Example:: - - >>> from tokenizers.normalizers import NFD - >>> normalizer = NFD() - >>> normalizer.normalize_str("Héllo") - 'He\u0301llo' - """ - def __new__(cls, /) -> NFD: ... - -@final -class NFKC(Normalizer): - """ - NFKC Unicode Normalizer - - Applies Unicode NFKC (Compatibility Decomposition, followed by Canonical Composition) - normalization. Like NFC but also maps compatibility characters to their canonical - equivalents. This is the normalization used by Python's :func:`str.casefold` and - by many NLP pipelines. - - Example:: - - >>> from tokenizers.normalizers import NFKC - >>> normalizer = NFKC() - >>> normalizer.normalize_str("fine caf\u00e9") - 'fine café' - """ - def __new__(cls, /) -> NFKC: ... - -@final -class NFKD(Normalizer): - """ - NFKD Unicode Normalizer - - Applies Unicode NFKD (Compatibility Decomposition) normalization. Like NFD but also - decomposes compatibility characters. For example, the ligature ``fi`` (U+FB01) is - decomposed into ``f`` + ``i``. - - Example:: - - >>> from tokenizers.normalizers import NFKD - >>> normalizer = NFKD() - >>> normalizer.normalize_str("fine") - 'fine' - """ - def __new__(cls, /) -> NFKD: ... - -@final -class Nmt(Normalizer): - """ - Nmt normalizer - - Normalizer used in the Google NMT pipeline. It handles various text cleaning tasks - including removing control characters, normalizing whitespace, and replacing certain - Unicode characters. This is equivalent to the normalization done in the original - SentencePiece NMT preprocessing. - - Example:: - - >>> from tokenizers.normalizers import Nmt - >>> normalizer = Nmt() - >>> normalizer.normalize_str("Hello\x00World") - 'Hello World' - """ - def __new__(cls, /) -> Nmt: ... - -class Normalizer: - """ - Base class for all normalizers - - This class is not supposed to be instantiated directly. Instead, any implementation of a - Normalizer will return an instance of this class when instantiated. - """ - def __getstate__(self, /) -> Any: ... - def __repr__(self, /) -> str: ... - def __setstate__(self, /, state: Any) -> None: ... - def __str__(self, /) -> str: ... - @staticmethod - def custom(obj: Any) -> Normalizer: ... - def normalize(self, /, normalized: NormalizedString | Any) -> None: - """ - Normalize a :class:`~tokenizers.NormalizedString` in-place - - This method allows to modify a :class:`~tokenizers.NormalizedString` to - keep track of the alignment information. If you just want to see the result - of the normalization on a raw string, you can use - :meth:`~tokenizers.normalizers.Normalizer.normalize_str` - - Args: - normalized (:class:`~tokenizers.NormalizedString`): - The normalized string on which to apply this - :class:`~tokenizers.normalizers.Normalizer` - """ - def normalize_str(self, /, sequence: str) -> str: - """ - Normalize the given string - - This method provides a way to visualize the effect of a - :class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment - information. If you need to get/convert offsets, you can use - :meth:`~tokenizers.normalizers.Normalizer.normalize` - - Args: - sequence (:obj:`str`): - A string to normalize - - Returns: - :obj:`str`: A string after normalization - """ - -@final -class Precompiled(Normalizer): - """ - Precompiled normalizer - - A normalizer that uses a precompiled character map built from a SentencePiece model. - This normalizer is automatically extracted from SentencePiece ``.model`` files and - should not be constructed manually — it is used internally for full compatibility - with SentencePiece-based tokenizers. - - Args: - precompiled_charsmap (:obj:`bytes`): - The raw bytes of the precompiled character map, as found inside a - SentencePiece ``.model`` file. - """ - def __new__(cls, /, precompiled_charsmap: Sequence2[int]) -> Precompiled: ... - -@final -class Prepend(Normalizer): - """ - Prepend normalizer - - Prepends a given string to the beginning of the input. This is typically used to - add a meta-symbol such as ``▁`` (U+2581) at the start of each sequence, which is - the convention used by SentencePiece-based models to indicate that a token appears - at the start of a word. - - Args: - prepend (:obj:`str`, defaults to :obj:`"▁"`): - The string to prepend to the input. - - Example:: - - >>> from tokenizers.normalizers import Prepend - >>> normalizer = Prepend("▁") - >>> normalizer.normalize_str("hello") - '▁hello' - """ - def __new__(cls, /, prepend: str = ...) -> Prepend: ... - @property - def prepend(self, /) -> str: ... - @prepend.setter - def prepend(self, /, prepend: str) -> None: ... - -@final -class Replace(Normalizer): - """ - Replace normalizer - - Replaces occurrences of a pattern in the input string with the given content. - The pattern can be either a plain string or a regular expression wrapped in - :class:`~tokenizers.Regex`. - - Args: - pattern (:obj:`str` or :class:`~tokenizers.Regex`): - The pattern to search for. Use a plain string for literal replacement, - or wrap a regex pattern in :class:`~tokenizers.Regex` for regex replacement. - - content (:obj:`str`): - The string to replace each match with. - - Example:: - - >>> from tokenizers import Regex - >>> from tokenizers.normalizers import Replace - >>> # Replace a literal string - >>> Replace(".", " ").normalize_str("hello.world") - 'hello world' - >>> # Replace using a regex - >>> Replace(Regex(r"\s+"), " ").normalize_str("hello world") - 'hello world' - """ - def __new__(cls, /, pattern: str | Regex, content: str) -> Replace: ... - @property - def content(self, /) -> str: ... - @content.setter - def content(self, /, content: str) -> None: ... - @property - def pattern(self, /) -> None: ... - @pattern.setter - def pattern(self, /, _pattern: str | Regex) -> None: ... - -@final -class Sequence(Normalizer): - """ - Allows concatenating multiple other Normalizer as a Sequence. - All the normalizers run in sequence in the given order - - Args: - normalizers (:obj:`List[Normalizer]`): - A list of Normalizer to be run as a sequence - - Example:: - - >>> from tokenizers.normalizers import NFD, Lowercase, StripAccents, Sequence - >>> normalizer = Sequence([NFD(), Lowercase(), StripAccents()]) - >>> normalizer.normalize_str("Héllo Wörld") - 'hello world' - """ - def __getitem__(self, /, index: int) -> Any: ... - def __getnewargs__(self, /) -> tuple: ... - def __len__(self, /) -> int: ... - def __new__(cls, /, normalizers: list) -> Sequence: ... - def __setitem__(self, /, index: int, value: Any) -> None: ... - -@final -class Strip(Normalizer): - """ - Strip normalizer - - Removes leading and/or trailing whitespace from the input string. - - Args: - left (:obj:`bool`, defaults to :obj:`True`): - Whether to strip leading (left) whitespace. - - right (:obj:`bool`, defaults to :obj:`True`): - Whether to strip trailing (right) whitespace. - - Example:: - - >>> from tokenizers.normalizers import Strip - >>> normalizer = Strip() - >>> normalizer.normalize_str(" hello world ") - 'hello world' - >>> Strip(right=False).normalize_str(" hello ") - 'hello ' - """ - def __new__(cls, /, left: bool = True, right: bool = True) -> Strip: ... - @property - def left(self, /) -> bool: ... - @left.setter - def left(self, /, left: bool) -> None: ... - @property - def right(self, /) -> bool: ... - @right.setter - def right(self, /, right: bool) -> None: ... - -@final -class StripAccents(Normalizer): - """ - StripAccents normalizer - - Strips all accent marks (combining diacritical characters) from the input. This - normalizer should typically be used after applying :class:`~tokenizers.normalizers.NFD` - or :class:`~tokenizers.normalizers.NFKD` decomposition, which separates base - characters from their combining accents. - - Example:: - - >>> from tokenizers.normalizers import NFD, StripAccents, Sequence - >>> normalizer = Sequence([NFD(), StripAccents()]) - >>> normalizer.normalize_str("café") - 'cafe' - """ - def __new__(cls, /) -> StripAccents: ... diff --git a/bindings/python/py_src/tokenizers/normalizers/__init__.py b/bindings/python/py_src/tokenizers/normalizers/__init__.py index 86d233bd2..c66373610 100644 --- a/bindings/python/py_src/tokenizers/normalizers/__init__.py +++ b/bindings/python/py_src/tokenizers/normalizers/__init__.py @@ -1,29 +1,31 @@ -from .. import normalizers +"""Text cleanup that runs before the text is split.""" +from .._native import normalizers as _normalizers -Normalizer = normalizers.Normalizer -BertNormalizer = normalizers.BertNormalizer -NFD = normalizers.NFD -NFKD = normalizers.NFKD -NFC = normalizers.NFC -NFKC = normalizers.NFKC -Sequence = normalizers.Sequence -Lowercase = normalizers.Lowercase -Prepend = normalizers.Prepend -Strip = normalizers.Strip -StripAccents = normalizers.StripAccents -Nmt = normalizers.Nmt -Precompiled = normalizers.Precompiled -Replace = normalizers.Replace -ByteLevel = normalizers.ByteLevel +Normalizer = _normalizers.Normalizer +BertNormalizer = _normalizers.BertNormalizer +Lowercase = _normalizers.Lowercase +NFC = _normalizers.NFC +NFD = _normalizers.NFD +NFKC = _normalizers.NFKC +NFKD = _normalizers.NFKD +Prepend = _normalizers.Prepend +Replace = _normalizers.Replace +Sequence = _normalizers.Sequence +Strip = _normalizers.Strip +StripAccents = _normalizers.StripAccents -NORMALIZERS = {"nfc": NFC, "nfd": NFD, "nfkc": NFKC, "nfkd": NFKD} - - -def unicode_normalizer_from_str(normalizer: str) -> Normalizer: - if normalizer not in NORMALIZERS: - raise ValueError( - "{} is not a known unicode normalizer. Available are {}".format(normalizer, NORMALIZERS.keys()) - ) - - return NORMALIZERS[normalizer]() +__all__ = [ + "Normalizer", + "BertNormalizer", + "Lowercase", + "NFC", + "NFD", + "NFKC", + "NFKD", + "Prepend", + "Replace", + "Sequence", + "Strip", + "StripAccents", +] diff --git a/bindings/python-pipeline/py_src/tokenizers_pipeline/normalizers/__init__.pyi b/bindings/python/py_src/tokenizers/normalizers/__init__.pyi similarity index 100% rename from bindings/python-pipeline/py_src/tokenizers_pipeline/normalizers/__init__.pyi rename to bindings/python/py_src/tokenizers/normalizers/__init__.pyi diff --git a/bindings/python/py_src/tokenizers/pre_tokenizers.pyi b/bindings/python/py_src/tokenizers/pre_tokenizers.pyi deleted file mode 100644 index 41c2abc21..000000000 --- a/bindings/python/py_src/tokenizers/pre_tokenizers.pyi +++ /dev/null @@ -1,385 +0,0 @@ -""" -PreTokenizers Module -""" - -from typing import Any, final - -from _typeshed import Incomplete - -from tokenizers import PreTokenizedString, Regex - -@final -class BertPreTokenizer(PreTokenizer): - """ - BertPreTokenizer - - This pre-tokenizer splits tokens on whitespace and punctuation. Each occurrence of - a punctuation character will be treated as a separate token. This is the pre-tokenizer - used by the original BERT model. - - Example:: - - >>> from tokenizers.pre_tokenizers import BertPreTokenizer - >>> pre_tokenizer = BertPreTokenizer() - >>> pre_tokenizer.pre_tokenize_str("Hello, I'm a single sentence!") - [('Hello', (0, 5)), (',', (5, 6)), ('I', (7, 8)), ("'", (8, 9)), ('m', (9, 10)), ('a', (11, 12)), ('single', (13, 19)), ('sentence', (20, 28)), ('!', (28, 29))] - """ - def __new__(cls, /) -> BertPreTokenizer: ... - -@final -class ByteLevel(PreTokenizer): - """ - ByteLevel PreTokenizer - - This pre-tokenizer takes care of replacing all bytes of the given string - with a corresponding representation, as well as splitting into words. - - Args: - add_prefix_space (:obj:`bool`, `optional`, defaults to :obj:`True`): - Whether to add a space to the first word if there isn't already one. This - lets us treat `hello` exactly like `say hello`. - use_regex (:obj:`bool`, `optional`, defaults to :obj:`True`): - Set this to :obj:`False` to prevent this `pre_tokenizer` from using - the GPT2 specific regexp for spliting on whitespace. - - Example:: - - >>> from tokenizers.pre_tokenizers import ByteLevel - >>> pre_tokenizer = ByteLevel() - >>> pre_tokenizer.pre_tokenize_str("Hello my friend, how is it going?") - [('ĠHello', (0, 5)), ('Ġmy', (5, 8)), ('Ġfriend,', (8, 15)), ('Ġhow', (15, 19)), ('Ġis', (19, 22)), ('Ġit', (22, 25)), ('Ġgoing?', (25, 32))] - """ - def __new__( - cls, /, add_prefix_space: bool = True, trim_offsets: bool = True, use_regex: bool = True, **_kwargs - ) -> ByteLevel: ... - @property - def add_prefix_space(self, /) -> bool: ... - @add_prefix_space.setter - def add_prefix_space(self, /, add_prefix_space: bool) -> None: ... - @staticmethod - def alphabet() -> list[str]: - """ - Returns the alphabet used by this PreTokenizer. - - Since the ByteLevel works as its name suggests, at the byte level, it - encodes each byte value to a unique visible character. This means that there is a - total of 256 different characters composing this alphabet. - - Returns: - :obj:`List[str]`: A list of characters that compose the alphabet - """ - @property - def trim_offsets(self, /) -> bool: ... - @trim_offsets.setter - def trim_offsets(self, /, trim_offsets: bool) -> None: ... - @property - def use_regex(self, /) -> bool: ... - @use_regex.setter - def use_regex(self, /, use_regex: bool) -> None: ... - -@final -class CharDelimiterSplit(PreTokenizer): - """ - This pre-tokenizer simply splits on the provided char. Works like :meth:`str.split` - with a single-character delimiter. - - Args: - delimiter (:obj:`str`): - The single character that will be used to split the input. The delimiter - is removed from the output. - - Example:: - - >>> from tokenizers.pre_tokenizers import CharDelimiterSplit - >>> pre_tokenizer = CharDelimiterSplit("x") - >>> pre_tokenizer.pre_tokenize_str("helloxthere") - [('hello', (0, 5)), ('there', (6, 11))] - """ - def __getnewargs__(self, /) -> tuple: ... - def __new__(cls, /, delimiter: str) -> CharDelimiterSplit: ... - @property - def delimiter(self, /) -> str: ... - @delimiter.setter - def delimiter(self, /, delimiter: str) -> None: ... - -@final -class Digits(PreTokenizer): - """ - This pre-tokenizer simply splits using the digits in separate tokens - - Args: - individual_digits (:obj:`bool`, `optional`, defaults to :obj:`False`): - If set to True, digits will each be separated as follows:: - - "Call 123 please" -> "Call ", "1", "2", "3", " please" - - If set to False, digits will grouped as follows:: - - "Call 123 please" -> "Call ", "123", " please" - """ - def __new__(cls, /, individual_digits: bool = False) -> Digits: ... - @property - def individual_digits(self, /) -> bool: ... - @individual_digits.setter - def individual_digits(self, /, individual_digits: bool) -> None: ... - -@final -class FixedLength(PreTokenizer): - """ - This pre-tokenizer splits the text into fixed length chunks as used - [here](https://www.biorxiv.org/content/10.1101/2023.01.11.523679v1.full) - - Args: - length (:obj:`int`, `optional`, defaults to :obj:`5`): - The length of the chunks to split the text into. - - Strings are split on the character level rather than the byte level to avoid - splitting unicode characters consisting of multiple bytes. - - Example:: - - >>> from tokenizers.pre_tokenizers import FixedLength - >>> pre_tokenizer = FixedLength(length=3) - >>> pre_tokenizer.pre_tokenize_str("Hello") - [('Hel', (0, 3)), ('lo', (3, 5))] - """ - def __new__(cls, /, length: int = 5) -> FixedLength: ... - @property - def length(self, /) -> int: ... - @length.setter - def length(self, /, length: int) -> None: ... - -@final -class Metaspace(PreTokenizer): - """ - Metaspace pre-tokenizer - - This pre-tokenizer replaces any whitespace by the provided replacement character. - It then tries to split on these spaces. - - Args: - replacement (:obj:`str`, `optional`, defaults to :obj:`▁`): - The replacement character. Must be exactly one character. By default we - use the `▁` (U+2581) meta symbol (Same as in SentencePiece). - - prepend_scheme (:obj:`str`, `optional`, defaults to :obj:`"always"`): - Whether to add a space to the first word if there isn't already one. This - lets us treat `hello` exactly like `say hello`. - Choices: "always", "never", "first". First means the space is only added on the first - token (relevant when special tokens are used or other pre_tokenizer are used). - - Example:: - - >>> from tokenizers.pre_tokenizers import Metaspace - >>> pre_tokenizer = Metaspace() - >>> pre_tokenizer.pre_tokenize_str("Hello my friend") - [('▁Hello', (0, 5)), ('▁my', (6, 8)), ('▁friend', (9, 15))] - """ - def __new__(cls, /, replacement: str = "▁", prepend_scheme: str = ..., split: bool = True) -> Metaspace: ... - @property - def prepend_scheme(self, /) -> str: ... - @prepend_scheme.setter - def prepend_scheme(self, /, prepend_scheme: str) -> None: ... - @property - def replacement(self, /) -> str: ... - @replacement.setter - def replacement(self, /, replacement: str) -> None: ... - @property - def split(self, /) -> bool: ... - @split.setter - def split(self, /, split: bool) -> None: ... - -class PreTokenizer: - """ - Base class for all pre-tokenizers - - This class is not supposed to be instantiated directly. Instead, any implementation of a - PreTokenizer will return an instance of this class when instantiated. - """ - def __getstate__(self, /) -> Any: ... - def __repr__(self, /) -> str: ... - def __setstate__(self, /, state: Any) -> None: ... - def __str__(self, /) -> str: ... - @staticmethod - def custom(pretok: Any) -> PreTokenizer: ... - def pre_tokenize(self, /, pretok: PreTokenizedString) -> None: - """ - Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place - - This method allows to modify a :class:`~tokenizers.PreTokenizedString` to - keep track of the pre-tokenization, and leverage the capabilities of the - :class:`~tokenizers.PreTokenizedString`. If you just want to see the result of - the pre-tokenization of a raw string, you can use - :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str` - - Args: - pretok (:class:`~tokenizers.PreTokenizedString): - The pre-tokenized string on which to apply this - :class:`~tokenizers.pre_tokenizers.PreTokenizer` - """ - def pre_tokenize_str(self, /, s: str) -> list[tuple[str, tuple[int, int]]]: - """ - Pre tokenize the given string - - This method provides a way to visualize the effect of a - :class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the - alignment, nor does it provide all the capabilities of the - :class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use - :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize` - - Args: - sequence (:obj:`str`): - A string to pre-tokeize - - Returns: - :obj:`List[Tuple[str, Offsets]]`: - A list of tuple with the pre-tokenized parts and their offsets - """ - -@final -class Punctuation(PreTokenizer): - """ - This pre-tokenizer simply splits on punctuation as individual characters. - - Args: - behavior (:class:`~tokenizers.SplitDelimiterBehavior`): - The behavior to use when splitting. - Choices: "removed", "isolated" (default), "merged_with_previous", "merged_with_next", - "contiguous" - - Example:: - - >>> from tokenizers.pre_tokenizers import Punctuation - >>> pre_tokenizer = Punctuation() - >>> pre_tokenizer.pre_tokenize_str("Hello, how are you?") - [('Hello', (0, 5)), (',', (5, 6)), ('how', (7, 10)), ('are', (11, 14)), ('you', (15, 18)), ('?', (18, 19))] - """ - def __new__(cls, /, behavior: Incomplete = ...) -> Punctuation: ... - @property - def behavior(self, /) -> str: ... - @behavior.setter - def behavior(self, /, behavior: str) -> None: ... - -@final -class Sequence(PreTokenizer): - """ - This pre-tokenizer composes other pre-tokenizers and applies them in sequence. - Each pre-tokenizer in the list is applied to the output of the previous one, - allowing complex tokenization strategies to be built by chaining simpler components. - - Args: - pretokenizers (:obj:`List[PreTokenizer]`): - A list of :class:`~tokenizers.pre_tokenizers.PreTokenizer` to be applied - in sequence. - - Example:: - - >>> from tokenizers.pre_tokenizers import Punctuation, Whitespace, Sequence - >>> pre_tokenizer = Sequence([Whitespace(), Punctuation()]) - >>> pre_tokenizer.pre_tokenize_str("Hello, world!") - [('Hello', (0, 5)), (',', (5, 6)), ('world', (7, 12)), ('!', (12, 13))] - """ - def __getitem__(self, /, index: int) -> Any: ... - def __getnewargs__(self, /) -> tuple: ... - def __new__(cls, /, pre_tokenizers: list) -> Sequence: ... - def __setitem__(self, /, index: int, value: Any) -> None: ... - -@final -class Split(PreTokenizer): - """ - Split PreTokenizer - - This versatile pre-tokenizer splits using the provided pattern and - according to the provided behavior. The pattern can be inverted by - making use of the invert flag. - - Args: - pattern (:obj:`str` or :class:`~tokenizers.Regex`): - A pattern used to split the string. Usually a string or a regex built with `tokenizers.Regex`. - If you want to use a regex pattern, it has to be wrapped around a `tokenizers.Regex`, - otherwise we consider is as a string pattern. For example `pattern="|"` - means you want to split on `|` (imagine a csv file for example), while - `pattern=tokenizers.Regex("1|2")` means you split on either '1' or '2'. - behavior (:class:`~tokenizers.SplitDelimiterBehavior`): - The behavior to use when splitting. - Choices: "removed", "isolated", "merged_with_previous", "merged_with_next", - "contiguous" - - invert (:obj:`bool`, `optional`, defaults to :obj:`False`): - Whether to invert the pattern. - - Example:: - - >>> from tokenizers import Regex - >>> from tokenizers.pre_tokenizers import Split - >>> # Split on commas, removing them - >>> pre_tokenizer = Split(",", behavior="removed") - >>> pre_tokenizer.pre_tokenize_str("one,two,three") - [('one', (0, 3)), ('two', (4, 7)), ('three', (8, 13))] - >>> # Split using a regex, keeping the delimiter isolated - >>> Split(Regex(r"\s+"), behavior="isolated").pre_tokenize_str("hello world") - [('hello', (0, 5)), (' ', (5, 8)), ('world', (8, 13))] - """ - def __getnewargs__(self, /) -> tuple: ... - def __new__(cls, /, pattern: str | Regex, behavior: Incomplete, invert: bool = False) -> Split: ... - @property - def behavior(self, /) -> str: ... - @behavior.setter - def behavior(self, /, behavior: str) -> None: ... - @property - def invert(self, /) -> bool: ... - @invert.setter - def invert(self, /, invert: bool) -> None: ... - @property - def pattern(self, /) -> None: ... - @pattern.setter - def pattern(self, /, _pattern: str | Regex) -> None: ... - -@final -class UnicodeScripts(PreTokenizer): - """ - This pre-tokenizer splits on characters that belong to different language families. - It roughly follows the SentencePiece script boundaries, with Hiragana and Katakana - fused into the Han script category. This mimics the SentencePiece Unigram - implementation and is useful for multilingual models that need to handle CJK text. - - Example:: - - >>> from tokenizers.pre_tokenizers import UnicodeScripts - >>> pre_tokenizer = UnicodeScripts() - >>> pre_tokenizer.pre_tokenize_str("どこ Where") - [('どこ', (0, 2)), ('Where', (3, 8))] - """ - def __new__(cls, /) -> UnicodeScripts: ... - -@final -class Whitespace(PreTokenizer): - """ - This pre-tokenizer splits on word boundaries according to the ``\w+|[^\w\s]+`` - regex pattern. It splits on word characters or characters that aren't words or - whitespaces (punctuation such as hyphens, apostrophes, commas, etc.). - - Example:: - - >>> from tokenizers.pre_tokenizers import Whitespace - >>> pre_tokenizer = Whitespace() - >>> pre_tokenizer.pre_tokenize_str("Hello, world! Let's tokenize.") - [('Hello', (0, 5)), (',', (5, 6)), ('world', (7, 12)), ('!', (12, 13)), ('Let', (14, 17)), ("'", (17, 18)), ('s', (18, 19)), ('tokenize', (20, 28)), ('.', (28, 29))] - """ - def __new__(cls, /) -> Whitespace: ... - -@final -class WhitespaceSplit(PreTokenizer): - """ - This pre-tokenizer simply splits on whitespace. Works like :meth:`str.split` with no - arguments — it splits on any whitespace and discards the whitespace tokens. Unlike - :class:`~tokenizers.pre_tokenizers.Whitespace`, it does not split on punctuation. - - Example:: - - >>> from tokenizers.pre_tokenizers import WhitespaceSplit - >>> pre_tokenizer = WhitespaceSplit() - >>> pre_tokenizer.pre_tokenize_str("Hello, world! How are you?") - [('Hello,', (0, 6)), ('world!', (7, 13)), ('How', (14, 17)), ('are', (18, 21)), ('you?', (22, 26))] - """ - def __new__(cls, /) -> WhitespaceSplit: ... diff --git a/bindings/python/py_src/tokenizers/pre_tokenizers/__init__.py b/bindings/python/py_src/tokenizers/pre_tokenizers/__init__.py index 54bf038c0..49c12046d 100644 --- a/bindings/python/py_src/tokenizers/pre_tokenizers/__init__.py +++ b/bindings/python/py_src/tokenizers/pre_tokenizers/__init__.py @@ -1,17 +1,31 @@ -# Generated content DO NOT EDIT +"""How text is cut into pieces before the model runs.""" -from .. import pre_tokenizers +from .._native import pre_tokenizers as _pre_tokenizers -BertPreTokenizer = pre_tokenizers.BertPreTokenizer -ByteLevel = pre_tokenizers.ByteLevel -CharDelimiterSplit = pre_tokenizers.CharDelimiterSplit -Digits = pre_tokenizers.Digits -FixedLength = pre_tokenizers.FixedLength -Metaspace = pre_tokenizers.Metaspace -PreTokenizer = pre_tokenizers.PreTokenizer -Punctuation = pre_tokenizers.Punctuation -Sequence = pre_tokenizers.Sequence -Split = pre_tokenizers.Split -UnicodeScripts = pre_tokenizers.UnicodeScripts -Whitespace = pre_tokenizers.Whitespace -WhitespaceSplit = pre_tokenizers.WhitespaceSplit +PreTokenizer = _pre_tokenizers.PreTokenizer +BertPreTokenizer = _pre_tokenizers.BertPreTokenizer +ByteLevel = _pre_tokenizers.ByteLevel +CharDelimiterSplit = _pre_tokenizers.CharDelimiterSplit +Digits = _pre_tokenizers.Digits +FixedLength = _pre_tokenizers.FixedLength +Punctuation = _pre_tokenizers.Punctuation +Sequence = _pre_tokenizers.Sequence +Split = _pre_tokenizers.Split +UnicodeScripts = _pre_tokenizers.UnicodeScripts +Whitespace = _pre_tokenizers.Whitespace +WhitespaceSplit = _pre_tokenizers.WhitespaceSplit + +__all__ = [ + "PreTokenizer", + "BertPreTokenizer", + "ByteLevel", + "CharDelimiterSplit", + "Digits", + "FixedLength", + "Punctuation", + "Sequence", + "Split", + "UnicodeScripts", + "Whitespace", + "WhitespaceSplit", +] diff --git a/bindings/python-pipeline/py_src/tokenizers_pipeline/pre_tokenizers/__init__.pyi b/bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi similarity index 100% rename from bindings/python-pipeline/py_src/tokenizers_pipeline/pre_tokenizers/__init__.pyi rename to bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi diff --git a/bindings/python/py_src/tokenizers/processors.pyi b/bindings/python/py_src/tokenizers/processors.pyi deleted file mode 100644 index 172f9d728..000000000 --- a/bindings/python/py_src/tokenizers/processors.pyi +++ /dev/null @@ -1,293 +0,0 @@ -""" -Processors Module -""" - -from collections.abc import Sequence as Sequence2 -from typing import Any, final - -from _typeshed import Incomplete - -from tokenizers import Encoding - -@final -class BertProcessing(PostProcessor): - """ - This post-processor takes care of adding the special tokens needed by - a Bert model: - - - a SEP token - - a CLS token - - Args: - sep (:obj:`Tuple[str, int]`): - A tuple with the string representation of the SEP token, and its id - - cls (:obj:`Tuple[str, int]`): - A tuple with the string representation of the CLS token, and its id - - Example:: - - >>> from tokenizers.processors import BertProcessing - >>> processor = BertProcessing(("[SEP]", 102), ("[CLS]", 101)) - >>> processor.process(encoding) - # Encoding with [CLS] at start and [SEP] at end - """ - def __getnewargs__(self, /) -> tuple: ... - def __new__(cls, /, sep: tuple[str, int], cls_token: tuple[str, int]) -> BertProcessing: ... - @property - def cls(self, /) -> tuple: ... - @cls.setter - def cls(self, /, cls: tuple) -> None: ... - @property - def sep(self, /) -> tuple: ... - @sep.setter - def sep(self, /, sep: tuple) -> None: ... - -@final -class ByteLevel(PostProcessor): - """ - This post-processor takes care of trimming the offsets. - - By default, the ByteLevel BPE might include whitespaces in the produced tokens. If you don't - want the offsets to include these whitespaces, then this PostProcessor must be used. - - Args: - trim_offsets (:obj:`bool`): - Whether to trim the whitespaces from the produced offsets. - - add_prefix_space (:obj:`bool`, `optional`, defaults to :obj:`True`): - If :obj:`True`, keeps the first token's offset as is. If :obj:`False`, increments - the start of the first token's offset by 1. Only has an effect if :obj:`trim_offsets` - is set to :obj:`True`. - - Example:: - - >>> from tokenizers.processors import ByteLevel - >>> processor = ByteLevel(trim_offsets=True) - >>> # Offsets will be trimmed to exclude leading whitespace bytes - """ - def __new__( - cls, - /, - add_prefix_space: bool | None = None, - trim_offsets: bool | None = None, - use_regex: bool | None = None, - **_kwargs, - ) -> ByteLevel: ... - @property - def add_prefix_space(self, /) -> bool: ... - @add_prefix_space.setter - def add_prefix_space(self, /, add_prefix_space: bool) -> None: ... - @property - def trim_offsets(self, /) -> bool: ... - @trim_offsets.setter - def trim_offsets(self, /, trim_offsets: bool) -> None: ... - @property - def use_regex(self, /) -> bool: ... - @use_regex.setter - def use_regex(self, /, use_regex: bool) -> None: ... - -class PostProcessor: - """ - Base class for all post-processors - - This class is not supposed to be instantiated directly. Instead, any implementation of - a PostProcessor will return an instance of this class when instantiated. - """ - def __getstate__(self, /) -> Any: ... - def __repr__(self, /) -> str: ... - def __setstate__(self, /, state: Any) -> None: ... - def __str__(self, /) -> str: ... - def num_special_tokens_to_add(self, /, is_pair: bool) -> int: - """ - Return the number of special tokens that would be added for single/pair sentences. - - Args: - is_pair (:obj:`bool`): - Whether the input would be a pair of sequences - - Returns: - :obj:`int`: The number of tokens to add - """ - def process( - self, /, encoding: Encoding, pair: Encoding | None = None, add_special_tokens: bool = True - ) -> "Encoding": - """ - Post-process the given encodings, generating the final one - - Args: - encoding (:class:`~tokenizers.Encoding`): - The encoding for the first sequence - - pair (:class:`~tokenizers.Encoding`, `optional`): - The encoding for the pair sequence - - add_special_tokens (:obj:`bool`): - Whether to add the special tokens - - Return: - :class:`~tokenizers.Encoding`: The final encoding - """ - -@final -class RobertaProcessing(PostProcessor): - """ - This post-processor takes care of adding the special tokens needed by - a Roberta model: - - - a SEP token - - a CLS token - - It also takes care of trimming the offsets. - By default, the ByteLevel BPE might include whitespaces in the produced tokens. If you don't - want the offsets to include these whitespaces, then this PostProcessor should be initialized - with :obj:`trim_offsets=True` - - Args: - sep (:obj:`Tuple[str, int]`): - A tuple with the string representation of the SEP token, and its id - - cls (:obj:`Tuple[str, int]`): - A tuple with the string representation of the CLS token, and its id - - trim_offsets (:obj:`bool`, `optional`, defaults to :obj:`True`): - Whether to trim the whitespaces from the produced offsets. - - add_prefix_space (:obj:`bool`, `optional`, defaults to :obj:`True`): - Whether the add_prefix_space option was enabled during pre-tokenization. This - is relevant because it defines the way the offsets are trimmed out. - - Example:: - - >>> from tokenizers.processors import RobertaProcessing - >>> processor = RobertaProcessing(("", 2), ("", 0)) - >>> processor.process(encoding) - # Encoding with at start and at end - """ - def __getnewargs__(self, /) -> tuple: ... - def __new__( - cls, - /, - sep: tuple[str, int], - cls_token: tuple[str, int], - trim_offsets: bool = True, - add_prefix_space: bool = True, - ) -> RobertaProcessing: ... - @property - def add_prefix_space(self, /) -> bool: ... - @add_prefix_space.setter - def add_prefix_space(self, /, add_prefix_space: bool) -> None: ... - @property - def cls(self, /) -> tuple: ... - @cls.setter - def cls(self, /, cls: tuple) -> None: ... - @property - def sep(self, /) -> tuple: ... - @sep.setter - def sep(self, /, sep: tuple) -> None: ... - @property - def trim_offsets(self, /) -> bool: ... - @trim_offsets.setter - def trim_offsets(self, /, trim_offsets: bool) -> None: ... - -@final -class Sequence(PostProcessor): - """ - Sequence Processor - - Chains multiple post-processors together, applying them in order. Each processor - in the sequence processes the output of the previous one. - - Args: - processors (:obj:`List[PostProcessor]`): - The list of post-processors to chain together. - - Example:: - - >>> from tokenizers.processors import BertProcessing, ByteLevel, Sequence - >>> processor = Sequence([ByteLevel(trim_offsets=True), BertProcessing(("[SEP]", 102), ("[CLS]", 101))]) - """ - def __getitem__(self, /, index: int) -> Any: ... - def __getnewargs__(self, /) -> tuple: ... - def __new__(cls, /, processors_py: list) -> Sequence: ... - def __setitem__(self, /, index: int, value: Any) -> None: ... - -@final -class TemplateProcessing(PostProcessor): - """ - Provides a way to specify templates in order to add the special tokens to each - input sequence as relevant. - - Let's take :obj:`BERT` tokenizer as an example. It uses two special tokens, used to - delimitate each sequence. :obj:`[CLS]` is always used at the beginning of the first - sequence, and :obj:`[SEP]` is added at the end of both the first, and the pair - sequences. The final result looks like this: - - - Single sequence: :obj:`[CLS] Hello there [SEP]` - - Pair sequences: :obj:`[CLS] My name is Anthony [SEP] What is my name? [SEP]` - - With the type ids as following:: - - [CLS] ... [SEP] ... [SEP] - 0 0 0 1 1 - - You can achieve such behavior using a TemplateProcessing:: - - TemplateProcessing( - single="[CLS] $0 [SEP]", - pair="[CLS] $A [SEP] $B:1 [SEP]:1", - special_tokens=[("[CLS]", 1), ("[SEP]", 0)], - ) - - In this example, each input sequence is identified using a ``$`` construct. This identifier - lets us specify each input sequence, and the type_id to use. When nothing is specified, - it uses the default values. Here are the different ways to specify it: - - - Specifying the sequence, with default ``type_id == 0``: ``$A`` or ``$B`` - - Specifying the `type_id` with default ``sequence == A``: ``$0``, ``$1``, ``$2``, ... - - Specifying both: ``$A:0``, ``$B:1``, ... - - The same construct is used for special tokens: ``(:)?``. - - **Warning**: You must ensure that you are giving the correct tokens/ids as these - will be added to the Encoding without any further check. If the given ids correspond - to something totally different in a `Tokenizer` using this `PostProcessor`, it - might lead to unexpected results. - - Args: - single (:obj:`Template`): - The template used for single sequences - - pair (:obj:`Template`): - The template used when both sequences are specified - - special_tokens (:obj:`Tokens`): - The list of special tokens used in each sequences - - Types: - - Template (:obj:`str` or :obj:`List`): - - If a :obj:`str` is provided, the whitespace is used as delimiter between tokens - - If a :obj:`List[str]` is provided, a list of tokens - - Tokens (:obj:`List[Union[Tuple[int, str], Tuple[str, int], dict]]`): - - A :obj:`Tuple` with both a token and its associated ID, in any order - - A :obj:`dict` with the following keys: - - "id": :obj:`str` => The special token id, as specified in the Template - - "ids": :obj:`List[int]` => The associated IDs - - "tokens": :obj:`List[str]` => The associated tokens - - The given dict expects the provided :obj:`ids` and :obj:`tokens` lists to have - the same length. - """ - def __new__( - cls, - /, - single: Incomplete | None = None, - pair: Incomplete | None = None, - special_tokens: Sequence2[Incomplete] | None = None, - ) -> TemplateProcessing: ... - @property - def single(self, /) -> str: ... - @single.setter - def single(self, /, single: Incomplete) -> None: ... diff --git a/bindings/python/py_src/tokenizers/processors/__init__.py b/bindings/python/py_src/tokenizers/processors/__init__.py deleted file mode 100644 index 9bc48012a..000000000 --- a/bindings/python/py_src/tokenizers/processors/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# Generated content DO NOT EDIT - -from .. import processors - -BertProcessing = processors.BertProcessing -ByteLevel = processors.ByteLevel -PostProcessor = processors.PostProcessor -RobertaProcessing = processors.RobertaProcessing -Sequence = processors.Sequence -TemplateProcessing = processors.TemplateProcessing diff --git a/bindings/python/py_src/tokenizers/tokenizers.pyi b/bindings/python/py_src/tokenizers/tokenizers.pyi deleted file mode 100644 index 6f26cd674..000000000 --- a/bindings/python/py_src/tokenizers/tokenizers.pyi +++ /dev/null @@ -1,17 +0,0 @@ -# Generated content DO NOT EDIT -from tokenizers import ( - AddedToken as AddedToken, - Encoding as Encoding, - NormalizedString as NormalizedString, - PreTokenizedString as PreTokenizedString, - Regex as Regex, - Token as Token, - Tokenizer as Tokenizer, - __version__ as __version__, - decoders as decoders, - models as models, - normalizers as normalizers, - pre_tokenizers as pre_tokenizers, - processors as processors, - trainers as trainers, -) diff --git a/bindings/python/py_src/tokenizers/tools/__init__.py b/bindings/python/py_src/tokenizers/tools/__init__.py deleted file mode 100644 index f941e2ed3..000000000 --- a/bindings/python/py_src/tokenizers/tools/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .visualizer import Annotation, EncodingVisualizer diff --git a/bindings/python/py_src/tokenizers/tools/visualizer-styles.css b/bindings/python/py_src/tokenizers/tools/visualizer-styles.css deleted file mode 100644 index f54fde45a..000000000 --- a/bindings/python/py_src/tokenizers/tools/visualizer-styles.css +++ /dev/null @@ -1,170 +0,0 @@ -.tokenized-text { - width:100%; - padding:2rem; - max-height: 400px; - overflow-y: auto; - box-sizing:border-box; - line-height:4rem; /* Lots of space between lines */ - font-family: "Roboto Light", "Ubuntu Light", "Ubuntu", monospace; - box-shadow: 2px 2px 2px rgba(0,0,0,0.2); - background-color: rgba(0,0,0,0.01); - letter-spacing:2px; /* Give some extra separation between chars */ -} -.non-token{ - /* White space and other things the tokenizer ignores*/ - white-space: pre; - letter-spacing:4px; - border-top:1px solid #A0A0A0; /* A gentle border on top and bottom makes tabs more ovious*/ - border-bottom:1px solid #A0A0A0; - line-height: 1rem; - height: calc(100% - 2px); -} - -.token { - white-space: pre; - position:relative; - color:black; - letter-spacing:2px; -} - -.annotation{ - white-space:nowrap; /* Important - ensures that annotations appears even if the annotated text wraps a line */ - border-radius:4px; - position:relative; - width:fit-content; -} -.annotation:before { - /*The before holds the text and the after holds the background*/ - z-index:1000; /* Make sure this is above the background */ - content:attr(data-label); /* The annotations label is on a data attribute */ - color:white; - position:absolute; - font-size:1rem; - text-align:center; - font-weight:bold; - - top:1.75rem; - line-height:0; - left:0; - width:100%; - padding:0.5rem 0; - /* These make it so an annotation doesn't stretch beyond the annotated text if the label is longer*/ - overflow: hidden; - white-space: nowrap; - text-overflow:ellipsis; -} - -.annotation:after { - content:attr(data-label); /* The content defines the width of the annotation*/ - position:absolute; - font-size:0.75rem; - text-align:center; - font-weight:bold; - text-overflow:ellipsis; - top:1.75rem; - line-height:0; - overflow: hidden; - white-space: nowrap; - - left:0; - width:100%; /* 100% of the parent, which is the annotation whose width is the tokens inside it*/ - - padding:0.5rem 0; - /* Nast hack below: - We set the annotations color in code because we don't know the colors at css time. - But you can't pass a color as a data attribute to get it into the pseudo element (this thing) - So to get around that, annotations have the color set on them with a style attribute and then we - can get the color with currentColor. - Annotations wrap tokens and tokens set the color back to black - */ - background-color: currentColor; -} -.annotation:hover::after, .annotation:hover::before{ - /* When the user hovers over an annotation expand the label to display in full - */ - min-width: fit-content; -} - -.annotation:hover{ - /* Emphasize the annotation start end with a border on hover*/ - border-color: currentColor; - border: 2px solid; -} -.special-token:not(:empty){ - /* - A none empty special token is like UNK (as opposed to CLS which has no representation in the text ) - */ - position:relative; -} -.special-token:empty::before{ - /* Special tokens that don't have text are displayed as pseudo elements so we dont select them with the mouse*/ - content:attr(data-stok); - background:#202020; - font-size:0.75rem; - color:white; - margin: 0 0.25rem; - padding: 0.25rem; - border-radius:4px -} - -.special-token:not(:empty):before { - /* Special tokens that have text (UNK) are displayed above the actual text*/ - content:attr(data-stok); - position:absolute; - bottom:1.75rem; - min-width:100%; - width:100%; - height:1rem; - line-height:1rem; - font-size:1rem; - text-align:center; - color:white; - font-weight:bold; - background:#202020; - border-radius:10%; -} -/* -We want to alternate the color of tokens, but we can't use nth child because tokens might be broken up by annotations -instead we apply even and odd class at generation time and color them that way - */ -.even-token{ - background:#DCDCDC ; - border: 1px solid #DCDCDC; -} -.odd-token{ - background:#A0A0A0; - border: 1px solid #A0A0A0; -} -.even-token.multi-token,.odd-token.multi-token{ - background: repeating-linear-gradient( - 45deg, - transparent, - transparent 1px, - #ccc 1px, - #ccc 1px - ), - /* on "bottom" */ - linear-gradient( - to bottom, - #FFB6C1, - #999 - ); -} - -.multi-token:hover::after { - content:"This char has more than 1 token"; /* The content defines the width of the annotation*/ - color:white; - background-color: black; - position:absolute; - font-size:0.75rem; - text-align:center; - font-weight:bold; - text-overflow:ellipsis; - top:1.75rem; - line-height:0; - overflow: hidden; - white-space: nowrap; - left:0; - width:fit-content; /* 100% of the parent, which is the annotation whose width is the tokens inside it*/ - padding:0.5rem 0; -} diff --git a/bindings/python/py_src/tokenizers/tools/visualizer.py b/bindings/python/py_src/tokenizers/tools/visualizer.py deleted file mode 100644 index 72ae287a2..000000000 --- a/bindings/python/py_src/tokenizers/tools/visualizer.py +++ /dev/null @@ -1,420 +0,0 @@ -import html -import itertools -import os -import re -from string import Template -from typing import Any, Callable, Dict, List, NamedTuple, Optional, Tuple - -from tokenizers import Encoding, Tokenizer - -dirname = os.path.dirname(__file__) -css_filename = os.path.join(dirname, "visualizer-styles.css") -with open(css_filename) as f: - css = f.read() - - -class Annotation: - start: int - end: int - label: str - - def __init__(self, start: int, end: int, label: str): - self.start = start - self.end = end - self.label = label - - -AnnotationList = List[Annotation] -PartialIntList = List[Optional[int]] - - -class CharStateKey(NamedTuple): - token_ix: Optional[int] - anno_ix: Optional[int] - - -class CharState: - char_ix: Optional[int] - - def __init__(self, char_ix): - self.char_ix = char_ix - - self.anno_ix: Optional[int] = None - self.tokens: List[int] = [] - - @property - def token_ix(self): - return self.tokens[0] if len(self.tokens) > 0 else None - - @property - def is_multitoken(self): - """ - BPE tokenizers can output more than one token for a char - """ - return len(self.tokens) > 1 - - def partition_key(self) -> CharStateKey: - return CharStateKey( - token_ix=self.token_ix, - anno_ix=self.anno_ix, - ) - - -class Aligned: - pass - - -class EncodingVisualizer: - """ - Build an EncodingVisualizer - - Args: - - tokenizer (:class:`~tokenizers.Tokenizer`): - A tokenizer instance - - default_to_notebook (:obj:`bool`): - Whether to render html output in a notebook by default - - annotation_converter (:obj:`Callable`, `optional`): - An optional (lambda) function that takes an annotation in any format and returns - an Annotation object - """ - - unk_token_regex = re.compile("(.{1}\b)?(unk|oov)(\b.{1})?", flags=re.IGNORECASE) - - def __init__( - self, - tokenizer: Tokenizer, - default_to_notebook: bool = True, - annotation_converter: Optional[Callable[[Any], Annotation]] = None, - ): - if default_to_notebook: - try: - from IPython.display import HTML, display # type: ignore[attr-defined] - except ImportError: - try: - from IPython.core.display import HTML, display # type: ignore[attr-defined] - except ImportError: - msg = ( - "We couldn't import IPython utils for html display.\n" - "Are you running in a notebook?\n" - "You can also pass `default_to_notebook=False` to get back raw HTML.\n" - ) - raise ImportError(msg) from None - self.tokenizer = tokenizer - self.default_to_notebook = default_to_notebook - self.annotation_coverter = annotation_converter - pass - - def __call__( - self, - text: str, - annotations: Optional[List[Any]] = None, - default_to_notebook: Optional[bool] = None, - ) -> Optional[str]: - """ - Build a visualization of the given text - - Args: - text (:obj:`str`): - The text to tokenize - - annotations (:obj:`List[Annotation]`, `optional`): - An optional list of annotations of the text. The can either be an annotation class - or anything else if you instantiated the visualizer with a converter function - - default_to_notebook (:obj:`bool`, `optional`, defaults to `False`): - If True, will render the html in a notebook. Otherwise returns an html string. - - Returns: - The HTML string if default_to_notebook is False, otherwise (default) returns None and - renders the HTML in the notebook - - """ - final_default_to_notebook = self.default_to_notebook - if default_to_notebook is not None: - final_default_to_notebook = default_to_notebook - if final_default_to_notebook: - try: - from IPython.display import HTML, display # type: ignore[attr-defined] - except ImportError: - try: - from IPython.core.display import HTML, display # type: ignore[attr-defined] - except ImportError: - msg = ( - "We couldn't import IPython utils for html display.\n" - "Are you running in a notebook?\n" - "You can also pass `default_to_notebook=False` to get back raw HTML.\n" - ) - raise ImportError(msg) from None - if annotations is None: - annotations = [] - if self.annotation_coverter is not None: - annotations = list(map(self.annotation_coverter, annotations)) - encoding = self.tokenizer.encode(text) - html = EncodingVisualizer.__make_html(text, encoding, annotations) - if final_default_to_notebook: - display(HTML(html)) - else: - return html - - @staticmethod - def calculate_label_colors(annotations: AnnotationList) -> Dict[str, str]: - """ - Generates a color palette for all the labels in a given set of annotations - - Args: - annotations (:obj:`Annotation`): - A list of annotations - - Returns: - :obj:`dict`: A dictionary mapping labels to colors in HSL format - """ - if len(annotations) == 0: - return {} - labels = set(map(lambda x: x.label, annotations)) - num_labels = len(labels) - h_step = int(255 / num_labels) - if h_step < 20: - h_step = 20 - s = 32 - l = 64 # noqa: E741 - h = 10 - colors = {} - - for label in sorted(labels): # sort so we always get the same colors for a given set of labels - colors[label] = f"hsl({h},{s}%,{l}%)" - h += h_step - return colors - - @staticmethod - def consecutive_chars_to_html( - consecutive_chars_list: List[CharState], - text: str, - encoding: Encoding, - ): - """ - Converts a list of "consecutive chars" into a single HTML element. - Chars are consecutive if they fall under the same word, token and annotation. - The CharState class is a named tuple with a "partition_key" method that makes it easy to - compare if two chars are consecutive. - - Args: - consecutive_chars_list (:obj:`List[CharState]`): - A list of CharStates that have been grouped together - - text (:obj:`str`): - The original text being processed - - encoding (:class:`~tokenizers.Encoding`): - The encoding returned from the tokenizer - - Returns: - :obj:`str`: The HTML span for a set of consecutive chars - """ - first = consecutive_chars_list[0] - if first.char_ix is None: - # its a special token - stoken = encoding.tokens[first.token_ix] - # special tokens are represented as empty spans. We use the data attribute and css - # magic to display it - return f'' - # We're not in a special token so this group has a start and end. - last = consecutive_chars_list[-1] - assert first.char_ix is not None - assert last.char_ix is not None - start = first.char_ix - end = last.char_ix + 1 - span_text = text[start:end] - css_classes = [] # What css classes will we apply on the resulting span - data_items = {} # What data attributes will we apply on the result span - if first.token_ix is not None: - # We can either be in a token or not (e.g. in white space) - css_classes.append("token") - if first.is_multitoken: - css_classes.append("multi-token") - if first.token_ix % 2: - # We use this to color alternating tokens. - # A token might be split by an annotation that ends in the middle of it, so this - # lets us visually indicate a consecutive token despite its possible splitting in - # the html markup - css_classes.append("odd-token") - else: - # Like above, but a different color so we can see the tokens alternate - css_classes.append("even-token") - if EncodingVisualizer.unk_token_regex.search(encoding.tokens[first.token_ix]) is not None: - # This is a special token that is in the text. probably UNK - css_classes.append("special-token") - # TODO is this the right name for the data attribute ? - data_items["stok"] = encoding.tokens[first.token_ix] - else: - # In this case we are looking at a group/single char that is not tokenized. - # e.g. white space - css_classes.append("non-token") - css = f'''class="{" ".join(css_classes)}"''' - data = "" - for key, val in data_items.items(): - data += f' data-{key}="{val}"' - span_text = html.escape(span_text) - return f"{span_text}" - - @staticmethod - def __make_html(text: str, encoding: Encoding, annotations: AnnotationList) -> str: - char_states = EncodingVisualizer.__make_char_states(text, encoding, annotations) - current_consecutive_chars = [char_states[0]] - prev_anno_ix = char_states[0].anno_ix - spans = [] - label_colors_dict = EncodingVisualizer.calculate_label_colors(annotations) - cur_anno_ix = char_states[0].anno_ix - if cur_anno_ix is not None: - # If we started in an annotation make a span for it - anno = annotations[cur_anno_ix] - label = anno.label - color = label_colors_dict[label] - spans.append(f'') - - for cs in char_states[1:]: - cur_anno_ix = cs.anno_ix - if cur_anno_ix != prev_anno_ix: - # If we've transitioned in or out of an annotation - spans.append( - # Create a span from the current consecutive characters - EncodingVisualizer.consecutive_chars_to_html( - current_consecutive_chars, - text=text, - encoding=encoding, - ) - ) - current_consecutive_chars = [cs] - - if prev_anno_ix is not None: - # if we transitioned out of an annotation close it's span - spans.append("") - if cur_anno_ix is not None: - # If we entered a new annotation make a span for it - anno = annotations[cur_anno_ix] - label = anno.label - color = label_colors_dict[label] - spans.append(f'') - prev_anno_ix = cur_anno_ix - - if cs.partition_key() == current_consecutive_chars[0].partition_key(): - # If the current charchter is in the same "group" as the previous one - current_consecutive_chars.append(cs) - else: - # Otherwise we make a span for the previous group - spans.append( - EncodingVisualizer.consecutive_chars_to_html( - current_consecutive_chars, - text=text, - encoding=encoding, - ) - ) - # An reset the consecutive_char_list to form a new group - current_consecutive_chars = [cs] - # All that's left is to fill out the final span - # TODO I think there is an edge case here where an annotation's span might not close - spans.append( - EncodingVisualizer.consecutive_chars_to_html( - current_consecutive_chars, - text=text, - encoding=encoding, - ) - ) - - # Close any remaining open annotation span - if cur_anno_ix is not None: - spans.append("") - - res = HTMLBody(spans) # Send the list of spans to the body of our html - return res - - @staticmethod - def __make_anno_map(text: str, annotations: AnnotationList) -> PartialIntList: - """ - Args: - text (:obj:`str`): - The raw text we want to align to - - annotations (:obj:`AnnotationList`): - A (possibly empty) list of annotations - - Returns: - A list of length len(text) whose entry at index i is None if there is no annotation on - character i or k, the index of the annotation that covers index i where k is with - respect to the list of annotations - """ - annotation_map = [None] * len(text) - for anno_ix, a in enumerate(annotations): - for i in range(a.start, a.end): - annotation_map[i] = anno_ix - return annotation_map - - @staticmethod - def __make_char_states(text: str, encoding: Encoding, annotations: AnnotationList) -> List[CharState]: - """ - For each character in the original text, we emit a tuple representing it's "state": - - * which token_ix it corresponds to - * which word_ix it corresponds to - * which annotation_ix it corresponds to - - Args: - text (:obj:`str`): - The raw text we want to align to - - annotations (:obj:`List[Annotation]`): - A (possibly empty) list of annotations - - encoding: (:class:`~tokenizers.Encoding`): - The encoding returned from the tokenizer - - Returns: - :obj:`List[CharState]`: A list of CharStates, indicating for each char in the text what - it's state is - """ - annotation_map = EncodingVisualizer.__make_anno_map(text, annotations) - # Todo make this a dataclass or named tuple - char_states: List[CharState] = [CharState(char_ix) for char_ix in range(len(text))] - for token_ix, token in enumerate(encoding.tokens): - offsets = encoding.token_to_chars(token_ix) - if offsets is not None: - start, end = offsets - for i in range(start, end): - char_states[i].tokens.append(token_ix) - for char_ix, anno_ix in enumerate(annotation_map): - char_states[char_ix].anno_ix = anno_ix - - return char_states - - -def HTMLBody(children: List[str], css_styles=css) -> str: - """ - Generates the full html with css from a list of html spans - - Args: - children (:obj:`List[str]`): - A list of strings, assumed to be html elements - - css_styles (:obj:`str`, `optional`): - Optional alternative implementation of the css - - Returns: - :obj:`str`: An HTML string with style markup - """ - children_text = "".join(children) - return f""" - - - - - -
- {children_text} -
- - - """ diff --git a/bindings/python/py_src/tokenizers/trainers.pyi b/bindings/python/py_src/tokenizers/trainers.pyi deleted file mode 100644 index 6a4008c0a..000000000 --- a/bindings/python/py_src/tokenizers/trainers.pyi +++ /dev/null @@ -1,399 +0,0 @@ -""" -Trainers Module -""" - -from collections.abc import Sequence -from typing import Any, final - -from tokenizers import AddedToken - -@final -class BpeTrainer(Trainer): - """ - Trainer capable of training a BPE model - - Args: - vocab_size (:obj:`int`, `optional`): - The size of the final vocabulary, including all tokens and alphabet. - - min_frequency (:obj:`int`, `optional`): - The minimum frequency a pair should have in order to be merged. - - show_progress (:obj:`bool`, `optional`): - Whether to show progress bars while training. - - special_tokens (:obj:`List[Union[str, AddedToken]]`, `optional`): - A list of special tokens the model should know of. - - limit_alphabet (:obj:`int`, `optional`): - The maximum different characters to keep in the alphabet. - - initial_alphabet (:obj:`List[str]`, `optional`): - A list of characters to include in the initial alphabet, even - if not seen in the training dataset. - If the strings contain more than one character, only the first one - is kept. - - continuing_subword_prefix (:obj:`str`, `optional`): - A prefix to be used for every subword that is not a beginning-of-word. - - end_of_word_suffix (:obj:`str`, `optional`): - A suffix to be used for every subword that is a end-of-word. - - max_token_length (:obj:`int`, `optional`): - Prevents creating tokens longer than the specified size. - This can help with reducing polluting your vocabulary with - highly repetitive tokens like `======` for wikipedia - - Example:: - - >>> from tokenizers.models import BPE - >>> from tokenizers.trainers import BpeTrainer - >>> trainer = BpeTrainer( - ... vocab_size=30000, - ... special_tokens=["", "", ""], - ... min_frequency=2, - ... ) - >>> tokenizer = Tokenizer(BPE()) - >>> tokenizer.train(["path/to/corpus.txt"], trainer) - """ - def __new__(cls, /, **kwargs) -> BpeTrainer: ... - @property - def continuing_subword_prefix(self, /) -> str | None: ... - @continuing_subword_prefix.setter - def continuing_subword_prefix(self, /, prefix: str | None) -> None: ... - @property - def end_of_word_suffix(self, /) -> str | None: ... - @end_of_word_suffix.setter - def end_of_word_suffix(self, /, suffix: str | None) -> None: ... - def get_word_count(self, /) -> int: - """ - Get the number of unique words after feeding the corpus - """ - @property - def initial_alphabet(self, /) -> list[str]: ... - @initial_alphabet.setter - def initial_alphabet(self, /, alphabet: Sequence[str]) -> None: ... - @property - def limit_alphabet(self, /) -> int | None: ... - @limit_alphabet.setter - def limit_alphabet(self, /, limit: int | None) -> None: ... - @property - def max_token_length(self, /) -> int | None: ... - @max_token_length.setter - def max_token_length(self, /, limit: int | None) -> None: ... - @property - def min_frequency(self, /) -> int: ... - @min_frequency.setter - def min_frequency(self, /, freq: int) -> None: ... - @property - def progress_format(self, /) -> str: - """ - Get the progress output format ("indicatif", "json", or "silent") - """ - @progress_format.setter - def progress_format(self, /, format: str) -> None: - """ - Set the progress output format ("indicatif", "json", or "silent") - """ - @property - def show_progress(self, /) -> bool: ... - @show_progress.setter - def show_progress(self, /, show_progress: bool) -> None: ... - @property - def special_tokens(self, /) -> list[AddedToken]: ... - @special_tokens.setter - def special_tokens(self, /, special_tokens: list) -> None: ... - @property - def vocab_size(self, /) -> int: ... - @vocab_size.setter - def vocab_size(self, /, vocab_size: int) -> None: ... - -@final -class ParityBpeTrainer: - def __getstate__(self, /) -> Any: ... - def __new__( - cls, - /, - num_merges: int = 32000, - variant: str = "base", - min_frequency: int = 0, - ratio: Sequence[float] | None = None, - global_merges: int = 0, - window_size: int = 100, - alpha: float = 2.0, - total_symbols: bool = False, - special_tokens: list | None = None, - show_progress: bool = True, - limit_alphabet: int | None = None, - initial_alphabet: Sequence[str] | None = None, - continuing_subword_prefix: str | None = None, - end_of_word_suffix: str | None = None, - max_token_length: int | None = None, - ) -> ParityBpeTrainer: - """Create and return a new object. See help(type) for accurate signature.""" - ... - def __repr__(self, /) -> str: - """Return repr(self).""" - ... - def __setstate__(self, /, state: Any) -> None: ... - def __str__(self, /) -> str: - """Return str(self).""" - ... - @property - def alpha(self, /) -> float: ... - @alpha.setter - def alpha(self, /, v: float) -> None: ... - @property - def continuing_subword_prefix(self, /) -> str | None: ... - @continuing_subword_prefix.setter - def continuing_subword_prefix(self, /, v: str | None) -> None: ... - @property - def end_of_word_suffix(self, /) -> str | None: ... - @end_of_word_suffix.setter - def end_of_word_suffix(self, /, v: str | None) -> None: ... - @property - def global_merges(self, /) -> int: ... - @global_merges.setter - def global_merges(self, /, v: int) -> None: ... - @property - def initial_alphabet(self, /) -> list[str]: ... - @initial_alphabet.setter - def initial_alphabet(self, /, alphabet: Sequence[str]) -> None: ... - @property - def limit_alphabet(self, /) -> int | None: ... - @limit_alphabet.setter - def limit_alphabet(self, /, v: int | None) -> None: ... - @property - def max_token_length(self, /) -> int | None: ... - @max_token_length.setter - def max_token_length(self, /, v: int | None) -> None: ... - @property - def min_frequency(self, /) -> int: ... - @min_frequency.setter - def min_frequency(self, /, v: int) -> None: ... - @property - def num_merges(self, /) -> int: ... - @num_merges.setter - def num_merges(self, /, v: int) -> None: ... - @property - def show_progress(self, /) -> bool: ... - @show_progress.setter - def show_progress(self, /, v: bool) -> None: ... - @property - def special_tokens(self, /) -> list[AddedToken]: ... - @special_tokens.setter - def special_tokens(self, /, special_tokens: list) -> None: ... - @property - def total_symbols(self, /) -> bool: ... - @total_symbols.setter - def total_symbols(self, /, v: bool) -> None: ... - @property - def variant(self, /) -> str: ... - @property - def window_size(self, /) -> int: ... - @window_size.setter - def window_size(self, /, v: int) -> None: ... - -class Trainer: - """ - Base class for all trainers - - This class is not supposed to be instantiated directly. Instead, any implementation of a - Trainer will return an instance of this class when instantiated. - """ - def __getstate__(self, /) -> Any: ... - def __repr__(self, /) -> str: ... - def __setstate__(self, /, state: Any) -> None: ... - def __str__(self, /) -> str: ... - -@final -class UnigramTrainer(Trainer): - """ - Trainer capable of training a Unigram model - - Args: - vocab_size (:obj:`int`): - The size of the final vocabulary, including all tokens and alphabet. - - show_progress (:obj:`bool`): - Whether to show progress bars while training. - - special_tokens (:obj:`List[Union[str, AddedToken]]`): - A list of special tokens the model should know of. - - initial_alphabet (:obj:`List[str]`): - A list of characters to include in the initial alphabet, even - if not seen in the training dataset. - If the strings contain more than one character, only the first one - is kept. - - shrinking_factor (:obj:`float`): - The shrinking factor used at each step of the training to prune the - vocabulary. - - unk_token (:obj:`str`): - The token used for out-of-vocabulary tokens. - - max_piece_length (:obj:`int`): - The maximum length of a given token. - - n_sub_iterations (:obj:`int`): - The number of iterations of the EM algorithm to perform before - pruning the vocabulary. - - Example:: - - >>> from tokenizers.models import Unigram - >>> from tokenizers.trainers import UnigramTrainer - >>> trainer = UnigramTrainer( - ... vocab_size=8000, - ... special_tokens=["", "", ""], - ... unk_token="", - ... ) - >>> tokenizer = Tokenizer(Unigram()) - >>> tokenizer.train(["path/to/corpus.txt"], trainer) - """ - def __new__(cls, /, **kwargs) -> UnigramTrainer: ... - @property - def initial_alphabet(self, /) -> list[str]: ... - @initial_alphabet.setter - def initial_alphabet(self, /, alphabet: Sequence[str]) -> None: ... - @property - def show_progress(self, /) -> bool: ... - @show_progress.setter - def show_progress(self, /, show_progress: bool) -> None: ... - @property - def special_tokens(self, /) -> list[AddedToken]: ... - @special_tokens.setter - def special_tokens(self, /, special_tokens: list) -> None: ... - @property - def vocab_size(self, /) -> int: ... - @vocab_size.setter - def vocab_size(self, /, vocab_size: int) -> None: ... - -@final -class WordLevelTrainer(Trainer): - """ - Trainer capable of training a WordLevel model - - Args: - vocab_size (:obj:`int`, `optional`): - The size of the final vocabulary, including all tokens and alphabet. - - min_frequency (:obj:`int`, `optional`): - The minimum frequency a pair should have in order to be merged. - - show_progress (:obj:`bool`, `optional`): - Whether to show progress bars while training. - - special_tokens (:obj:`List[Union[str, AddedToken]]`): - A list of special tokens the model should know of. - - Example:: - - >>> from tokenizers.models import WordLevel - >>> from tokenizers.trainers import WordLevelTrainer - >>> trainer = WordLevelTrainer( - ... vocab_size=10000, - ... special_tokens=[""], - ... min_frequency=1, - ... ) - >>> tokenizer = Tokenizer(WordLevel(unk_token="")) - >>> tokenizer.train(["path/to/corpus.txt"], trainer) - """ - def __new__(cls, /, **kwargs) -> WordLevelTrainer: ... - @property - def min_frequency(self, /) -> int: ... - @min_frequency.setter - def min_frequency(self, /, freq: int) -> None: ... - @property - def show_progress(self, /) -> bool: ... - @show_progress.setter - def show_progress(self, /, show_progress: bool) -> None: ... - @property - def special_tokens(self, /) -> list[AddedToken]: ... - @special_tokens.setter - def special_tokens(self, /, special_tokens: list) -> None: ... - @property - def vocab_size(self, /) -> int: ... - @vocab_size.setter - def vocab_size(self, /, vocab_size: int) -> None: ... - -@final -class WordPieceTrainer(Trainer): - """ - Trainer capable of training a WordPiece model - - Args: - vocab_size (:obj:`int`, `optional`): - The size of the final vocabulary, including all tokens and alphabet. - - min_frequency (:obj:`int`, `optional`): - The minimum frequency a pair should have in order to be merged. - - show_progress (:obj:`bool`, `optional`): - Whether to show progress bars while training. - - special_tokens (:obj:`List[Union[str, AddedToken]]`, `optional`): - A list of special tokens the model should know of. - - limit_alphabet (:obj:`int`, `optional`): - The maximum different characters to keep in the alphabet. - - initial_alphabet (:obj:`List[str]`, `optional`): - A list of characters to include in the initial alphabet, even - if not seen in the training dataset. - If the strings contain more than one character, only the first one - is kept. - - continuing_subword_prefix (:obj:`str`, `optional`): - A prefix to be used for every subword that is not a beginning-of-word. - - end_of_word_suffix (:obj:`str`, `optional`): - A suffix to be used for every subword that is a end-of-word. - - Example:: - - >>> from tokenizers.models import WordPiece - >>> from tokenizers.trainers import WordPieceTrainer - >>> trainer = WordPieceTrainer( - ... vocab_size=30000, - ... special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"], - ... ) - >>> tokenizer = Tokenizer(WordPiece(unk_token="[UNK]")) - >>> tokenizer.train(["path/to/corpus.txt"], trainer) - """ - def __new__(cls, /, **kwargs) -> WordPieceTrainer: ... - @property - def continuing_subword_prefix(self, /) -> str | None: ... - @continuing_subword_prefix.setter - def continuing_subword_prefix(self, /, prefix: str | None) -> None: ... - @property - def end_of_word_suffix(self, /) -> str | None: ... - @end_of_word_suffix.setter - def end_of_word_suffix(self, /, suffix: str | None) -> None: ... - @property - def initial_alphabet(self, /) -> list[str]: ... - @initial_alphabet.setter - def initial_alphabet(self, /, alphabet: Sequence[str]) -> None: ... - @property - def limit_alphabet(self, /) -> int | None: ... - @limit_alphabet.setter - def limit_alphabet(self, /, limit: int | None) -> None: ... - @property - def min_frequency(self, /) -> int: ... - @min_frequency.setter - def min_frequency(self, /, freq: int) -> None: ... - @property - def show_progress(self, /) -> bool: ... - @show_progress.setter - def show_progress(self, /, show_progress: bool) -> None: ... - @property - def special_tokens(self, /) -> list[AddedToken]: ... - @special_tokens.setter - def special_tokens(self, /, special_tokens: list) -> None: ... - @property - def vocab_size(self, /) -> int: ... - @vocab_size.setter - def vocab_size(self, /, vocab_size: int) -> None: ... diff --git a/bindings/python/py_src/tokenizers/trainers/__init__.py b/bindings/python/py_src/tokenizers/trainers/__init__.py index b4cefae41..99cc2a3eb 100644 --- a/bindings/python/py_src/tokenizers/trainers/__init__.py +++ b/bindings/python/py_src/tokenizers/trainers/__init__.py @@ -1,10 +1,17 @@ -# Generated content DO NOT EDIT +"""Recipes for learning a vocabulary from text.""" -from .. import trainers +from .._native import trainers as _trainers -BpeTrainer = trainers.BpeTrainer -ParityBpeTrainer = trainers.ParityBpeTrainer -Trainer = trainers.Trainer -UnigramTrainer = trainers.UnigramTrainer -WordLevelTrainer = trainers.WordLevelTrainer -WordPieceTrainer = trainers.WordPieceTrainer +Trainer = _trainers.Trainer +BpeTrainer = _trainers.BpeTrainer +UnigramTrainer = _trainers.UnigramTrainer +WordLevelTrainer = _trainers.WordLevelTrainer +WordPieceTrainer = _trainers.WordPieceTrainer + +__all__ = [ + "Trainer", + "BpeTrainer", + "UnigramTrainer", + "WordLevelTrainer", + "WordPieceTrainer", +] diff --git a/bindings/python/py_src/tokenizers/trainers/__init__.pyi b/bindings/python/py_src/tokenizers/trainers/__init__.pyi index 430991960..c754b2452 100644 --- a/bindings/python/py_src/tokenizers/trainers/__init__.pyi +++ b/bindings/python/py_src/tokenizers/trainers/__init__.pyi @@ -1,712 +1,53 @@ -# Generated content DO NOT EDIT -class Trainer: - """ - Base class for all trainers +""" +Recipes for learning a vocabulary from text. +""" - This class is not supposed to be instantiated directly. Instead, any implementation of a - Trainer will return an instance of this class when instantiated. - """ - def __getstate__(self): - """ """ - pass - - def __setstate__(self, state): - """ """ - pass +from tokenizers import AddedToken +from collections.abc import Sequence +from typing import final +@final class BpeTrainer(Trainer): """ - Trainer capable of training a BPE model - - Args: - vocab_size (:obj:`int`, `optional`): - The size of the final vocabulary, including all tokens and alphabet. - - min_frequency (:obj:`int`, `optional`): - The minimum frequency a pair should have in order to be merged. - - show_progress (:obj:`bool`, `optional`): - Whether to show progress bars while training. - - special_tokens (:obj:`List[Union[str, AddedToken]]`, `optional`): - A list of special tokens the model should know of. - - limit_alphabet (:obj:`int`, `optional`): - The maximum different characters to keep in the alphabet. - - initial_alphabet (:obj:`List[str]`, `optional`): - A list of characters to include in the initial alphabet, even - if not seen in the training dataset. - If the strings contain more than one character, only the first one - is kept. - - continuing_subword_prefix (:obj:`str`, `optional`): - A prefix to be used for every subword that is not a beginning-of-word. - - end_of_word_suffix (:obj:`str`, `optional`): - A suffix to be used for every subword that is a end-of-word. - - max_token_length (:obj:`int`, `optional`): - Prevents creating tokens longer than the specified size. - This can help with reducing polluting your vocabulary with - highly repetitive tokens like `======` for wikipedia - + Learns a BPE vocabulary: keeps merging the most frequent pair until + `vocab_size` is reached, ignoring pairs seen fewer than `min_frequency` + times. `special_tokens` get the first ids. `limit_alphabet` caps how many + distinct characters are kept; `initial_alphabet` forces characters in even + if the data never shows them; `max_token_length` caps merged token length. """ - def __init__( - self, - vocab_size=30000, - min_frequency=0, - show_progress=True, - special_tokens=[], - limit_alphabet=None, - initial_alphabet=[], - continuing_subword_prefix=None, - end_of_word_suffix=None, - max_token_length=None, - words={}, - ): - pass - - def __getstate__(self): - """ """ - pass - - def __setstate__(self, state): - """ """ - pass - - @property - def continuing_subword_prefix(self): - """ """ - pass - - @continuing_subword_prefix.setter - def continuing_subword_prefix(self, value): - """ """ - pass - - @property - def end_of_word_suffix(self): - """ """ - pass + def __new__(cls, /, *, vocab_size: int = 30000, min_frequency: int = 0, special_tokens: Sequence[str |AddedToken] = ..., limit_alphabet: int |None = None, initial_alphabet: Sequence[str] = ..., continuing_subword_prefix: str |None = None, end_of_word_suffix: str |None = None, max_token_length: int |None = None, show_progress: bool = True) -> BpeTrainer: ... - @end_of_word_suffix.setter - def end_of_word_suffix(self, value): - """ """ - pass - - @property - def initial_alphabet(self): - """ """ - pass - - @initial_alphabet.setter - def initial_alphabet(self, value): - """ """ - pass - - @property - def limit_alphabet(self): - """ """ - pass - - @limit_alphabet.setter - def limit_alphabet(self, value): - """ """ - pass - - @property - def max_token_length(self): - """ """ - pass - - @max_token_length.setter - def max_token_length(self, value): - """ """ - pass - - @property - def min_frequency(self): - """ """ - pass - - @min_frequency.setter - def min_frequency(self, value): - """ """ - pass - - @property - def show_progress(self): - """ """ - pass - - @show_progress.setter - def show_progress(self, value): - """ """ - pass - - @property - def special_tokens(self): - """ """ - pass - - @special_tokens.setter - def special_tokens(self, value): - """ """ - pass - - @property - def vocab_size(self): - """ """ - pass - - @vocab_size.setter - def vocab_size(self, value): - """ """ - pass +class Trainer: + """ + Base class for all trainers. + + A trainer is the recipe for learning a model's vocabulary from text; pass + one to `Tokenizer.train` or `train_from_iterator`. Trainers are plain + configuration values — training copies them and writes nothing back. + """ + def __repr__(self, /) -> str: ... +@final class UnigramTrainer(Trainer): """ - Trainer capable of training a Unigram model - - Args: - vocab_size (:obj:`int`): - The size of the final vocabulary, including all tokens and alphabet. - - show_progress (:obj:`bool`): - Whether to show progress bars while training. - - special_tokens (:obj:`List[Union[str, AddedToken]]`): - A list of special tokens the model should know of. - - initial_alphabet (:obj:`List[str]`): - A list of characters to include in the initial alphabet, even - if not seen in the training dataset. - If the strings contain more than one character, only the first one - is kept. - - shrinking_factor (:obj:`float`): - The shrinking factor used at each step of the training to prune the - vocabulary. - - unk_token (:obj:`str`): - The token used for out-of-vocabulary tokens. - - max_piece_length (:obj:`int`): - The maximum length of a given token. - - n_sub_iterations (:obj:`int`): - The number of iterations of the EM algorithm to perform before - pruning the vocabulary. + Learns a Unigram vocabulary: starts from a large candidate set and prunes + it by `shrinking_factor` each round until `vocab_size` pieces remain. + `unk_token` names the fallback piece for unknown characters. """ - def __init__( - self, - vocab_size=8000, - show_progress=True, - special_tokens=[], - initial_alphabet=[], - shrinking_factor=0.75, - unk_token=None, - max_piece_length=16, - n_sub_iterations=2, - ): - pass - - def __getstate__(self): - """ """ - pass - - def __setstate__(self, state): - """ """ - pass - - @property - def initial_alphabet(self): - """ """ - pass - - @initial_alphabet.setter - def initial_alphabet(self, value): - """ """ - pass - - @property - def show_progress(self): - """ """ - pass - - @show_progress.setter - def show_progress(self, value): - """ """ - pass - - @property - def special_tokens(self): - """ """ - pass - - @special_tokens.setter - def special_tokens(self, value): - """ """ - pass - - @property - def vocab_size(self): - """ """ - pass - - @vocab_size.setter - def vocab_size(self, value): - """ """ - pass + def __new__(cls, /, *, vocab_size: int = 8000, special_tokens: Sequence[str |AddedToken] = ..., initial_alphabet: Sequence[str] = ..., unk_token: str |None = None, shrinking_factor: float = 0.75, max_piece_length: int = 16, n_sub_iterations: int = 2, show_progress: bool = True) -> UnigramTrainer: ... +@final class WordLevelTrainer(Trainer): """ - Trainer capable of training a WorldLevel model - - Args: - vocab_size (:obj:`int`, `optional`): - The size of the final vocabulary, including all tokens and alphabet. - - min_frequency (:obj:`int`, `optional`): - The minimum frequency a pair should have in order to be merged. - - show_progress (:obj:`bool`, `optional`): - Whether to show progress bars while training. - - special_tokens (:obj:`List[Union[str, AddedToken]]`): - A list of special tokens the model should know of. - """ - def __init__(self, vocab_size=30000, min_frequency=0, show_progress=True, special_tokens=[]): - pass - - def __getstate__(self): - """ """ - pass - - def __setstate__(self, state): - """ """ - pass - - @property - def min_frequency(self): - """ """ - pass - - @min_frequency.setter - def min_frequency(self, value): - """ """ - pass - - @property - def show_progress(self): - """ """ - pass - - @show_progress.setter - def show_progress(self, value): - """ """ - pass - - @property - def special_tokens(self): - """ """ - pass - - @special_tokens.setter - def special_tokens(self, value): - """ """ - pass - - @property - def vocab_size(self): - """ """ - pass - - @vocab_size.setter - def vocab_size(self, value): - """ """ - pass - -class ParityBpeTrainer: + Learns a WordLevel vocabulary: the `vocab_size` most frequent words, + keeping only those seen at least `min_frequency` times. """ - Trainer for parity-aware BPE that ensures cross-lingual fairness in tokenization. - - Unlike standard BPE, this trainer takes one Python iterator per language and - balances merge operations across languages using a development set or target - compression ratios. The single training entry point is :meth:`train_from_iterator`, - the multi-corpus analogue of :meth:`tokenizers.Tokenizer.train_from_iterator`. - - Args: - num_merges (:obj:`int`, `optional`): - Number of BPE merge operations to perform. Defaults to ``32000``. - - variant (:obj:`str`, `optional`): - Algorithm variant: ``"base"`` (default) or ``"window"`` (moving-window balancing). - - min_frequency (:obj:`int`, `optional`): - Minimum pair frequency to merge. Defaults to ``0``. - - global_merges (:obj:`int`, `optional`): - Number of initial standard BPE merges before switching to parity mode. - Defaults to ``0``. - - window_size (:obj:`int`, `optional`): - Window size for the ``"window"`` variant. Defaults to ``100``. - - alpha (:obj:`float`, `optional`): - Alpha parameter for the ``"window"`` variant. Defaults to ``2.0``. - - total_symbols (:obj:`bool`, `optional`): - If True, subtract unique character count from ``num_merges``. - Defaults to ``False``. - - special_tokens (:obj:`List[Union[str, AddedToken]]`, `optional`): - A list of special tokens the model should know of. - - show_progress (:obj:`bool`, `optional`): - Whether to show progress bars while training. Defaults to ``True``. - - limit_alphabet (:obj:`int`, `optional`): - The maximum different characters to keep in the alphabet. - - initial_alphabet (:obj:`List[str]`, `optional`): - A list of characters to include in the initial alphabet, even - if not seen in the training dataset. - - continuing_subword_prefix (:obj:`str`, `optional`): - A prefix to be used for every subword that is not a beginning-of-word. - - end_of_word_suffix (:obj:`str`, `optional`): - A suffix to be used for every subword that is a end-of-word. - - max_token_length (:obj:`int`, `optional`): - Prevents creating tokens longer than the specified size. - """ - def __init__( - self, - num_merges=32000, - variant="base", - min_frequency=0, - ratio=None, - global_merges=0, - window_size=100, - alpha=2.0, - total_symbols=False, - special_tokens=None, - show_progress=True, - limit_alphabet=None, - initial_alphabet=None, - continuing_subword_prefix=None, - end_of_word_suffix=None, - max_token_length=None, - ): - pass - - def __repr__(self) -> str: ... - def __str__(self) -> str: ... - def __getstate__(self): ... - def __setstate__(self, state): ... - def train_from_iterator( - self, - tokenizer, - train_iterators, - dev_iterators=None, - ratio=None, - ): - """ - Train a user-configured tokenizer with parity-aware BPE from per-language - Python iterators. - - This is the multi-corpus analogue of - :meth:`~tokenizers.Tokenizer.train_from_iterator`: file I/O happens in - Python, so users can pull data from plain text, parquet (via ``pyarrow``), - ``datasets``, etc. - - Args: - tokenizer (:class:`~tokenizers.Tokenizer`): - A tokenizer instance to train. Its pre-tokenizer (and optionally - normalizer) should already be configured. - - train_iterators (:obj:`List[Iterator]`): - One Python iterator per language, each yielding ``str`` or - ``List[str]``. - - dev_iterators (:obj:`List[Iterator]`, `optional`): - One Python iterator per language, used to drive parity-aware - language selection. Must have the same length as - ``train_iterators``. - - ratio (:obj:`List[float]`, `optional`): - Target compression ratios per language (alternative to - ``dev_iterators``). - """ - pass - - @property - def special_tokens(self): - """ """ - pass - - @special_tokens.setter - def special_tokens(self, value): - """ """ - pass - - @property - def show_progress(self): - """ """ - pass - - @show_progress.setter - def show_progress(self, value): - """ """ - pass - - @property - def limit_alphabet(self): - """ """ - pass - - @limit_alphabet.setter - def limit_alphabet(self, value): - """ """ - pass - - @property - def initial_alphabet(self): - """ """ - pass - - @initial_alphabet.setter - def initial_alphabet(self, value): - """ """ - pass - - @property - def continuing_subword_prefix(self): - """ """ - pass - - @continuing_subword_prefix.setter - def continuing_subword_prefix(self, value): - """ """ - pass - - @property - def end_of_word_suffix(self): - """ """ - pass - - @end_of_word_suffix.setter - def end_of_word_suffix(self, value): - """ """ - pass - - @property - def max_token_length(self): - """ """ - pass - - @max_token_length.setter - def max_token_length(self, value): - """ """ - pass - - @property - def min_frequency(self): - """ """ - pass - - @min_frequency.setter - def min_frequency(self, value): - """ """ - pass - - @property - def num_merges(self): - """ """ - pass - - @num_merges.setter - def num_merges(self, value): - """ """ - pass - - @property - def variant(self): - """ """ - pass - - @property - def global_merges(self): - """ """ - pass - - @global_merges.setter - def global_merges(self, value): - """ """ - pass - - @property - def window_size(self): - """ """ - pass - - @window_size.setter - def window_size(self, value): - """ """ - pass - - @property - def alpha(self): - """ """ - pass - - @alpha.setter - def alpha(self, value): - """ """ - pass - - @property - def total_symbols(self): - """ """ - pass - - @total_symbols.setter - def total_symbols(self, value): - """ """ - pass + def __new__(cls, /, *, vocab_size: int = 30000, min_frequency: int = 0, special_tokens: Sequence[str |AddedToken] = ..., show_progress: bool = True) -> WordLevelTrainer: ... +@final class WordPieceTrainer(Trainer): """ - Trainer capable of training a WordPiece model - - Args: - vocab_size (:obj:`int`, `optional`): - The size of the final vocabulary, including all tokens and alphabet. - - min_frequency (:obj:`int`, `optional`): - The minimum frequency a pair should have in order to be merged. - - show_progress (:obj:`bool`, `optional`): - Whether to show progress bars while training. - - special_tokens (:obj:`List[Union[str, AddedToken]]`, `optional`): - A list of special tokens the model should know of. - - limit_alphabet (:obj:`int`, `optional`): - The maximum different characters to keep in the alphabet. - - initial_alphabet (:obj:`List[str]`, `optional`): - A list of characters to include in the initial alphabet, even - if not seen in the training dataset. - If the strings contain more than one character, only the first one - is kept. - - continuing_subword_prefix (:obj:`str`, `optional`): - A prefix to be used for every subword that is not a beginning-of-word. - - end_of_word_suffix (:obj:`str`, `optional`): - A suffix to be used for every subword that is a end-of-word. + Learns a WordPiece vocabulary. Same knobs as `BpeTrainer`, plus the + continuation prefix ("##" by default). """ - def __init__( - self, - vocab_size=30000, - min_frequency=0, - show_progress=True, - special_tokens=[], - limit_alphabet=None, - initial_alphabet=[], - continuing_subword_prefix="##", - end_of_word_suffix=None, - ): - pass - - def __getstate__(self): - """ """ - pass - - def __setstate__(self, state): - """ """ - pass - - @property - def continuing_subword_prefix(self): - """ """ - pass - - @continuing_subword_prefix.setter - def continuing_subword_prefix(self, value): - """ """ - pass - - @property - def end_of_word_suffix(self): - """ """ - pass - - @end_of_word_suffix.setter - def end_of_word_suffix(self, value): - """ """ - pass - - @property - def initial_alphabet(self): - """ """ - pass - - @initial_alphabet.setter - def initial_alphabet(self, value): - """ """ - pass - - @property - def limit_alphabet(self): - """ """ - pass - - @limit_alphabet.setter - def limit_alphabet(self, value): - """ """ - pass - - @property - def min_frequency(self): - """ """ - pass - - @min_frequency.setter - def min_frequency(self, value): - """ """ - pass - - @property - def show_progress(self): - """ """ - pass - - @show_progress.setter - def show_progress(self, value): - """ """ - pass - - @property - def special_tokens(self): - """ """ - pass - - @special_tokens.setter - def special_tokens(self, value): - """ """ - pass - - @property - def vocab_size(self): - """ """ - pass - - @vocab_size.setter - def vocab_size(self, value): - """ """ - pass + def __new__(cls, /, *, vocab_size: int = 30000, min_frequency: int = 0, special_tokens: Sequence[str |AddedToken] = ..., limit_alphabet: int |None = None, initial_alphabet: Sequence[str] = ..., continuing_subword_prefix: str = ..., end_of_word_suffix: str |None = None, show_progress: bool = True) -> WordPieceTrainer: ... diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml index 497e3f6a1..51ecd2ad9 100644 --- a/bindings/python/pyproject.toml +++ b/bindings/python/pyproject.toml @@ -1,70 +1,25 @@ +[build-system] +requires = ["maturin>=1.5,<2.0"] +build-backend = "maturin" + [project] name = "tokenizers" +description = "Fast Python bindings for 🤗 tokenizers, built on the PipelineTokenizer encode path" +readme = "README.md" requires-python = ">=3.10" -authors = [ - { name = "Nicolas Patry", email = "patry.nicolas@protonmail.com" }, - { name = "Anthony Moi", email = "anthony@huggingface.co" }, -] -classifiers = [ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "Intended Audience :: Education", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: Apache Software License", - "Operating System :: OS Independent", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: 3.14", - "Programming Language :: Python :: 3 :: Only", - "Topic :: Scientific/Engineering :: Artificial Intelligence", -] keywords = ["NLP", "tokenizer", "BPE", "transformer", "deep learning"] -dynamic = ["description", "license", "readme", "version"] -dependencies = ["huggingface_hub>=0.16.4,<2.0"] +dependencies = ["numpy>=1.24"] +dynamic = ["version"] [project.urls] Homepage = "https://github.com/huggingface/tokenizers" Source = "https://github.com/huggingface/tokenizers" - [project.optional-dependencies] -testing = ["pytest", "pytest-asyncio", "requests", "numpy", "datasets", "ruff", "ty"] -docs = ["sphinx", "sphinx_rtd_theme", "setuptools_rust"] -dev = ["tokenizers[testing]"] - - -[build-system] -requires = ["maturin>=1.0,<2.0"] -build-backend = "maturin" +hub = ["huggingface_hub>=0.16.4"] [tool.maturin] python-source = "py_src" -module-name = "tokenizers.tokenizers" +module-name = "tokenizers._native" bindings = "pyo3" -features = ["pyo3/extension-module", "abi3"] - - -[tool.ruff] -line-length = 119 -target-version = "py311" -lint.ignore = [ - # a == None in tests vs is None. - "E711", - # a == False in tests vs is False. - "E712", - # try.. import except.. pattern without using the lib. - "F401", - # Raw type equality is required in asserts - "E721", - # Import order - "E402", - # Fixtures unused import - "F811", -] - -[tool.ty.rules] -invalid-method-override = "ignore" -invalid-type-form = "ignore" +features = ["ext-module"] diff --git a/bindings/python/pytest.ini b/bindings/python/pytest.ini deleted file mode 100644 index 98794c9af..000000000 --- a/bindings/python/pytest.ini +++ /dev/null @@ -1,3 +0,0 @@ -[pytest] -markers = - network: mark a test that requires network access. diff --git a/bindings/python/rust-toolchain b/bindings/python/rust-toolchain deleted file mode 100644 index 2bf5ad044..000000000 --- a/bindings/python/rust-toolchain +++ /dev/null @@ -1 +0,0 @@ -stable diff --git a/bindings/python/scripts/convert.py b/bindings/python/scripts/convert.py deleted file mode 100644 index a8aea13d1..000000000 --- a/bindings/python/scripts/convert.py +++ /dev/null @@ -1,417 +0,0 @@ -import transformers # type: ignore[import] -from tokenizers.implementations import SentencePieceUnigramTokenizer, BaseTokenizer -from tokenizers.processors import TemplateProcessing -from tokenizers.models import Unigram, BPE -from tokenizers import decoders -from tokenizers import Tokenizer, Regex -from tokenizers.normalizers import ( - StripAccents, - NFKD, - Lowercase, - Sequence, - BertNormalizer, - Precompiled, - Replace, -) -from tokenizers.pre_tokenizers import ( - Digits, - WhitespaceSplit, - Metaspace, - Sequence as PSequence, -) -import json -import unicodedata -import sys -import os -import datetime -import argparse - -sys.path.append(".") - -from spm_parity_check import check_details # type: ignore[import] -from sentencepiece_extractor import SentencePieceExtractor # type: ignore[import] - - -def check_number_comma(piece: str) -> bool: - return len(piece) < 2 or piece[-1] != "," or not piece[-2].isdigit() - - -def get_proto(filename: str): - try: - import sys - - sys.path.append(".") - - import sentencepiece_model_pb2 as model # type: ignore[import] - except Exception: - raise Exception( - "You don't seem to have the required protobuf file, in order to use this function you need to run `pip install protobuf` and `wget https://raw.githubusercontent.com/google/sentencepiece/master/python/sentencepiece_model_pb2.py` for us to be able to read the intrinsics of your spm_file. `pip install sentencepiece` is not required." - ) - - m = model.ModelProto() - m.ParseFromString(open(filename, "rb").read()) - return m - - -class Converter: - def __init__(self, original_tokenizer): - self.original_tokenizer = original_tokenizer - - def converted(self) -> Tokenizer: - raise NotImplementedError() - - -class SpmConverter(Converter): - def __init__(self, *args): - super().__init__(*args) - self.proto = get_proto(self.original_tokenizer.vocab_file) - - def vocab(self, proto): - return [(piece.piece, piece.score) for piece in proto.pieces] - - def unk_id(self, proto): - return proto.trainer_spec.unk_id - - def tokenizer(self, proto): - model_type = proto.trainer_spec.model_type - vocab = self.vocab(proto) - unk_id = self.unk_id(proto) - if model_type == 1: - tokenizer = Tokenizer(Unigram(vocab, unk_id)) - elif model_type == 2: - vocab, merges = SentencePieceExtractor(self.original_tokenizer.vocab_file).extract() - tokenizer = Tokenizer(BPE(vocab, merges, unk_token=proto.trainer_spec.unk_piece, fuse_unk=True)) - else: - raise Exception( - "You're trying to run a `Unigram` model but you're file was trained with a different algorithm" - ) - - return tokenizer - - def normalizer(self, proto): - precompiled_charsmap = proto.normalizer_spec.precompiled_charsmap - return Sequence([Precompiled(precompiled_charsmap), Replace(Regex(" {2,}"), " ")]) - - def post_processor(self, tokenizer): - return None - - def converted(self): - tokenizer = self.tokenizer(self.proto) - - # Tokenizer assemble - tokenizer.normalizer = self.normalizer(self.proto) - - replacement = "▁" - prepend_scheme = "always" - tokenizer.pre_tokenizer = Metaspace(replacement=replacement, prepend_scheme=prepend_scheme) - tokenizer.decoder = decoders.Metaspace(replacement=replacement, prepend_scheme=prepend_scheme) - post_processor = self.post_processor(tokenizer) - if post_processor: - tokenizer.post_processor = post_processor - - # TODO what parameters should we give ? - parameters = {} - - return BaseTokenizer(tokenizer, parameters) - - -class AlbertConverter(SpmConverter): - def vocab(self, proto): - return [ - (piece.piece, piece.score) if check_number_comma(piece.piece) else (piece.piece, piece.score - 100) - for piece in proto.pieces - ] - - def normalizer(self, proto): - normalizers = [Replace("``", '"'), Replace("''", '"')] - if not self.original_tokenizer.keep_accents: - normalizers.append(NFKD()) - normalizers.append(StripAccents()) - if self.original_tokenizer.do_lower_case: - normalizers.append(Lowercase()) - - precompiled_charsmap = proto.normalizer_spec.precompiled_charsmap - normalizers.append(Precompiled(precompiled_charsmap)) - normalizers.append(Replace(Regex(" {2,}"), " ")) - return Sequence(normalizers) - - def post_processor(self, tokenizer): - return TemplateProcessing( - single=["[CLS]", "$0", "[SEP]"], - pair=["$1", "[SEP]"], - special_tokens=[ - ("[CLS]", tokenizer.get_vocab()["[CLS]"]), - ("[SEP]", tokenizer.get_vocab()["[SEP]"]), - ], - ) - - -class CamembertConverter(SpmConverter): - def vocab(self, proto): - vocab = [ - ("NOTUSED", 0.0), - ("", 0.0), - ("NOTUSED", 0.0), - ("", 0.0), - ] - vocab += [(piece.piece, piece.score) for piece in proto.pieces] - return vocab - - def unk_id(self, proto): - # See vocab unk position - return 3 - - def post_processor(self, tokenizer): - return TemplateProcessing( - single=["", "$0", ""], - pair=["$1", ""], - special_tokens=[ - ("", tokenizer.get_vocab()[""]), - ("", tokenizer.get_vocab()[""]), - ], - ) - - -class MBartConverter(SpmConverter): - def vocab(self, proto): - vocab = [ - ("", 0.0), - ("", 0.0), - ("", 0.0), - ("", 0.0), - ] - vocab += [(piece.piece, piece.score) for piece in proto.pieces[3:]] - vocab += [ - ("ar_AR", 0.0), - ("cs_CZ", 0.0), - ("de_DE", 0.0), - ("en_XX", 0.0), - ("es_XX", 0.0), - ("et_EE", 0.0), - ("fi_FI", 0.0), - ("fr_XX", 0.0), - ("gu_IN", 0.0), - ("hi_IN", 0.0), - ("it_IT", 0.0), - ("ja_XX", 0.0), - ("kk_KZ", 0.0), - ("ko_KR", 0.0), - ("lt_LT", 0.0), - ("lv_LV", 0.0), - ("my_MM", 0.0), - ("ne_NP", 0.0), - ("nl_XX", 0.0), - ("ro_RO", 0.0), - ("ru_RU", 0.0), - ("si_LK", 0.0), - ("tr_TR", 0.0), - ("vi_VN", 0.0), - ("zh_CN", 0.0), - ] - return vocab - - def unk_id(self, proto): - return 3 - - def post_processor(self, tokenizer): - return TemplateProcessing( - single=["$0", "", "en_XX"], - pair=["$1", ""], - special_tokens=[ - ("en_XX", tokenizer.get_vocab()["en_XX"]), - ("", tokenizer.get_vocab()[""]), - ], - ) - - -class XLMRobertaConverter(SpmConverter): - def vocab(self, proto): - vocab = [ - ("", 0.0), - ("", 0.0), - ("", 0.0), - ("", 0.0), - ] - vocab += [(piece.piece, piece.score) for piece in proto.pieces[3:]] - return vocab - - def unk_id(self, proto): - unk_id = 3 - return unk_id - - def post_processor(self, tokenizer): - return TemplateProcessing( - single=["", "$0", ""], - pair=["$1", ""], - special_tokens=[ - ("", tokenizer.get_vocab()[""]), - ("", tokenizer.get_vocab()[""]), - ], - ) - - -class XLNetConverter(SpmConverter): - def vocab(self, proto): - return [ - (piece.piece, piece.score) if check_number_comma(piece.piece) else (piece.piece, piece.score - 100) - for piece in proto.pieces - ] - - def normalizer(self, proto): - normalizers = [Replace("``", '"'), Replace("''", '"')] - if not self.original_tokenizer.keep_accents: - normalizers.append(NFKD()) - normalizers.append(StripAccents()) - if self.original_tokenizer.do_lower_case: - normalizers.append(Lowercase()) - - precompiled_charsmap = proto.normalizer_spec.precompiled_charsmap - normalizers.append(Precompiled(precompiled_charsmap)) - normalizers.append(Replace(Regex(" {2,}"), " ")) - return Sequence(normalizers) - - def post_processor(self, tokenizer): - return TemplateProcessing( - single=["$0", "", ""], - pair=["$1", ""], - special_tokens=[ - ("", tokenizer.get_vocab()[""]), - ("", tokenizer.get_vocab()[""]), - ], - ) - - -class ReformerConverter(SpmConverter): - pass - - -class PegasusConverter(SpmConverter): - offset = 103 - - def vocab(self, proto): - vocab = [ - (self.original_tokenizer.pad_token, 0), - (self.original_tokenizer.eos_token, 0), - ] - vocab += [(f"unk_{i}", -100) for i in range(2, 2 + self.offset)] - vocab += [(piece.piece, piece.score) for piece in proto.pieces[2:]] - return vocab - - def unk_id(self, proto): - return proto.trainer_spec.unk_id + self.offset - - def post_processor(self, tokenizer): - eos = self.original_tokenizer.eos_token - return TemplateProcessing( - single=["$0", eos], - pair=["$1", eos], - special_tokens=[(eos, tokenizer.get_vocab()[eos])], - ) - - -class T5Converter(SpmConverter): - def post_processor(self, tokenizer): - return TemplateProcessing( - single=["$0", ""], - pair=["$1", ""], - special_tokens=[("", tokenizer.get_vocab()[""])], - ) - - -CONVERTERS = { - "AlbertTokenizer": AlbertConverter, - "CamembertTokenizer": CamembertConverter, - "XLMRobertaTokenizer": XLMRobertaConverter, - "MBartTokenizer": MBartConverter, - "XLNetTokenizer": XLNetConverter, - "ReformerTokenizer": ReformerConverter, - "PegasusTokenizer": PegasusConverter, - "T5Tokenizer": T5Converter, -} - - -def check(pretrained, filename): - transformer_tokenizer = transformers.AutoTokenizer.from_pretrained(pretrained) - converter_class = CONVERTERS[transformer_tokenizer.__class__.__name__] - tokenizer = converter_class(transformer_tokenizer).converted() - - now = datetime.datetime.now - trans_total_time = datetime.timedelta(seconds=0) - tok_total_time = datetime.timedelta(seconds=0) - - with open(filename, "r") as f: - for i, line in enumerate(f): - line = line.strip() - - start = now() - ids = transformer_tokenizer.encode(line) - trans = now() - tok_ids = tokenizer.encode(line).ids - tok = now() - - trans_total_time += trans - start - tok_total_time += tok - trans - - if ids != tok_ids: - if check_details(line, ids, tok_ids, transformer_tokenizer, tokenizer): - continue - assert ids == tok_ids, f"Error in line {i}: {line} {ids} != {tok_ids}" - - tokenizer.save(f"{pretrained.replace('/', '-')}.json") - return ("OK", trans_total_time / tok_total_time) - - -def main(): - pretraineds = [ - "albert-base-v1", - "albert-large-v1", - "albert-xlarge-v1", - "albert-xxlarge-v1", - "albert-base-v2", - "albert-large-v2", - "albert-xlarge-v2", - "albert-xxlarge-v2", - "camembert-base", - "xlm-roberta-base", - "xlm-roberta-large", - "xlm-roberta-large-finetuned-conll02-dutch", - "xlm-roberta-large-finetuned-conll02-spanish", - "xlm-roberta-large-finetuned-conll03-english", - "xlm-roberta-large-finetuned-conll03-german", - "facebook/mbart-large-en-ro", - "facebook/mbart-large-cc25", - "xlnet-base-cased", - "xlnet-large-cased", - "google/reformer-crime-and-punishment", - "t5-small", - "google/pegasus-large", - ] - parser = argparse.ArgumentParser() - parser.add_argument( - "--filename", - required=True, - type=str, - help="The filename that we are going to encode in both versions to check that conversion worked", - ) - parser.add_argument( - "--models", - type=lambda s: s.split(","), - default=pretraineds, - help=f"The pretrained tokenizers you want to test against, (default: {pretraineds})", - ) - args = parser.parse_args() - - print(args.filename) - - model_len = 50 - status_len = 6 - speedup_len = 8 - print(f"|{'Model':^{model_len}}|{'Status':^{status_len}}|{'Speedup':^{speedup_len}}|") - print(f"|{'-' * model_len}|{'-' * status_len}|{'-' * speedup_len}|") - for pretrained in args.models: - status, speedup = check(pretrained, args.filename) - print(f"|{pretrained:<{model_len}}|{status:^{status_len}}|{speedup:^{speedup_len - 1}.2f}x|") - - -if __name__ == "__main__": - main() diff --git a/bindings/python/scripts/sentencepiece_extractor.py b/bindings/python/scripts/sentencepiece_extractor.py deleted file mode 100644 index dc9b4edad..000000000 --- a/bindings/python/scripts/sentencepiece_extractor.py +++ /dev/null @@ -1,145 +0,0 @@ -from argparse import ArgumentParser -from json import dump -from logging import basicConfig, getLogger -from os import linesep, remove -from os.path import exists -from tempfile import NamedTemporaryFile -from typing import Dict, List, Tuple - -from requests import get -from sentencepiece import SentencePieceProcessor # type: ignore[import] -from tqdm import trange, tqdm - -basicConfig() -logger = getLogger() - - -class SentencePieceExtractor: - """ - Extractor implementation for SentencePiece trained models. - https://github.com/google/sentencepiece - """ - - def __init__(self, model: str): - # Get SentencePiece - self.sp = SentencePieceProcessor() - self.sp.Load(model) - - def extract(self) -> Tuple[Dict[str, int], List[Tuple]]: - sp = self.sp - vocab = {sp.id_to_piece(index): index for index in trange(sp.GetPieceSize())} # type: ignore[attr-defined] - - # Merges - merges = [] - for piece_l in tqdm(vocab.keys(), total=sp.GetPieceSize()): - for piece_r in vocab.keys(): - merge = f"{piece_l}{piece_r}" - piece_id = vocab.get(merge, None) - if piece_id: - merges += [(piece_l, piece_r, piece_id)] - merges = sorted(merges, key=lambda val: val[2]) - merges = [(val[0], val[1]) for val in merges] - - return vocab, merges - - -class YouTokenToMeExtractor: - """ - Extractor implementation for YouTokenToMe trained models format. - Model are as follow: - vocab_size nb_merges - piece piece_id - ...(repeated vocab_size) - piece_id_left piece_id_right piece_id - ...(repeated nb merges) - """ - - def __init__(self, model: str): - self._model = model - - def extract(self) -> Tuple[Dict[str, int], List[Tuple]]: - with open(self._model, "r") as model_f: - # Retrieve information - nb_pieces, nb_merges = map(int, model_f.readline().split()) - vocab, merges = {}, [] - - # Vocab - for _ in trange(nb_pieces): - piece, piece_id = map(int, model_f.readline().split()) - vocab[piece_id] = chr(piece) - - # Merges - for _ in trange(nb_merges): - piece_id_l, piece_id_r, piece = map(int, model_f.readline().split()) - piece_l, piece_r = vocab[piece_id_l], vocab[piece_id_r] - vocab[piece] = f"{piece_l}{piece_r}" - merges += [(piece_l, piece_r)] - - # Special tokens - unk, pad, bos, eos = map(int, model_f.readline().split()) - vocab[unk] = "" - vocab[pad] = "" - vocab[bos] = "" - vocab[eos] = "" - - # Invert key and value for vocab - vocab = dict(zip(vocab.values(), vocab.keys())) - return vocab, merges - - -if __name__ == "__main__": - parser = ArgumentParser("SentencePiece vocab extractor") - parser.add_argument( - "--provider", - type=str, - required=True, - choices=["sentencepiece", "youtokentome"], - help="Indicate the format of the file.", - ) - parser.add_argument("--model", type=str, required=True, help="SentencePiece model to extract vocab from.") - parser.add_argument( - "--vocab-output-path", - type=str, - required=True, - help="Path where the vocab.json file will be extracted", - ) - parser.add_argument( - "--merges-output-path", - type=str, - required=True, - help="Path where the merges file will be extracted", - ) - - # Parse cli arguments - args = parser.parse_args() - - try: - if args.model.startswith("http"): - # Saving model - with NamedTemporaryFile("wb", delete=False) as f: - logger.info("Writing content from {} to {}".format(args.model, f.name)) - response = get(args.model, allow_redirects=True) - f.write(response.content) - - args.remote_model = args.model - args.model = f.name - - # Allocate extractor - extractor = SentencePieceExtractor if args.provider == "sentencepiece" else YouTokenToMeExtractor - extractor = extractor(args.model) - - logger.info(f"Using {type(extractor).__name__}") - - # Open output files and let's extract model information - with open(args.vocab_output_path, "w") as vocab_f: - with open(args.merges_output_path, "w") as merges_f: - # Do the extraction - vocab, merges = extractor.extract() - - # Save content - dump(vocab, vocab_f) - merges_f.writelines(map(lambda x: f"{x[0]} {x[1]}{linesep}", merges)) - finally: - # If model was downloaded from internet we need to cleanup the tmp folder. - if hasattr(args, "remote_model") and exists(args.model): - remove(args.model) diff --git a/bindings/python/scripts/spm_parity_check.py b/bindings/python/scripts/spm_parity_check.py deleted file mode 100644 index e1d74bd89..000000000 --- a/bindings/python/scripts/spm_parity_check.py +++ /dev/null @@ -1,264 +0,0 @@ -import tokenizers -from argparse import ArgumentParser -import sentencepiece as spm -from collections import Counter -import json -import os -import datetime -from typing import Any, cast - -try: - from termcolor import colored - - has_color = True -except Exception: - has_color = False - - -def main(): - parser = ArgumentParser("SentencePiece parity checker") - parser.add_argument( - "--input-file", - "-i", - type=str, - required=True, - help="Which files do you want to train from", - ) - parser.add_argument( - "--model-file", - "-m", - type=str, - required=False, - default=None, - help="Use a pretrained token file", - ) - parser.add_argument( - "--model-prefix", - type=str, - default="spm_parity", - help="Model prefix for spm_train", - ) - parser.add_argument( - "--vocab-size", - "-v", - type=int, - default=8000, - help="Vocab size for spm_train", - ) - parser.add_argument( - "--verbose", - action="store_true", - help="Verbosity", - ) - parser.add_argument( - "--train", - action="store_true", - help="Instead of checking the encoder part, we check the trainer part", - ) - parser.add_argument( - "--from-spm", - action="store_true", - help="Directly load the spm file with it's own normalizer", - ) - - args = parser.parse_args() - - trained = False - if args.model_file is None: - spm.SentencePieceTrainer.Train( - f"--input={args.input_file} --model_prefix={args.model_prefix}" - f" --character_coverage=1.0" - f" --max_sentence_length=40000" - f" --num_threads=1" - f" --vocab_size={args.vocab_size}" - ) - trained = True - args.model_file = f"{args.model_prefix}.model" - - try: - if args.train: - check_train(args) - else: - check_encode(args) - finally: - if trained: - os.remove(f"{args.model_prefix}.model") - os.remove(f"{args.model_prefix}.vocab") - - -def check_train(args): - sp = spm.SentencePieceProcessor() - sp.Load(args.model_file) - - tokenizer = tokenizers.SentencePieceUnigramTokenizer() - tokenizer.train(args.input_file, show_progress=False) - - spm_tokens = 0 - tokenizer_tokens = 0 - - with open(args.input_file, "r") as f: - for i, line in enumerate(f): - line = line.strip() - ids = sp.EncodeAsIds(line) - - encoded = tokenizer.encode(line) - - spm_tokens += len(ids) - tokenizer_tokens += len(encoded.ids) - - vocab = [0 for i in range(args.vocab_size)] - spm_vocab = [0 for i in range(args.vocab_size)] - - for token, index in tokenizer.get_vocab().items(): - vocab[index] = token - - for i in range(args.vocab_size): - spm_vocab[i] = sp.id_to_piece(i) - - # 0 is unk in tokenizers, 0, 1, 2 are unk bos, eos in spm by default. - for i, (token, spm_token) in enumerate(zip(vocab[1:], spm_vocab[3:])): - if token != spm_token: - print(f"First different token is token {i} ({token} != {spm_token})") - break - - print(f"Tokenizer used {tokenizer_tokens}, where spm used {spm_tokens}") - assert tokenizer_tokens < spm_tokens, "Our trainer should be at least more efficient than the SPM one" - print("Ok our trainer is at least more efficient than the SPM one") - - -def check_diff(spm_diff, tok_diff, sp, tok): - if spm_diff == list(reversed(tok_diff)): - # AAA -> AA+A vs A+AA case. - return True - elif len(spm_diff) == len(tok_diff) and tok.decode(spm_diff) == tok.decode(tok_diff): - # Second order OK - # Barrich -> Barr + ich vs Bar + rich - return True - spm_reencoded = sp.encode(sp.decode(spm_diff)) - tok_reencoded = tok.encode(tok.decode(spm_diff)).ids - if spm_reencoded != spm_diff and spm_reencoded == tok_reencoded: - # Type 3 error. - # Snehagatha -> - # Sne, h, aga, th, a - # Sne, ha, gat, ha - # Encoding the wrong with sp does not even recover what spm gave us - # It fits tokenizer however... - return True - return False - - -def check_details(line, spm_ids, tok_ids, sp, tok): - # Encoding can be the same with same result AAA -> A + AA vs AA + A - # We can check that we use at least exactly the same number of tokens. - for i, (spm_id, tok_id) in enumerate(zip(spm_ids, tok_ids)): - if spm_id != tok_id: - break - first = i - for i, (spm_id, tok_id) in enumerate(zip(reversed(spm_ids), reversed(tok_ids))): - if spm_id != tok_id: - break - last = len(spm_ids) - i - - spm_diff = spm_ids[first:last] - tok_diff = tok_ids[first:last] - - if check_diff(spm_diff, tok_diff, sp, tok): - return True - - if last - first > 5: - # We might have twice a single problem, attempt to subdivide the disjointed tokens into smaller problems - spms = Counter(spm_ids[first:last]) - toks = Counter(tok_ids[first:last]) - - removable_tokens = {spm_ for (spm_, si) in spms.items() if toks.get(spm_, 0) == si} - min_width = 3 - for i in range(last - first - min_width): - if all(spm_ids[first + i + j] in removable_tokens for j in range(min_width)): - possible_matches = [ - k - for k in range(last - first - min_width) - if tok_ids[first + k : first + k + min_width] == spm_ids[first + i : first + i + min_width] - ] - for j in possible_matches: - if check_diff(spm_ids[first : first + i], tok_ids[first : first + j], sp, tok) and check_details( - line, - spm_ids[first + i : last], - tok_ids[first + j : last], - sp, - tok, - ): - return True - - print(f"Spm: {[tok.decode([spm_ids[i]]) for i in range(first, last)]}") - try: - print(f"Tok: {[tok.decode([tok_ids[i]]) for i in range(first, last)]}") - except Exception: - pass - - ok_start = tok.decode(spm_ids[:first]) - ok_end = tok.decode(spm_ids[last:]) - wrong = tok.decode(spm_ids[first:last]) - print() - if has_color: - print(f"{colored(ok_start, 'grey')}{colored(wrong, 'red')}{colored(ok_end, 'grey')}") - else: - print(wrong) - return False - - -def check_encode(args): - sp = cast(Any, spm.SentencePieceProcessor()) - sp.Load(args.model_file) - - if args.from_spm: - tok = tokenizers.SentencePieceUnigramTokenizer.from_spm(args.model_file) - else: - vocab = [(sp.id_to_piece(i), sp.get_score(i)) for i in range(sp.piece_size())] - unk_id = sp.unk_id() - tok = tokenizers.SentencePieceUnigramTokenizer(vocab, unk_id) - - perfect = 0 - imperfect = 0 - wrong = 0 - now = datetime.datetime.now - spm_total_time = datetime.timedelta(seconds=0) - tok_total_time = datetime.timedelta(seconds=0) - with open(args.input_file, "r", encoding="utf-8-sig") as f: - for i, line in enumerate(f): - line = line.strip() - - start = now() - ids = sp.EncodeAsIds(line) - spm_time = now() - - encoded = tok.encode(line) - tok_time = now() - - spm_total_time += spm_time - start - tok_total_time += tok_time - spm_time - - if args.verbose: - if i % 10000 == 0: - print(f"({perfect} / {imperfect} / {wrong} ----- {perfect + imperfect + wrong})") - print(f"SPM: {spm_total_time} - TOK: {tok_total_time}") - - if ids != encoded.ids: - if check_details(line, ids, encoded.ids, sp, tok): - imperfect += 1 - continue - else: - wrong += 1 - else: - perfect += 1 - - assert ids == encoded.ids, ( - f"line {i}: {line} : \n\n{ids}\n{encoded.ids}\n{list(zip(encoded.ids, encoded.tokens))}" - ) - - print(f"({perfect} / {imperfect} / {wrong} ----- {perfect + imperfect + wrong})") - total = perfect + imperfect + wrong - print(f"Accuracy {perfect * 100 / total:.2f} Slowdown : {tok_total_time / spm_total_time:.2f}") - - -if __name__ == "__main__": - main() diff --git a/bindings/python/setup.cfg b/bindings/python/setup.cfg deleted file mode 100644 index 4b1039cfd..000000000 --- a/bindings/python/setup.cfg +++ /dev/null @@ -1,55 +0,0 @@ -[isort] -default_section = FIRSTPARTY -ensure_newline_before_comments = True -force_grid_wrap = 0 -include_trailing_comma = True -known_first_party = transformers -known_third_party = - absl - conllu - datasets - elasticsearch - fairseq - faiss-cpu - fastprogress - fire - fugashi - git - h5py - matplotlib - nltk - numpy - packaging - pandas - PIL - psutil - pytest - pytorch_lightning - rouge_score - sacrebleu - seqeval - sklearn - streamlit - tensorboardX - tensorflow - tensorflow_datasets - timeout_decorator - torch - torchaudio - torchtext - torchvision - torch_xla - tqdm - -line_length = 119 -lines_after_imports = 2 -multi_line_output = 3 -use_parentheses = True - -[flake8] -ignore = E203, E501, E741, W503, W605 -max-line-length = 119 - -[tool:pytest] -doctest_optionflags=NUMBER NORMALIZE_WHITESPACE ELLIPSIS -pythonpath = py_src diff --git a/bindings/python-pipeline/src/added_token.rs b/bindings/python/src/added_token.rs similarity index 96% rename from bindings/python-pipeline/src/added_token.rs rename to bindings/python/src/added_token.rs index e1136b949..0deca0c3a 100644 --- a/bindings/python-pipeline/src/added_token.rs +++ b/bindings/python/src/added_token.rs @@ -7,12 +7,7 @@ use tk_encode::tokenizer::AddedToken; /// it; `normalized` matches against normalized instead of raw text (defaults /// to the opposite of `special`); `special` marks template tokens like "" /// that decoding should be able to skip. -#[pyclass( - frozen, - from_py_object, - name = "AddedToken", - module = "tokenizers_pipeline" -)] +#[pyclass(frozen, from_py_object, name = "AddedToken", module = "tokenizers")] #[derive(Clone)] pub struct PyAddedToken { pub inner: AddedToken, diff --git a/bindings/python/src/decoders.rs b/bindings/python/src/decoders.rs deleted file mode 100644 index 46dbab4b0..000000000 --- a/bindings/python/src/decoders.rs +++ /dev/null @@ -1,933 +0,0 @@ -use std::sync::{Arc, RwLock}; - -use crate::pre_tokenizers::from_string; -use crate::tokenizer::PyTokenizer; -use crate::utils::PyPattern; -use pyo3::exceptions; -use pyo3::prelude::*; -use pyo3::types::*; -use serde::de::Error; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; -use tk::Decoder; -use tk::decoders::DecoderWrapper; -use tk::decoders::bpe::BPEDecoder; -use tk::decoders::byte_fallback::ByteFallback; -use tk::decoders::byte_level::ByteLevel; -use tk::decoders::ctc::CTC; -use tk::decoders::fuse::Fuse; -use tk::decoders::metaspace::{Metaspace, PrependScheme}; -use tk::decoders::sequence::Sequence; -use tk::decoders::strip::Strip; -use tk::decoders::wordpiece::WordPiece; -use tk::normalizers::replace::Replace; -use tokenizers as tk; - -use super::error::ToPyResult; - -/// Base class for all decoders -/// -/// This class is not supposed to be instantiated directly. Instead, any implementation of -/// a Decoder will return an instance of this class when instantiated. -#[pyclass( - dict, - module = "tokenizers.decoders", - name = "Decoder", - subclass, - from_py_object -)] -#[derive(Clone, Deserialize, Serialize)] -#[serde(transparent)] -pub struct PyDecoder { - pub(crate) decoder: PyDecoderWrapper, -} - -impl PyDecoder { - pub(crate) fn new(decoder: PyDecoderWrapper) -> Self { - PyDecoder { decoder } - } - - pub(crate) fn get_as_subtype(&self, py: Python<'_>) -> PyResult> { - let base = self.clone(); - Ok(match &self.decoder { - PyDecoderWrapper::Custom(_) => Py::new(py, base)?.into_any(), - PyDecoderWrapper::Wrapped(inner) => match &*inner.as_ref().read().unwrap() { - DecoderWrapper::Metaspace(_) => Py::new(py, (PyMetaspaceDec {}, base))?.into_any(), - DecoderWrapper::WordPiece(_) => Py::new(py, (PyWordPieceDec {}, base))?.into_any(), - DecoderWrapper::ByteFallback(_) => { - Py::new(py, (PyByteFallbackDec {}, base))?.into_any() - } - DecoderWrapper::Strip(_) => Py::new(py, (PyStrip {}, base))?.into_any(), - DecoderWrapper::Fuse(_) => Py::new(py, (PyFuseDec {}, base))?.into_any(), - DecoderWrapper::ByteLevel(_) => Py::new(py, (PyByteLevelDec {}, base))?.into_any(), - DecoderWrapper::Replace(_) => Py::new(py, (PyReplaceDec {}, base))?.into_any(), - DecoderWrapper::BPE(_) => Py::new(py, (PyBPEDecoder {}, base))?.into_any(), - DecoderWrapper::CTC(_) => Py::new(py, (PyCTCDecoder {}, base))?.into_any(), - DecoderWrapper::Sequence(_) => { - Py::new(py, (PySequenceDecoder {}, base))?.into_any() - } - }, - }) - } -} - -impl Decoder for PyDecoder { - fn decode_chain(&self, tokens: Vec) -> tk::Result> { - self.decoder.decode_chain(tokens) - } -} - -#[pymethods] -impl PyDecoder { - #[staticmethod] - #[pyo3(text_signature = "(decoder)")] - fn custom(decoder: Py) -> Self { - let decoder = PyDecoderWrapper::Custom(Arc::new(RwLock::new(CustomDecoder::new(decoder)))); - PyDecoder::new(decoder) - } - - fn __getstate__(&self, py: Python) -> PyResult> { - let data = serde_json::to_string(&self.decoder).map_err(|e| { - exceptions::PyException::new_err(format!( - "Error while attempting to pickle Decoder: {e}" - )) - })?; - Ok(PyBytes::new(py, data.as_bytes()).into()) - } - - fn __setstate__(&mut self, py: Python, state: Py) -> PyResult<()> { - match state.extract::<&[u8]>(py) { - Ok(s) => { - self.decoder = serde_json::from_slice(s).map_err(|e| { - exceptions::PyException::new_err(format!( - "Error while attempting to unpickle Decoder: {e}" - )) - })?; - Ok(()) - } - Err(e) => Err(e.into()), - } - } - - /// Decode the given list of tokens to a final string - /// - /// Args: - /// tokens (:obj:`List[str]`): - /// The list of tokens to decode - /// - /// Returns: - /// :obj:`str`: The decoded string - #[pyo3(text_signature = "(self, tokens)")] - fn decode(&self, tokens: Vec) -> PyResult { - ToPyResult(self.decoder.decode(tokens)).into() - } - - fn __repr__(&self) -> PyResult { - crate::utils::serde_pyo3::repr(self) - .map_err(|e| exceptions::PyException::new_err(e.to_string())) - } - - fn __str__(&self) -> PyResult { - crate::utils::serde_pyo3::to_string(self) - .map_err(|e| exceptions::PyException::new_err(e.to_string())) - } -} - -macro_rules! getter { - ($self: ident, $variant: ident, $($name: tt)+) => {{ - let super_ = $self.as_ref(); - if let PyDecoderWrapper::Wrapped(ref wrap) = super_.decoder { - if let DecoderWrapper::$variant(ref dec) = *wrap.read().unwrap() { - dec.$($name)+ - } else { - unreachable!() - } - } else { - unreachable!() - } - }}; -} - -macro_rules! setter { - ($self: ident, $variant: ident, $name: ident, $value: expr) => {{ - let super_ = $self.as_ref(); - if let PyDecoderWrapper::Wrapped(ref wrap) = super_.decoder { - if let DecoderWrapper::$variant(ref mut dec) = *wrap.write().unwrap() { - dec.$name = $value; - } - } - }}; - ($self: ident, $variant: ident, @$name: ident, $value: expr) => {{ - let super_ = $self.as_ref(); - if let PyDecoderWrapper::Wrapped(ref wrap) = super_.decoder { - if let DecoderWrapper::$variant(ref mut dec) = *wrap.write().unwrap() { - dec.$name($value); - } - } - }}; -} - -/// ByteLevel Decoder -/// -/// This decoder is to be used in tandem with the -/// :class:`~tokenizers.pre_tokenizers.ByteLevel` pre-tokenizer. It reverses the -/// byte-to-unicode mapping applied during pre-tokenization, converting the special -/// Unicode characters back into the original bytes to reconstruct the original string. -/// -/// Example:: -/// -/// >>> from tokenizers.decoders import ByteLevel -/// >>> decoder = ByteLevel() -/// >>> decoder.decode(["ĠHello", "Ġworld"]) -/// ' Hello world' -/// -#[pyclass(extends=PyDecoder, module = "tokenizers.decoders", name = "ByteLevel")] -pub struct PyByteLevelDec {} -#[pymethods] -impl PyByteLevelDec { - #[new] - #[pyo3(signature = (**_kwargs), text_signature = "(self)")] - fn new(_kwargs: Option<&Bound<'_, PyDict>>) -> PyClassInitializer { - PyClassInitializer::::from(PyDecoder::from(ByteLevel::default())) - .add_subclass(PyByteLevelDec {}) - } -} - -/// Replace Decoder -/// -/// This decoder is to be used in tandem with the -/// :class:`~tokenizers.normalizers.Replace` normalizer or a similar replace operation. -/// It reverses a string replacement by substituting the replacement content back -/// with the original pattern. -/// -/// Args: -/// pattern (:obj:`str` or :class:`~tokenizers.Regex`): -/// The pattern that was used as the replacement target during encoding. -/// -/// content (:obj:`str`): -/// The string to replace each match of the pattern with during decoding. -/// -/// Example:: -/// -/// >>> from tokenizers.decoders import Replace -/// >>> decoder = Replace("▁", " ") -/// >>> decoder.decode(["▁Hello", "▁world"]) -/// ' Hello world' -/// -#[pyclass(extends=PyDecoder, module = "tokenizers.decoders", name = "Replace")] -pub struct PyReplaceDec {} -#[pymethods] -impl PyReplaceDec { - #[new] - #[pyo3(text_signature = "(self, pattern, content)")] - fn new(pattern: PyPattern, content: String) -> PyResult> { - Ok(PyClassInitializer::::from(PyDecoder::from( - ToPyResult(Replace::new(pattern, content)).into_py()?, - )) - .add_subclass(PyReplaceDec {})) - } -} - -/// WordPiece Decoder -/// -/// Args: -/// prefix (:obj:`str`, `optional`, defaults to :obj:`##`): -/// The prefix to use for subwords that are not a beginning-of-word -/// -/// cleanup (:obj:`bool`, `optional`, defaults to :obj:`True`): -/// Whether to cleanup some tokenization artifacts. Mainly spaces before punctuation, -/// and some abbreviated english forms. -/// -/// Example:: -/// -/// >>> from tokenizers.decoders import WordPiece -/// >>> decoder = WordPiece() -/// >>> decoder.decode(["Hello", ",", "##world", "!"]) -/// 'Hello, world!' -/// -#[pyclass(extends=PyDecoder, module = "tokenizers.decoders", name = "WordPiece")] -pub struct PyWordPieceDec {} -#[pymethods] -impl PyWordPieceDec { - #[getter] - fn get_prefix(self_: PyRef) -> String { - getter!(self_, WordPiece, prefix.clone()) - } - - #[setter] - fn set_prefix(self_: PyRef, prefix: String) { - setter!(self_, WordPiece, prefix, prefix); - } - - #[getter] - fn get_cleanup(self_: PyRef) -> bool { - getter!(self_, WordPiece, cleanup) - } - - #[setter] - fn set_cleanup(self_: PyRef, cleanup: bool) { - setter!(self_, WordPiece, cleanup, cleanup); - } - - #[new] - #[pyo3(signature = (prefix = String::from("##"), cleanup = true), text_signature = "(self, prefix=\"##\", cleanup=True)")] - fn new(prefix: String, cleanup: bool) -> PyClassInitializer { - PyClassInitializer::::from(PyDecoder::from(WordPiece::new(prefix, cleanup))) - .add_subclass(PyWordPieceDec {}) - } -} - -/// ByteFallback Decoder -/// -/// ByteFallback is a decoder that handles tokens representing raw bytes in the -/// ``<0xNN>`` format (e.g., ``<0x61>`` for the byte ``0x61`` = ``'a'``). It converts -/// such tokens to their corresponding bytes and attempts to decode the resulting byte -/// sequence as UTF-8. This is used in LLaMA/SentencePiece models that use byte fallback -/// for unknown characters. Inconvertible byte tokens are replaced with the Unicode -/// replacement character (U+FFFD). -/// -/// Example:: -/// -/// >>> from tokenizers.decoders import ByteFallback, Fuse, Sequence -/// >>> decoder = Sequence([ByteFallback(), Fuse()]) -/// >>> decoder.decode(["<0x48>", "<0x65>", "<0x6C>", "<0x6C>", "<0x6F>"]) -/// 'Hello' -/// -#[pyclass(extends=PyDecoder, module = "tokenizers.decoders", name = "ByteFallback")] -pub struct PyByteFallbackDec {} -#[pymethods] -impl PyByteFallbackDec { - #[new] - #[pyo3(signature = (), text_signature = "(self)")] - fn new() -> PyClassInitializer { - PyClassInitializer::::from(PyDecoder::from(ByteFallback::new())) - .add_subclass(PyByteFallbackDec {}) - } -} - -/// Fuse Decoder -/// -/// Fuse simply concatenates every token into a single string without any separator. -/// This is typically the last step in a decoder chain when other decoders need to -/// operate on individual tokens before they are joined together. -/// -/// Example:: -/// -/// >>> from tokenizers.decoders import Fuse -/// >>> decoder = Fuse() -/// >>> decoder.decode(["Hello", ",", " ", "world", "!"]) -/// 'Hello, world!' -/// -#[pyclass(extends=PyDecoder, module = "tokenizers.decoders", name = "Fuse")] -pub struct PyFuseDec {} -#[pymethods] -impl PyFuseDec { - #[new] - #[pyo3(signature = (), text_signature = "(self)")] - fn new() -> PyClassInitializer { - PyClassInitializer::::from(PyDecoder::from(Fuse::new())) - .add_subclass(PyFuseDec {}) - } -} - -/// Strip Decoder -/// -/// Strips a given number of occurrences of a character from the left and/or right -/// side of each token. This is useful for removing padding characters or special -/// prefix/suffix markers added during tokenization. -/// -/// Args: -/// content (:obj:`str`, defaults to :obj:`" "`): -/// The character to strip from each token. -/// -/// left (:obj:`int`, defaults to :obj:`0`): -/// The number of occurrences of :obj:`content` to remove from the left -/// side of each token. -/// -/// right (:obj:`int`, defaults to :obj:`0`): -/// The number of occurrences of :obj:`content` to remove from the right -/// side of each token. -/// -/// Example:: -/// -/// >>> from tokenizers.decoders import Strip -/// >>> decoder = Strip(content="▁", left=1) -/// >>> decoder.decode(["▁Hello", "▁world"]) -/// 'Hello world' -/// -#[pyclass(extends=PyDecoder, module = "tokenizers.decoders", name = "Strip")] -pub struct PyStrip {} -#[pymethods] -impl PyStrip { - #[getter] - fn get_start(self_: PyRef) -> usize { - getter!(self_, Strip, start) - } - - #[setter] - fn set_start(self_: PyRef, start: usize) { - setter!(self_, Strip, start, start) - } - - #[getter] - fn get_stop(self_: PyRef) -> usize { - getter!(self_, Strip, stop) - } - - #[setter] - fn set_stop(self_: PyRef, stop: usize) { - setter!(self_, Strip, stop, stop) - } - - #[getter] - fn get_content(self_: PyRef) -> char { - getter!(self_, Strip, content) - } - - #[setter] - fn set_content(self_: PyRef, content: char) { - setter!(self_, Strip, content, content) - } - - #[new] - #[pyo3( - signature = (content=' ', left=0, right=0), - text_signature = "(self, content=' ', left=0, right=0)" - )] - fn new(content: char, left: usize, right: usize) -> PyClassInitializer { - PyClassInitializer::::from(PyDecoder::from(Strip::new(content, left, right))) - .add_subclass(PyStrip {}) - } -} - -/// Metaspace Decoder -/// -/// Args: -/// replacement (:obj:`str`, `optional`, defaults to :obj:`▁`): -/// The replacement character. Must be exactly one character. By default we -/// use the `▁` (U+2581) meta symbol (Same as in SentencePiece). -/// -/// prepend_scheme (:obj:`str`, `optional`, defaults to :obj:`"always"`): -/// Whether to add a space to the first word if there isn't already one. This -/// lets us treat `hello` exactly like `say hello`. -/// Choices: "always", "never", "first". First means the space is only added on the first -/// token (relevant when special tokens are used or other pre_tokenizer are used). -/// -/// Example:: -/// -/// >>> from tokenizers.decoders import Metaspace -/// >>> decoder = Metaspace() -/// >>> decoder.decode(["▁Hello", "▁my", "▁friend"]) -/// 'Hello my friend' -/// -#[pyclass(extends=PyDecoder, module = "tokenizers.decoders", name = "Metaspace")] -pub struct PyMetaspaceDec {} -#[pymethods] -impl PyMetaspaceDec { - #[getter] - fn get_replacement(self_: PyRef) -> String { - getter!(self_, Metaspace, get_replacement().to_string()) - } - - #[setter] - fn set_replacement(self_: PyRef, replacement: char) { - setter!(self_, Metaspace, @set_replacement, replacement); - } - - #[getter] - fn get_split(self_: PyRef) -> bool { - getter!(self_, Metaspace, get_split()) - } - - #[setter] - fn set_split(self_: PyRef, split: bool) { - setter!(self_, Metaspace, @set_split, split); - } - - #[getter] - fn get_prepend_scheme(self_: PyRef) -> String { - // Assuming Metaspace has a method to get the prepend_scheme as a string - let scheme: PrependScheme = getter!(self_, Metaspace, get_prepend_scheme()); - match scheme { - PrependScheme::First => "first", - PrependScheme::Never => "never", - PrependScheme::Always => "always", - } - .to_string() - } - - #[setter] - fn set_prepend_scheme(self_: PyRef, prepend_scheme: String) -> PyResult<()> { - let scheme = from_string(prepend_scheme)?; - setter!(self_, Metaspace, @set_prepend_scheme, scheme); - Ok(()) - } - - #[new] - #[pyo3(signature = (replacement = '▁', prepend_scheme = String::from("always"), split = true), text_signature = "(self, replacement = \"▁\", prepend_scheme = \"always\", split = True)")] - fn new( - replacement: char, - prepend_scheme: String, - split: bool, - ) -> PyResult> { - let prepend_scheme = from_string(prepend_scheme)?; - Ok( - PyClassInitializer::::from(PyDecoder::from(Metaspace::new( - replacement, - prepend_scheme, - split, - ))) - .add_subclass(PyMetaspaceDec {}), - ) - } -} - -/// BPEDecoder Decoder -/// -/// Args: -/// suffix (:obj:`str`, `optional`, defaults to :obj:``): -/// The suffix that was used to characterize an end-of-word. This suffix will -/// be replaced by whitespaces during the decoding -/// -/// Example:: -/// -/// >>> from tokenizers.decoders import BPEDecoder -/// >>> decoder = BPEDecoder() -/// >>> decoder.decode(["Hello", "world"]) -/// 'Hello world' -/// -#[pyclass(extends=PyDecoder, module = "tokenizers.decoders", name = "BPEDecoder")] -pub struct PyBPEDecoder {} -#[pymethods] -impl PyBPEDecoder { - #[getter] - fn get_suffix(self_: PyRef) -> String { - getter!(self_, BPE, suffix.clone()) - } - - #[setter] - fn set_suffix(self_: PyRef, suffix: String) { - setter!(self_, BPE, suffix, suffix); - } - - #[new] - #[pyo3(signature = (suffix = String::from("")), text_signature = "(self, suffix=\"\")")] - fn new(suffix: String) -> PyClassInitializer { - PyClassInitializer::::from(PyDecoder::from(BPEDecoder::new(suffix))) - .add_subclass(PyBPEDecoder {}) - } -} - -/// CTC Decoder -/// -/// Args: -/// pad_token (:obj:`str`, `optional`, defaults to :obj:``): -/// The pad token used by CTC to delimit a new token. -/// word_delimiter_token (:obj:`str`, `optional`, defaults to :obj:`|`): -/// The word delimiter token. It will be replaced by a -/// cleanup (:obj:`bool`, `optional`, defaults to :obj:`True`): -/// Whether to cleanup some tokenization artifacts. -/// Mainly spaces before punctuation, and some abbreviated english forms. -/// -/// Example:: -/// -/// >>> from tokenizers.decoders import CTC -/// >>> decoder = CTC() -/// >>> decoder.decode(["h", "e", "e", "", "l", "l", "o", "|", "w", "o", "r", "l", "d"]) -/// 'hello world' -/// -#[pyclass(extends=PyDecoder, module = "tokenizers.decoders", name = "CTC")] -pub struct PyCTCDecoder {} -#[pymethods] -impl PyCTCDecoder { - #[getter] - fn get_pad_token(self_: PyRef) -> String { - getter!(self_, CTC, pad_token.clone()) - } - - #[setter] - fn set_pad_token(self_: PyRef, pad_token: String) { - setter!(self_, CTC, pad_token, pad_token); - } - - #[getter] - fn get_word_delimiter_token(self_: PyRef) -> String { - getter!(self_, CTC, word_delimiter_token.clone()) - } - - #[setter] - fn set_word_delimiter_token(self_: PyRef, word_delimiter_token: String) { - setter!(self_, CTC, word_delimiter_token, word_delimiter_token); - } - - #[getter] - fn get_cleanup(self_: PyRef) -> bool { - getter!(self_, CTC, cleanup) - } - - #[setter] - fn set_cleanup(self_: PyRef, cleanup: bool) { - setter!(self_, CTC, cleanup, cleanup); - } - - #[new] - #[pyo3(signature = ( - pad_token = String::from(""), - word_delimiter_token = String::from("|"), - cleanup = true - ), - text_signature = "(self, pad_token=\"\", word_delimiter_token=\"|\", cleanup=True)")] - fn new( - pad_token: String, - word_delimiter_token: String, - cleanup: bool, - ) -> PyClassInitializer { - PyClassInitializer::::from(PyDecoder::from(CTC::new( - pad_token, - word_delimiter_token, - cleanup, - ))) - .add_subclass(PyCTCDecoder {}) - } -} - -/// Sequence Decoder -/// -/// Chains multiple decoders together, applying them in order. Each decoder in the -/// sequence processes the output of the previous one, allowing complex decoding -/// pipelines to be built from simpler components. -/// -/// Args: -/// decoders (:obj:`List[Decoder]`): -/// The list of decoders to chain together. -/// -/// Example:: -/// -/// >>> from tokenizers.decoders import ByteFallback, Fuse, Metaspace, Sequence -/// >>> decoder = Sequence([ByteFallback(), Fuse(), Metaspace()]) -/// >>> decoder.decode(["▁Hello", "▁world"]) -/// 'Hello world' -/// -#[pyclass(extends=PyDecoder, module = "tokenizers.decoders", name="Sequence")] -pub struct PySequenceDecoder {} -#[pymethods] -impl PySequenceDecoder { - #[new] - #[pyo3(signature = (decoders_py), text_signature = "(self, decoders)")] - fn new(decoders_py: &Bound<'_, PyList>) -> PyResult> { - let mut decoders: Vec = Vec::with_capacity(decoders_py.len()); - for decoder_py in decoders_py.iter() { - let decoder: PyRef = decoder_py.extract()?; - let decoder = match &decoder.decoder { - PyDecoderWrapper::Wrapped(inner) => inner, - PyDecoderWrapper::Custom(_) => unimplemented!(), - }; - decoders.push(decoder.read().unwrap().clone()); - } - Ok( - PyClassInitializer::::from(PyDecoder::from(Sequence::new(decoders))) - .add_subclass(PySequenceDecoder {}), - ) - } - - fn __getnewargs__<'p>(&self, py: Python<'p>) -> PyResult> { - PyTuple::new(py, [PyList::empty(py)]) - } -} - -pub(crate) struct CustomDecoder { - inner: Py, -} - -impl CustomDecoder { - pub(crate) fn new(inner: Py) -> Self { - CustomDecoder { inner } - } -} - -impl Decoder for CustomDecoder { - fn decode(&self, tokens: Vec) -> tk::Result { - Python::attach(|py| { - let decoded = self - .inner - .call_method(py, "decode", (tokens,), None)? - .extract(py)?; - Ok(decoded) - }) - } - - fn decode_chain(&self, tokens: Vec) -> tk::Result> { - Python::attach(|py| { - let decoded = self - .inner - .call_method(py, "decode_chain", (tokens,), None)? - .extract(py)?; - Ok(decoded) - }) - } -} - -impl Serialize for CustomDecoder { - fn serialize(&self, _serializer: S) -> std::result::Result - where - S: Serializer, - { - Err(serde::ser::Error::custom( - "Custom PyDecoder cannot be serialized", - )) - } -} - -impl<'de> Deserialize<'de> for CustomDecoder { - fn deserialize(_deserializer: D) -> std::result::Result - where - D: Deserializer<'de>, - { - Err(D::Error::custom("PyDecoder cannot be deserialized")) - } -} - -#[derive(Clone, Deserialize, Serialize)] -#[serde(untagged)] -pub(crate) enum PyDecoderWrapper { - Custom(Arc>), - Wrapped(Arc>), -} - -impl From for PyDecoderWrapper -where - I: Into, -{ - fn from(norm: I) -> Self { - PyDecoderWrapper::Wrapped(Arc::new(RwLock::new(norm.into()))) - } -} - -impl From for PyDecoder -where - I: Into, -{ - fn from(dec: I) -> Self { - PyDecoder { - decoder: dec.into().into(), - } - } -} - -impl Decoder for PyDecoderWrapper { - fn decode_chain(&self, tokens: Vec) -> tk::Result> { - match self { - PyDecoderWrapper::Wrapped(inner) => inner.read().unwrap().decode_chain(tokens), - PyDecoderWrapper::Custom(inner) => inner.read().unwrap().decode_chain(tokens), - } - } -} - -/// Decoders Module -#[pymodule(gil_used = false)] -pub mod decoders { - #[pymodule_export] - pub use super::PyBPEDecoder; - #[pymodule_export] - pub use super::PyByteFallbackDec; - #[pymodule_export] - pub use super::PyByteLevelDec; - #[pymodule_export] - pub use super::PyCTCDecoder; - #[pymodule_export] - pub use super::PyDecodeStream; - #[pymodule_export] - pub use super::PyDecoder; - #[pymodule_export] - pub use super::PyFuseDec; - #[pymodule_export] - pub use super::PyMetaspaceDec; - #[pymodule_export] - pub use super::PyReplaceDec; - #[pymodule_export] - pub use super::PySequenceDecoder; - #[pymodule_export] - pub use super::PyStrip; - #[pymodule_export] - pub use super::PyWordPieceDec; -} - -/// Provides incremental decoding of token IDs as they are generated, yielding -/// decoded text chunks as soon as they are available. -/// -/// Unlike batch decoding, streaming decode is designed for use with autoregressive -/// generation — tokens arrive one at a time and the decoder needs to handle -/// multi-byte sequences (e.g., UTF-8 characters split across token boundaries) and -/// byte-fallback tokens gracefully. -/// -/// The decoder internally buffers tokens until it can produce a valid UTF-8 string -/// chunk, then yields that chunk and advances its internal state. This means -/// individual calls to :meth:`~tokenizers.decoders.DecodeStream.step` may return -/// :obj:`None` when the current token completes a partial sequence that cannot yet -/// be decoded. -/// -/// Args: -/// skip_special_tokens (:obj:`bool`, defaults to :obj:`False`): -/// Whether to skip special tokens (e.g. ``[CLS]``, ``[SEP]``, ````) when -/// decoding. -/// -/// Example:: -/// -/// >>> from tokenizers import Tokenizer -/// >>> from tokenizers.decoders import DecodeStream -/// >>> tokenizer = Tokenizer.from_pretrained("gpt2") -/// >>> stream = DecodeStream(skip_special_tokens=True) -/// >>> # Simulate streaming token-by-token generation -/// >>> token_ids = tokenizer.encode("Hello, streaming world!").ids -/// >>> for token_id in token_ids: -/// ... chunk = stream.step(tokenizer, token_id) -/// ... if chunk is not None: -/// ... print(chunk, end="", flush=True) -/// -#[pyclass(module = "tokenizers.decoders", name = "DecodeStream", from_py_object)] -#[derive(Clone)] -pub struct PyDecodeStream { - /// Regular decode option that is kept throughout. - skip_special_tokens: bool, - /// A temporary buffer of the necessary token_ids needed - /// to produce valid string chunks. - /// This typically contains 3 parts: - /// - read - /// - prefix - /// - rest - /// - /// Read is the bit necessary to surround the prefix - /// so decoding the whole ids produces a valid prefix. - /// Prefix is the previously produced string, kept around to trim off of - /// the next valid chunk - ids: Vec, - /// The previously returned chunk that needs to be discarded from the - /// decoding of the current ids to produce the next chunk - prefix: String, - /// The index within the ids corresponding to the prefix so we can drain - /// correctly - prefix_index: usize, -} - -#[derive(Clone)] -enum StreamInput { - Id(u32), - Ids(Vec), -} - -impl<'a, 'py> FromPyObject<'a, 'py> for StreamInput { - type Error = PyErr; - - fn extract(obj: Borrowed<'a, 'py, PyAny>) -> Result { - if let Ok(id) = obj.extract::() { - Ok(StreamInput::Id(id)) - } else if let Ok(ids) = obj.extract::>() { - Ok(StreamInput::Ids(ids)) - } else { - Err(PyErr::new::( - "StreamInput must be either an integer or a list of integers", - )) - } - } -} - -#[pymethods] -impl PyDecodeStream { - #[new] - #[pyo3(signature = (ids=None, skip_special_tokens=false), text_signature = "(self, ids=None, skip_special_tokens=False)")] - fn new(ids: Option>, skip_special_tokens: Option) -> Self { - PyDecodeStream { - skip_special_tokens: skip_special_tokens.unwrap_or(false), - ids: ids.unwrap_or_default(), - prefix: String::new(), - prefix_index: 0, - } - } - - /// Add the next token ID (or list of IDs) to the stream and return the next - /// decoded text chunk if one is available. - /// - /// Because some characters span multiple tokens (e.g. multi-byte UTF-8 - /// sequences or byte-fallback tokens), this method may return :obj:`None` - /// when the provided token does not yet complete a decodable unit. Callers - /// should simply continue feeding tokens until a non-:obj:`None` value is - /// returned. - /// - /// Args: - /// tokenizer (:class:`~tokenizers.Tokenizer`): - /// The tokenizer whose decoder pipeline will be used. - /// - /// id (:obj:`int` or :obj:`List[int]`): - /// The next token ID, or a list of token IDs to append to the stream. - /// - /// Returns: - /// :obj:`Optional[str]`: The next decoded text chunk if enough tokens have - /// accumulated, or :obj:`None` if more tokens are still needed. - #[pyo3(signature = (tokenizer, id), text_signature = "(self, tokenizer, id)")] - fn step(&mut self, tokenizer: &PyTokenizer, id: StreamInput) -> PyResult> { - let id: Vec = match id { - StreamInput::Id(id) => vec![id], - StreamInput::Ids(ids) => ids, - }; - let tokenizer_guard = tokenizer.read_inner()?; - ToPyResult(tk::tokenizer::step_decode_stream( - &tokenizer_guard, - id, - self.skip_special_tokens, - &mut self.ids, - &mut self.prefix, - &mut self.prefix_index, - )) - .into() - } - fn __copy__(&self) -> Self { - self.clone() - } - - fn __deepcopy__(&self, _memo: &Bound<'_, PyDict>) -> Self { - self.clone() - } -} - -#[cfg(test)] -mod test { - use std::sync::{Arc, RwLock}; - - use pyo3::prelude::*; - use tk::decoders::DecoderWrapper; - use tk::decoders::metaspace::Metaspace; - - use crate::decoders::{CustomDecoder, PyDecoder, PyDecoderWrapper}; - - #[test] - fn get_subtype() { - Python::attach(|py| { - let py_dec = PyDecoder::new(Metaspace::default().into()); - let py_meta = py_dec.get_as_subtype(py).unwrap(); - assert_eq!("Metaspace", py_meta.bind(py).get_type().qualname().unwrap()); - }) - } - - #[test] - fn serialize() { - let py_wrapped: PyDecoderWrapper = Metaspace::default().into(); - let py_ser = serde_json::to_string(&py_wrapped).unwrap(); - let rs_wrapped = DecoderWrapper::Metaspace(Metaspace::default()); - let rs_ser = serde_json::to_string(&rs_wrapped).unwrap(); - assert_eq!(py_ser, rs_ser); - let py_dec: PyDecoder = serde_json::from_str(&rs_ser).unwrap(); - match py_dec.decoder { - PyDecoderWrapper::Wrapped(msp) => match *msp.as_ref().read().unwrap() { - DecoderWrapper::Metaspace(_) => {} - _ => panic!("Expected Metaspace"), - }, - _ => panic!("Expected wrapped, not custom."), - } - - let obj = Python::attach(|py| { - let py_msp = PyDecoder::new(Metaspace::default().into()); - Py::new(py, py_msp).unwrap().into_any() - }); - let py_seq = PyDecoderWrapper::Custom(Arc::new(RwLock::new(CustomDecoder::new(obj)))); - assert!(serde_json::to_string(&py_seq).is_err()); - } -} diff --git a/bindings/python-pipeline/src/detached_lock.rs b/bindings/python/src/detached_lock.rs similarity index 100% rename from bindings/python-pipeline/src/detached_lock.rs rename to bindings/python/src/detached_lock.rs diff --git a/bindings/python/src/encoding.rs b/bindings/python/src/encoding.rs deleted file mode 100644 index f0ab9a3e8..000000000 --- a/bindings/python/src/encoding.rs +++ /dev/null @@ -1,473 +0,0 @@ -use pyo3::exceptions; -use pyo3::prelude::*; -use pyo3::types::*; -use tk::tokenizer::{Offsets, PaddingDirection}; -use tk::utils::truncation::TruncationDirection; -use tokenizers as tk; - -use crate::error::{PyError, deprecation_warning}; - -/// The :class:`~tokenizers.Encoding` represents the output of a :class:`~tokenizers.Tokenizer`. -/// -/// It holds all the information about the tokenized input, including the token IDs, -/// token strings, attention masks, offsets, and more. This is the main data structure -/// returned by :meth:`~tokenizers.Tokenizer.encode` and -/// :meth:`~tokenizers.Tokenizer.encode_batch`. -/// -/// Example:: -/// -/// >>> from tokenizers import Tokenizer -/// >>> tokenizer = Tokenizer.from_pretrained("bert-base-uncased") -/// >>> encoding = tokenizer.encode("Hello, world!") -/// >>> encoding.ids -/// [101, 7592, 1010, 2088, 999, 102] -/// >>> encoding.tokens -/// ['[CLS]', 'hello', ',', 'world', '!', '[SEP]'] -/// >>> encoding.offsets -/// [(0, 0), (0, 5), (5, 6), (7, 12), (12, 13), (0, 0)] -/// -#[pyclass(dict, module = "tokenizers", name = "Encoding")] -#[repr(transparent)] -pub struct PyEncoding { - pub encoding: tk::tokenizer::Encoding, -} - -impl From for PyEncoding { - fn from(v: tk::tokenizer::Encoding) -> Self { - Self { encoding: v } - } -} - -#[pymethods] -impl PyEncoding { - #[new] - #[pyo3(signature = (), text_signature = "(self)")] - fn new() -> Self { - Self { - encoding: tk::tokenizer::Encoding::default(), - } - } - - fn __getstate__(&self, py: Python) -> PyResult> { - let data = serde_json::to_string(&self.encoding).map_err(|e| { - exceptions::PyException::new_err(format!( - "Error while attempting to pickle Encoding: {e}" - )) - })?; - Ok(PyBytes::new(py, data.as_bytes()).into()) - } - - fn __setstate__(&mut self, py: Python, state: Py) -> PyResult<()> { - match state.extract::<&[u8]>(py) { - Ok(s) => { - self.encoding = serde_json::from_slice(s).map_err(|e| { - exceptions::PyException::new_err(format!( - "Error while attempting to unpickle Encoding: {e}" - )) - })?; - Ok(()) - } - Err(e) => Err(e.into()), - } - } - - fn __repr__(&self) -> PyResult { - Ok(format!( - "Encoding(num_tokens={}, attributes=[ids, type_ids, tokens, offsets, \ - attention_mask, special_tokens_mask, overflowing])", - self.encoding.get_ids().len() - )) - } - - fn __len__(&self) -> PyResult { - Ok(self.encoding.len()) - } - - /// Merge the list of encodings into one final :class:`~tokenizers.Encoding` - /// - /// Args: - /// encodings (A :obj:`List` of :class:`~tokenizers.Encoding`): - /// The list of encodings that should be merged in one - /// - /// growing_offsets (:obj:`bool`, defaults to :obj:`True`): - /// Whether the offsets should accumulate while merging - /// - /// Returns: - /// :class:`~tokenizers.Encoding`: The resulting Encoding - #[staticmethod] - #[pyo3(signature = (encodings, growing_offsets = true) -> "Encoding")] - #[pyo3(text_signature = "(encodings, growing_offsets=True)")] - fn merge(encodings: Vec>, growing_offsets: bool) -> PyEncoding { - tk::tokenizer::Encoding::merge( - encodings.into_iter().map(|e| e.encoding.clone()), - growing_offsets, - ) - .into() - } - - /// The number of sequences represented - /// - /// Returns: - /// :obj:`int`: The number of sequences in this :class:`~tokenizers.Encoding` - #[getter] - fn get_n_sequences(&self) -> usize { - self.encoding.n_sequences() - } - - /// Set the given sequence index - /// - /// Set the given sequence index for the whole range of tokens contained in this - /// :class:`~tokenizers.Encoding`. - #[pyo3(text_signature = "(self, sequence_id)")] - fn set_sequence_id(&mut self, sequence_id: usize) { - self.encoding.set_sequence_id(sequence_id); - } - - /// The generated IDs - /// - /// The IDs are the main input to a Language Model. They are the token indices, - /// the numerical representations that a LM understands. - /// - /// Returns: - /// :obj:`List[int]`: The list of IDs - #[getter] - fn get_ids(&self) -> Vec { - self.encoding.get_ids().to_vec() - } - - /// The generated tokens - /// - /// They are the string representation of the IDs. - /// - /// Returns: - /// :obj:`List[str]`: The list of tokens - #[getter] - fn get_tokens(&self) -> Vec { - self.encoding.get_tokens().to_vec() - } - - /// The generated word indices. - /// - /// .. warning:: - /// This is deprecated and will be removed in a future version. - /// Please use :obj:`~tokenizers.Encoding.word_ids` instead. - /// - /// They represent the index of the word associated to each token. - /// When the input is pre-tokenized, they correspond to the ID of the given input label, - /// otherwise they correspond to the words indices as defined by the - /// :class:`~tokenizers.pre_tokenizers.PreTokenizer` that was used. - /// - /// For special tokens and such (any token that was generated from something that was - /// not part of the input), the output is :obj:`None` - /// - /// Returns: - /// A :obj:`List` of :obj:`Optional[int]`: A list of optional word index. - #[getter] - fn get_words(&self, py: Python<'_>) -> PyResult>> { - deprecation_warning( - py, - "0.9.4", - "Encoding.words is deprecated, please use Encoding.word_ids instead.", - )?; - Ok(self.get_word_ids()) - } - - /// The generated word indices. - /// - /// They represent the index of the word associated to each token. - /// When the input is pre-tokenized, they correspond to the ID of the given input label, - /// otherwise they correspond to the words indices as defined by the - /// :class:`~tokenizers.pre_tokenizers.PreTokenizer` that was used. - /// - /// For special tokens and such (any token that was generated from something that was - /// not part of the input), the output is :obj:`None` - /// - /// Returns: - /// A :obj:`List` of :obj:`Optional[int]`: A list of optional word index. - #[getter] - fn get_word_ids(&self) -> Vec> { - self.encoding.get_word_ids().to_vec() - } - - /// The generated sequence indices. - /// - /// They represent the index of the input sequence associated to each token. - /// The sequence id can be None if the token is not related to any input sequence, - /// like for example with special tokens. - /// - /// Returns: - /// A :obj:`List` of :obj:`Optional[int]`: A list of optional sequence index. - #[getter] - fn get_sequence_ids(&self) -> Vec> { - self.encoding.get_sequence_ids() - } - - /// The generated type IDs - /// - /// Generally used for tasks like sequence classification or question answering, - /// these tokens let the LM know which input sequence corresponds to each tokens. - /// - /// Returns: - /// :obj:`List[int]`: The list of type ids - #[getter] - fn get_type_ids(&self) -> Vec { - self.encoding.get_type_ids().to_vec() - } - - /// The offsets associated to each token - /// - /// These offsets let's you slice the input string, and thus retrieve the original - /// part that led to producing the corresponding token. - /// - /// Returns: - /// A :obj:`List` of :obj:`Tuple[int, int]`: The list of offsets - #[getter] - fn get_offsets(&self) -> Vec<(usize, usize)> { - self.encoding.get_offsets().to_vec() - } - - /// The special token mask - /// - /// This indicates which tokens are special tokens, and which are not. - /// - /// Returns: - /// :obj:`List[int]`: The special tokens mask - #[getter] - fn get_special_tokens_mask(&self) -> Vec { - self.encoding.get_special_tokens_mask().to_vec() - } - - /// The attention mask - /// - /// This indicates to the LM which tokens should be attended to, and which should not. - /// This is especially important when batching sequences, where we need to applying - /// padding. - /// - /// Returns: - /// :obj:`List[int]`: The attention mask - #[getter] - fn get_attention_mask(&self) -> Vec { - self.encoding.get_attention_mask().to_vec() - } - - /// A :obj:`List` of overflowing :class:`~tokenizers.Encoding` - /// - /// When using truncation, the :class:`~tokenizers.Tokenizer` takes care of splitting - /// the output into as many pieces as required to match the specified maximum length. - /// This field lets you retrieve all the subsequent pieces. - /// - /// When you use pairs of sequences, the overflowing pieces will contain enough - /// variations to cover all the possible combinations, while respecting the provided - /// maximum length. - #[getter] - fn get_overflowing(&self) -> Vec { - self.encoding - .get_overflowing() - .clone() - .into_iter() - .map(|e| e.into()) - .collect() - } - - /// Get the encoded tokens corresponding to the word at the given index - /// in one of the input sequences. - /// - /// Args: - /// word_index (:obj:`int`): - /// The index of a word in one of the input sequences. - /// sequence_index (:obj:`int`, defaults to :obj:`0`): - /// The index of the sequence that contains the target word - /// - /// Returns: - /// :obj:`Tuple[int, int]`: The range of tokens: :obj:`(first, last + 1)` - #[pyo3(signature = (word_index, sequence_index = 0))] - #[pyo3(text_signature = "(self, word_index, sequence_index=0)")] - fn word_to_tokens(&self, word_index: u32, sequence_index: usize) -> Option<(usize, usize)> { - self.encoding.word_to_tokens(word_index, sequence_index) - } - - /// Get the offsets of the word at the given index in one of the input sequences. - /// - /// Args: - /// word_index (:obj:`int`): - /// The index of a word in one of the input sequences. - /// sequence_index (:obj:`int`, defaults to :obj:`0`): - /// The index of the sequence that contains the target word - /// - /// Returns: - /// :obj:`Tuple[int, int]`: The range of characters (span) :obj:`(first, last + 1)` - #[pyo3(signature = (word_index, sequence_index = 0))] - #[pyo3(text_signature = "(self, word_index, sequence_index=0)")] - fn word_to_chars(&self, word_index: u32, sequence_index: usize) -> Option { - self.encoding.word_to_chars(word_index, sequence_index) - } - - /// Get the index of the sequence represented by the given token. - /// - /// In the general use case, this method returns :obj:`0` for a single sequence or - /// the first sequence of a pair, and :obj:`1` for the second sequence of a pair - /// - /// Args: - /// token_index (:obj:`int`): - /// The index of a token in the encoded sequence. - /// - /// Returns: - /// :obj:`int`: The sequence id of the given token - #[pyo3(text_signature = "(self, token_index)")] - fn token_to_sequence(&self, token_index: usize) -> Option { - self.encoding.token_to_sequence(token_index) - } - - /// Get the offsets of the token at the given index. - /// - /// The returned offsets are related to the input sequence that contains the - /// token. In order to determine in which input sequence it belongs, you - /// must call :meth:`~tokenizers.Encoding.token_to_sequence()`. - /// - /// Args: - /// token_index (:obj:`int`): - /// The index of a token in the encoded sequence. - /// - /// Returns: - /// :obj:`Tuple[int, int]`: The token offsets :obj:`(first, last + 1)` - #[pyo3(text_signature = "(self, token_index)")] - fn token_to_chars(&self, token_index: usize) -> Option { - let (_, offsets) = self.encoding.token_to_chars(token_index)?; - Some(offsets) - } - - /// Get the index of the word that contains the token in one of the input sequences. - /// - /// The returned word index is related to the input sequence that contains - /// the token. In order to determine in which input sequence it belongs, you - /// must call :meth:`~tokenizers.Encoding.token_to_sequence()`. - /// - /// Args: - /// token_index (:obj:`int`): - /// The index of a token in the encoded sequence. - /// - /// Returns: - /// :obj:`int`: The index of the word in the relevant input sequence. - #[pyo3(text_signature = "(self, token_index)")] - fn token_to_word(&self, token_index: usize) -> Option { - let (_, word_idx) = self.encoding.token_to_word(token_index)?; - Some(word_idx) - } - - /// Get the token that contains the char at the given position in the input sequence. - /// - /// Args: - /// char_pos (:obj:`int`): - /// The position of a char in the input string - /// sequence_index (:obj:`int`, defaults to :obj:`0`): - /// The index of the sequence that contains the target char - /// - /// Returns: - /// :obj:`int`: The index of the token that contains this char in the encoded sequence - #[pyo3(signature = (char_pos, sequence_index = 0))] - #[pyo3(text_signature = "(self, char_pos, sequence_index=0)")] - fn char_to_token(&self, char_pos: usize, sequence_index: usize) -> Option { - self.encoding.char_to_token(char_pos, sequence_index) - } - - /// Get the word that contains the char at the given position in the input sequence. - /// - /// Args: - /// char_pos (:obj:`int`): - /// The position of a char in the input string - /// sequence_index (:obj:`int`, defaults to :obj:`0`): - /// The index of the sequence that contains the target char - /// - /// Returns: - /// :obj:`int`: The index of the word that contains this char in the input sequence - #[pyo3(signature = (char_pos, sequence_index = 0))] - #[pyo3(text_signature = "(self, char_pos, sequence_index=0)")] - fn char_to_word(&self, char_pos: usize, sequence_index: usize) -> Option { - self.encoding.char_to_word(char_pos, sequence_index) - } - - /// Pad the :class:`~tokenizers.Encoding` at the given length - /// - /// Args: - /// length (:obj:`int`): - /// The desired length - /// - /// direction: (:obj:`str`, defaults to :obj:`right`): - /// The expected padding direction. Can be either :obj:`right` or :obj:`left` - /// - /// pad_id (:obj:`int`, defaults to :obj:`0`): - /// The ID corresponding to the padding token - /// - /// pad_type_id (:obj:`int`, defaults to :obj:`0`): - /// The type ID corresponding to the padding token - /// - /// pad_token (:obj:`str`, defaults to `[PAD]`): - /// The pad token to use - #[pyo3(signature = (length, **kwargs) -> "None")] - #[pyo3( - text_signature = "(self, length, direction='right', pad_id=0, pad_type_id=0, pad_token='[PAD]')" - )] - fn pad(&mut self, length: usize, kwargs: Option<&Bound<'_, PyDict>>) -> PyResult<()> { - let mut pad_id = 0; - let mut pad_type_id = 0; - let mut pad_token = "[PAD]".to_string(); - let mut direction = PaddingDirection::Right; - - if let Some(kwargs) = kwargs { - for (key, value) in kwargs { - let key: String = key.extract()?; - match key.as_ref() { - "direction" => { - let value: String = value.extract()?; - direction = match value.as_ref() { - "left" => Ok(PaddingDirection::Left), - "right" => Ok(PaddingDirection::Right), - other => Err(PyError(format!( - "Unknown `direction`: `{other}`. Use \ - one of `left` or `right`" - )) - .into_pyerr::()), - }?; - } - "pad_id" => pad_id = value.extract()?, - "pad_type_id" => pad_type_id = value.extract()?, - "pad_token" => pad_token = value.extract()?, - _ => println!("Ignored unknown kwarg option {key}"), - } - } - } - self.encoding - .pad(length, pad_id, pad_type_id, &pad_token, direction); - Ok(()) - } - - /// Truncate the :class:`~tokenizers.Encoding` at the given length - /// - /// If this :class:`~tokenizers.Encoding` represents multiple sequences, when truncating - /// this information is lost. It will be considered as representing a single sequence. - /// - /// Args: - /// max_length (:obj:`int`): - /// The desired length - /// - /// stride (:obj:`int`, defaults to :obj:`0`): - /// The length of previous content to be included in each overflowing piece - /// - /// direction (:obj:`str`, defaults to :obj:`right`): - /// Truncate direction - #[pyo3(signature = (max_length, stride = 0, direction = "right") -> "None")] - #[pyo3(text_signature = "(self, max_length, stride=0, direction='right')")] - fn truncate(&mut self, max_length: usize, stride: usize, direction: &str) -> PyResult<()> { - let tdir = match direction { - "left" => Ok(TruncationDirection::Left), - "right" => Ok(TruncationDirection::Right), - _ => Err( - PyError(format!("Invalid truncation direction value : {direction}")) - .into_pyerr::(), - ), - }?; - - self.encoding.truncate(max_length, stride, tdir); - Ok(()) - } -} diff --git a/bindings/python/src/error.rs b/bindings/python/src/error.rs index bd9f5b455..314d7e9e0 100644 --- a/bindings/python/src/error.rs +++ b/bindings/python/src/error.rs @@ -1,42 +1,9 @@ -use pyo3::exceptions; -use pyo3::prelude::*; -use pyo3::type_object::PyTypeInfo; -use std::ffi::CString; -use std::fmt::{Display, Formatter, Result as FmtResult}; -use tokenizers::tokenizer::Result; +use pyo3::PyErr; +use pyo3::create_exception; +use pyo3::exceptions::PyException; -#[derive(Debug)] -pub struct PyError(pub String); -impl PyError { - #[allow(dead_code)] - pub fn from(s: &str) -> Self { - PyError(String::from(s)) - } - pub fn into_pyerr(self) -> PyErr { - PyErr::new::(format!("{self}")) - } -} -impl Display for PyError { - fn fmt(&self, fmt: &mut Formatter) -> FmtResult { - write!(fmt, "{}", self.0) - } -} -impl std::error::Error for PyError {} - -pub struct ToPyResult(pub Result); -impl From> for PyResult { - fn from(v: ToPyResult) -> Self { - v.0.map_err(|e| exceptions::PyException::new_err(format!("{e}"))) - } -} -impl ToPyResult { - pub fn into_py(self) -> PyResult { - self.into() - } -} +create_exception!(tokenizers, TokenizersError, PyException); -pub(crate) fn deprecation_warning(py: Python<'_>, version: &str, message: &str) -> PyResult<()> { - let deprecation_warning = py.import("builtins")?.getattr("DeprecationWarning")?; - let full_message = format!("Deprecated in {version}: {message}"); - pyo3::PyErr::warn(py, &deprecation_warning, &CString::new(full_message)?, 0) +pub fn to_pyerr(e: tk_encode::Error) -> PyErr { + TokenizersError::new_err(e.to_string()) } diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs index b670e95c2..6bcc17ebe 100644 --- a/bindings/python/src/lib.rs +++ b/bindings/python/src/lib.rs @@ -1,73 +1,46 @@ #![warn(clippy::all)] -#![allow(clippy::upper_case_acronyms)] -// Many false positives with pyo3 it seems &str, and &PyAny get flagged -#![allow(clippy::borrow_deref_ref)] -extern crate tokenizers as tk; - -use once_cell::sync::Lazy; -use std::sync::Arc; -use tokio::runtime::Runtime; - -// We create a global runtime that will be initialized once when first needed -// This ensures we always have a runtime available for tokio::task::spawn_blocking -static TOKIO_RUNTIME: Lazy> = Lazy::new(|| { - let rt = tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build() - .expect("Failed to create global Tokio runtime"); - Arc::new(rt) -}); -mod decoders; -mod encoding; -mod error; -mod models; -mod normalizers; -mod pre_tokenizers; -mod processors; -mod token; -mod tokenizer; -mod trainers; -mod utils; +pub mod added_token; +pub mod detached_lock; +pub mod error; +pub mod models; +pub mod normalizers; +pub mod pre_tokenizers; +pub mod tokenizer; +pub mod trainers; use pyo3::prelude::*; -pub const VERSION: &str = env!("CARGO_PKG_VERSION"); -// For users using multiprocessing in python, it is quite easy to fork the process running -// tokenizers, ending up with a deadlock because we internally make use of multithreading. So -// we register a callback to be called in the event of a fork to disable parallelism. -#[cfg(target_family = "unix")] -static mut REGISTERED_FORK_CALLBACK: bool = false; +/// Components repr as their tokenizer.json serialization: compact, and always +/// in sync with what `Tokenizer.save` writes. +pub fn component_repr(component: &T) -> String { + serde_json::to_string(component).unwrap_or_else(|_| "".to_owned()) +} + +// Forked children of a process that used our rayon threads would inherit a +// poisoned thread pool; disable parallelism there unless the user configured +// it explicitly (same behavior as the v1 bindings). #[cfg(target_family = "unix")] extern "C" fn child_after_fork() { - use tk::parallelism::*; - if has_parallelism_been_used() && !is_parallelism_configured() { + use std::sync::atomic::Ordering; + use tk_encode::utils::parallelism::{is_parallelism_configured, set_parallelism}; + if crate::tokenizer::USED_PARALLELISM.load(Ordering::SeqCst) && !is_parallelism_configured() { set_parallelism(false); } } -/// Tokenizers Module +/// Fast tokenizers built on the pipeline encode path. Start with `Tokenizer`. #[pymodule(gil_used = false)] -pub mod tokenizers { +pub mod _native { use super::*; #[pymodule_export] - pub use super::encoding::PyEncoding; - #[pymodule_export] - pub use super::token::PyToken; + pub use super::added_token::PyAddedToken; #[pymodule_export] - pub use super::tokenizer::PyAddedToken; + pub use super::error::TokenizersError; #[pymodule_export] pub use super::tokenizer::PyTokenizer; - #[pymodule_export] - pub use super::utils::PyNormalizedString; - #[pymodule_export] - pub use super::utils::PyPreTokenizedString; - #[pymodule_export] - pub use super::utils::PyRegex; - #[pymodule_export] - pub use super::decoders::decoders; #[pymodule_export] pub use super::models::models; #[pymodule_export] @@ -75,8 +48,6 @@ pub mod tokenizers { #[pymodule_export] pub use super::pre_tokenizers::pre_tokenizers; #[pymodule_export] - pub use super::processors::processors; - #[pymodule_export] pub use super::trainers::trainers; #[allow(non_upper_case_globals)] @@ -85,17 +56,14 @@ pub mod tokenizers { #[pymodule_init] fn init(_m: &Bound<'_, PyModule>) -> PyResult<()> { - let _ = env_logger::try_init_from_env("TOKENIZERS_LOG"); - - // Register the fork callback #[cfg(target_family = "unix")] - unsafe { - if !REGISTERED_FORK_CALLBACK { + { + use std::sync::Once; + static REGISTER_FORK_CALLBACK: Once = Once::new(); + REGISTER_FORK_CALLBACK.call_once(|| unsafe { libc::pthread_atfork(None, None, Some(child_after_fork)); - REGISTERED_FORK_CALLBACK = true; - } + }); } - Ok(()) } } diff --git a/bindings/python/src/models.rs b/bindings/python/src/models.rs index d6c900c74..578a78123 100644 --- a/bindings/python/src/models.rs +++ b/bindings/python/src/models.rs @@ -1,1075 +1,157 @@ -use std::collections::HashMap; -use std::path::{Path, PathBuf}; -use std::sync::{Arc, RwLock}; - -use crate::token::PyToken; -use crate::trainers::PyTrainer; -use ahash::AHashMap; -use pyo3::exceptions; use pyo3::prelude::*; -use pyo3::types::*; -use serde::{Deserialize, Serialize}; -use tk::models::ModelWrapper; -use tk::models::bpe::{BPE, BpeBuilder, Merges}; -use tk::models::unigram::Unigram; -use tk::models::wordlevel::WordLevel; -use tk::models::wordpiece::{WordPiece, WordPieceBuilder}; -use tk::tokenizer::PreTokenizedString; -use tk::{Model, Token, Trainable}; -use tokenizers as tk; +use tk_encode::models::ModelWrapper; +use tk_encode::models::bpe::BPE; +use tk_encode::models::unigram::Unigram; +use tk_encode::models::wordlevel::WordLevel; +use tk_encode::models::wordpiece::WordPiece; -use super::error::{ToPyResult, deprecation_warning}; +use crate::error::to_pyerr; -/// Base class for all models -/// -/// The model represents the actual tokenization algorithm. This is the part that -/// will contain and manage the learned vocabulary. +/// Base class for all models. /// -/// This class cannot be constructed directly. Please use one of the concrete models. -#[pyclass(module = "tokenizers.models", name = "Model", subclass, from_py_object)] -#[derive(Clone, Serialize, Deserialize)] -#[serde(transparent)] +/// The model is the trained part of a tokenizer: it turns each pre-tokenized +/// piece into token ids using its vocabulary. Models are immutable values — +/// assigning one to a tokenizer copies it. +#[pyclass(frozen, subclass, name = "Model", module = "tokenizers.models")] pub struct PyModel { - pub model: Arc>, -} - -impl PyModel { - pub(crate) fn get_as_subtype(&self, py: Python<'_>) -> PyResult> { - let base = self.clone(); - Ok(match *self.model.as_ref().read().unwrap() { - ModelWrapper::BPE(_) => Py::new(py, (PyBPE {}, base))?.into_any(), - ModelWrapper::WordPiece(_) => Py::new(py, (PyWordPiece {}, base))?.into_any(), - ModelWrapper::WordLevel(_) => Py::new(py, (PyWordLevel {}, base))?.into_any(), - ModelWrapper::Unigram(_) => Py::new(py, (PyUnigram {}, base))?.into_any(), - }) - } -} - -impl Model for PyModel { - fn tokenize(&self, tokens: &str) -> tk::Result> { - self.model.read().unwrap().tokenize(tokens) - } - - /// See [`Model::tokenize_in_pretokenized`] for the lock-once rationale. - fn tokenize_in_pretokenized( - &self, - pretokenized: &mut PreTokenizedString, - truncation: Option<(usize, tk::TruncationDirection)>, - ) -> tk::Result<()> { - let guard = self.model.read().unwrap(); - match truncation { - Some((max_tokens, direction)) => pretokenized.tokenize_with_limit( - |normalized| guard.tokenize(normalized.get()), - max_tokens, - direction, - ), - None => pretokenized.tokenize(|normalized| guard.tokenize(normalized.get())), - } - } - - fn token_to_id(&self, token: &str) -> Option { - self.model.read().unwrap().token_to_id(token) - } - - fn id_to_token(&self, id: u32) -> Option { - self.model.read().unwrap().id_to_token(id) - } - - fn get_vocab(&self) -> HashMap { - self.model.read().unwrap().get_vocab() - } - - fn get_vocab_size(&self) -> usize { - self.model.read().unwrap().get_vocab_size() - } - - fn save(&self, folder: &Path, name: Option<&str>) -> tk::Result> { - self.model.read().unwrap().save(folder, name) - } -} - -impl Trainable for PyModel { - type Trainer = PyTrainer; - - fn get_trainer(&self) -> Self::Trainer { - self.model.read().unwrap().get_trainer().into() - } -} - -impl From for PyModel -where - I: Into, -{ - fn from(model: I) -> Self { - Self { - model: Arc::new(RwLock::new(model.into())), - } - } + pub inner: ModelWrapper, } #[pymethods] impl PyModel { - #[new] - #[pyo3(signature = () -> "Model", text_signature = "(self)")] - fn __new__() -> Self { - // Instantiate a default empty model. This doesn't really make sense, but we need - // to be able to instantiate an empty model for pickle capabilities. - PyModel { - model: Arc::new(RwLock::new(BPE::default().into())), - } - } - - fn __getstate__(&self, py: Python) -> PyResult> { - let data = serde_json::to_string(&self.model).map_err(|e| { - exceptions::PyException::new_err(format!("Error while attempting to pickle Model: {e}")) - })?; - Ok(PyBytes::new(py, data.as_bytes()).into()) - } - - fn __setstate__(&mut self, py: Python, state: Py) -> PyResult<()> { - match state.extract::<&[u8]>(py) { - Ok(s) => { - self.model = serde_json::from_slice(s).map_err(|e| { - exceptions::PyException::new_err(format!( - "Error while attempting to unpickle Model: {e}" - )) - })?; - Ok(()) - } - Err(e) => Err(e.into()), - } - } - - /// Tokenize a sequence - /// - /// Args: - /// sequence (:obj:`str`): - /// A sequence to tokenize - /// - /// Returns: - /// A :obj:`List` of :class:`~tokenizers.Token`: The generated tokens - #[pyo3(text_signature = "(self, sequence)")] - fn tokenize(&self, sequence: &str) -> PyResult> { - Ok(ToPyResult(self.model.read().unwrap().tokenize(sequence)) - .into_py()? - .into_iter() - .map(|t| t.into()) - .collect()) - } - - /// Get the ID associated to a token - /// - /// Args: - /// token (:obj:`str`): - /// A token to convert to an ID - /// - /// Returns: - /// :obj:`int`: The ID associated to the token - #[pyo3(text_signature = "(self, tokens)")] - fn token_to_id(&self, token: &str) -> Option { - self.model.read().unwrap().token_to_id(token) - } - - /// Get the token associated to an ID - /// - /// Args: - /// id (:obj:`int`): - /// An ID to convert to a token - /// - /// Returns: - /// :obj:`str`: The token associated to the ID - #[pyo3(text_signature = "(self, id)")] - fn id_to_token(&self, id: u32) -> Option { - self.model.read().unwrap().id_to_token(id) - } - - /// Save the current model - /// - /// Save the current model in the given folder, using the given prefix for the various - /// files that will get created. - /// Any file with the same name that already exists in this folder will be overwritten. - /// - /// Args: - /// folder (:obj:`str`): - /// The path to the target folder in which to save the various files - /// - /// prefix (:obj:`str`, `optional`): - /// An optional prefix, used to prefix each file name - /// - /// Returns: - /// :obj:`List[str]`: The list of saved files - #[pyo3(signature = (folder, prefix=None, name=None) -> "list[str]", text_signature = "(self, folder, prefix)")] - fn save<'a>( - &self, - py: Python<'_>, - folder: &str, - mut prefix: Option<&'a str>, - name: Option<&'a str>, - ) -> PyResult> { - if name.is_some() { - deprecation_warning( - py, - "0.10.0", - "Parameter `name` of Model.save has been renamed `prefix`", - )?; - if prefix.is_none() { - prefix = name; - } - } - - let saved: PyResult> = - ToPyResult(self.model.read().unwrap().save(Path::new(folder), prefix)).into(); - - Ok(saved? - .into_iter() - .map(|path| path.to_string_lossy().into_owned()) - .collect()) - } - - /// Get the associated :class:`~tokenizers.trainers.Trainer` - /// - /// Retrieve the :class:`~tokenizers.trainers.Trainer` associated to this - /// :class:`~tokenizers.models.Model`. - /// - /// Returns: - /// :class:`~tokenizers.trainers.Trainer`: The Trainer used to train this model - #[pyo3(text_signature = "(self)")] - fn get_trainer(&self, py: Python<'_>) -> PyResult> { - PyTrainer::from(self.model.read().unwrap().get_trainer()).get_as_subtype(py) - } - - fn __repr__(&self) -> PyResult { - crate::utils::serde_pyo3::repr(self) - .map_err(|e| exceptions::PyException::new_err(e.to_string())) - } - - fn __str__(&self) -> PyResult { - crate::utils::serde_pyo3::to_string(self) - .map_err(|e| exceptions::PyException::new_err(e.to_string())) + fn __repr__(&self) -> String { + crate::component_repr(&self.inner) } } -/// An implementation of the BPE (Byte-Pair Encoding) algorithm -/// -/// Args: -/// vocab (:obj:`Dict[str, int]`, `optional`): -/// A dictionary of string keys and their ids :obj:`{"am": 0,...}` -/// -/// merges (:obj:`List[Tuple[str, str]]`, `optional`): -/// A list of pairs of tokens (:obj:`Tuple[str, str]`) :obj:`[("a", "b"),...]` -/// -/// cache_capacity (:obj:`int`, `optional`): -/// The number of words that the BPE cache can contain. The cache allows -/// to speed-up the process by keeping the result of the merge operations -/// for a number of words. -/// -/// dropout (:obj:`float`, `optional`): -/// A float between 0 and 1 that represents the BPE dropout to use. -/// -/// unk_token (:obj:`str`, `optional`): -/// The unknown token to be used by the model. -/// -/// continuing_subword_prefix (:obj:`str`, `optional`): -/// The prefix to attach to subword units that don't represent a beginning of word. -/// -/// end_of_word_suffix (:obj:`str`, `optional`): -/// The suffix to attach to subword units that represent an end of word. -/// -/// fuse_unk (:obj:`bool`, `optional`): -/// Whether to fuse any subsequent unknown tokens into a single one -/// -/// byte_fallback (:obj:`bool`, `optional`): -/// Whether to use spm byte-fallback trick (defaults to False) -/// -/// ignore_merges (:obj:`bool`, `optional`): -/// Whether or not to match tokens with the vocab before using merges. -/// -/// Example:: -/// -/// >>> from tokenizers.models import BPE -/// >>> # Build an empty model (to be trained) -/// >>> model = BPE(unk_token="") -/// >>> # Load from vocabulary and merges files -/// >>> model = BPE.from_file("vocab.json", "merges.txt") -/// -#[pyclass(extends=PyModel, module = "tokenizers.models", name = "BPE")] -pub struct PyBPE {} - -impl PyBPE { - fn with_builder( - mut builder: BpeBuilder, - kwargs: Option<&Bound<'_, PyDict>>, - ) -> PyResult> { - if let Some(kwargs) = kwargs { - for (key, value) in kwargs { - let key: String = key.extract()?; - match key.as_ref() { - "cache_capacity" => builder = builder.cache_capacity(value.extract()?), - "dropout" => { - if let Some(dropout) = value.extract()? { - builder = builder.dropout(dropout); - } - } - "unk_token" => { - if let Some(unk) = value.extract()? { - builder = builder.unk_token(unk); - } - } - "continuing_subword_prefix" => { - builder = builder.continuing_subword_prefix(value.extract()?) - } - "end_of_word_suffix" => builder = builder.end_of_word_suffix(value.extract()?), - "fuse_unk" => builder = builder.fuse_unk(value.extract()?), - "byte_fallback" => builder = builder.byte_fallback(value.extract()?), - "ignore_merges" => builder = builder.ignore_merges(value.extract()?), - _ => println!("Ignored unknown kwarg option {key}"), - }; - } - } - - match builder.build() { - Err(e) => Err(exceptions::PyException::new_err(format!( - "Error while initializing BPE: {e}" - ))), - Ok(bpe) => { - Ok(PyClassInitializer::::from(PyModel::from(bpe)).add_subclass(PyBPE {})) - } - } - } -} - -macro_rules! getter { - ($self: ident, $variant: ident, $($name: tt)+) => {{ - let super_ = $self.as_ref(); - let model = super_.model.read().unwrap(); - if let ModelWrapper::$variant(ref mo) = *model { - mo.$($name)+ - } else { - unreachable!() - } - }}; -} - -macro_rules! setter { - ($self: ident, $variant: ident, $name: ident, $value: expr) => {{ - let super_ = $self.as_ref(); - let mut model = super_.model.write().unwrap(); - if let ModelWrapper::$variant(ref mut mo) = *model { - mo.$name = $value; - } - }}; -} - -#[derive(FromPyObject)] -enum PyVocab { - Vocab(HashMap), - Filename(String), +pub fn wrap_model(py: Python<'_>, inner: ModelWrapper) -> PyResult> { + let base = PyModel { + inner: inner.clone(), + }; + let init = PyClassInitializer::from(base); + let obj = match inner { + ModelWrapper::BPE(_) => Bound::new(py, init.add_subclass(PyBPE))?.into_super(), + ModelWrapper::WordPiece(_) => Bound::new(py, init.add_subclass(PyWordPiece))?.into_super(), + ModelWrapper::WordLevel(_) => Bound::new(py, init.add_subclass(PyWordLevel))?.into_super(), + ModelWrapper::Unigram(_) => Bound::new(py, init.add_subclass(PyUnigram))?.into_super(), + }; + Ok(obj.unbind()) } -#[derive(FromPyObject)] -enum PyMerges { - Merges(Merges), - Filename(String), -} +/// Byte-Pair Encoding: builds tokens by applying the merges learned during +/// training. `unk_token` stands in for characters the vocabulary cannot +/// represent; `byte_fallback` encodes them as raw bytes instead. `dropout` +/// randomly skips merges (a training-time regularization). `ignore_merges` +/// looks whole pieces up in the vocabulary before merging. +#[pyclass(frozen, extends = PyModel, name = "BPE", module = "tokenizers.models")] +pub struct PyBPE; #[pymethods] impl PyBPE { - #[getter] - fn get_dropout(self_: PyRef) -> Option { - getter!(self_, BPE, dropout) - } - - #[setter] - fn set_dropout(self_: PyRef, dropout: Option) { - setter!(self_, BPE, dropout, dropout); - } - - #[getter] - fn get_unk_token(self_: PyRef) -> Option { - getter!(self_, BPE, unk_token.clone()) - } - - #[setter] - fn set_unk_token(self_: PyRef, unk_token: Option) { - setter!(self_, BPE, unk_token, unk_token); - } - - #[getter] - fn get_continuing_subword_prefix(self_: PyRef) -> Option { - getter!(self_, BPE, continuing_subword_prefix.clone()) - } - - #[setter] - fn set_continuing_subword_prefix( - self_: PyRef, - continuing_subword_prefix: Option, - ) { - setter!( - self_, - BPE, - continuing_subword_prefix, - continuing_subword_prefix - ); - } - - #[getter] - fn get_end_of_word_suffix(self_: PyRef) -> Option { - getter!(self_, BPE, end_of_word_suffix.clone()) - } - - #[setter] - fn set_end_of_word_suffix(self_: PyRef, end_of_word_suffix: Option) { - setter!(self_, BPE, end_of_word_suffix, end_of_word_suffix); - } - - #[getter] - fn get_fuse_unk(self_: PyRef) -> bool { - getter!(self_, BPE, fuse_unk) - } - - #[setter] - fn set_fuse_unk(self_: PyRef, fuse_unk: bool) { - setter!(self_, BPE, fuse_unk, fuse_unk); - } - - #[getter] - fn get_byte_fallback(self_: PyRef) -> bool { - getter!(self_, BPE, byte_fallback) - } - - #[setter] - fn set_byte_fallback(self_: PyRef, byte_fallback: bool) { - setter!(self_, BPE, byte_fallback, byte_fallback); - } - #[getter] - fn get_ignore_merges(self_: PyRef) -> bool { - getter!(self_, BPE, ignore_merges) - } - - #[setter] - fn set_ignore_merges(self_: PyRef, ignore_merges: bool) { - setter!(self_, BPE, ignore_merges, ignore_merges); - } #[new] - #[pyo3( - signature = (vocab=None, merges=None, **kwargs), - text_signature = "(self, vocab=None, merges=None, cache_capacity=None, dropout=None, unk_token=None, continuing_subword_prefix=None, end_of_word_suffix=None, fuse_unk=None, byte_fallback=False, ignore_merges=False)")] + #[pyo3(signature = (*, unk_token = None, dropout = None, fuse_unk = false, byte_fallback = false, ignore_merges = false))] fn new( - _py: Python<'_>, - vocab: Option, - merges: Option, - kwargs: Option<&Bound<'_, PyDict>>, + unk_token: Option, + dropout: Option, + fuse_unk: bool, + byte_fallback: bool, + ignore_merges: bool, ) -> PyResult> { - if (vocab.is_some() && merges.is_none()) || (vocab.is_none() && merges.is_some()) { - return Err(exceptions::PyValueError::new_err( - "`vocab` and `merges` must be both specified", - )); + let mut builder = BPE::builder() + .fuse_unk(fuse_unk) + .byte_fallback(byte_fallback) + .ignore_merges(ignore_merges); + if let Some(unk) = unk_token { + builder = builder.unk_token(unk); } - - let mut builder = BPE::builder(); - if let (Some(vocab), Some(merges)) = (vocab, merges) { - match (vocab, merges) { - (PyVocab::Vocab(vocab), PyMerges::Merges(merges)) => { - let vocab: AHashMap<_, _> = vocab.into_iter().collect(); - builder = builder.vocab_and_merges(vocab, merges); - } - (PyVocab::Filename(vocab_filename), PyMerges::Filename(merges_filename)) => { - builder = - builder.files(vocab_filename.to_string(), merges_filename.to_string()); - } - _ => { - return Err(exceptions::PyValueError::new_err( - "`vocab` and `merges` must be both be from memory or both filenames", - )); - } - } + if let Some(d) = dropout { + builder = builder.dropout(d); } - - PyBPE::with_builder(builder, kwargs) + let bpe = builder.build().map_err(to_pyerr)?; + Ok(PyClassInitializer::from(PyModel { inner: bpe.into() }).add_subclass(PyBPE)) } - /// Read a :obj:`vocab.json` and a :obj:`merges.txt` files - /// - /// This method provides a way to read and parse the content of these files, - /// returning the relevant data structures. If you want to instantiate some BPE models - /// from memory, this method gives you the expected input from the standard files. - /// - /// Args: - /// vocab (:obj:`str`): - /// The path to a :obj:`vocab.json` file - /// - /// merges (:obj:`str`): - /// The path to a :obj:`merges.txt` file - /// - /// Returns: - /// A :obj:`Tuple` with the vocab and the merges: - /// The vocabulary and merges loaded into memory + /// Load a BPE from the legacy vocab.json + merges.txt format. #[staticmethod] - #[pyo3(text_signature = "(vocab, merges)")] - fn read_file(vocab: &str, merges: &str) -> PyResult<(HashMap, Merges)> { - let (vocab, merges) = BPE::read_file(vocab, merges).map_err(|e| { - exceptions::PyException::new_err(format!( - "Error while reading vocab & merges files: {e}" - )) - })?; - let vocab = vocab.into_iter().collect(); - Ok((vocab, merges)) - } - - /// Instantiate a BPE model from the given files. - /// - /// This method is roughly equivalent to doing:: - /// - /// vocab, merges = BPE.read_file(vocab_filename, merges_filename) - /// bpe = BPE(vocab, merges) - /// - /// If you don't need to keep the :obj:`vocab, merges` values lying around, - /// this method is more optimized than manually calling - /// :meth:`~tokenizers.models.BPE.read_file` to initialize a :class:`~tokenizers.models.BPE` - /// - /// Args: - /// vocab (:obj:`str`): - /// The path to a :obj:`vocab.json` file - /// - /// merges (:obj:`str`): - /// The path to a :obj:`merges.txt` file - /// - /// Returns: - /// :class:`~tokenizers.models.BPE`: An instance of BPE loaded from these files - #[classmethod] - #[pyo3(signature = (vocab, merges, **kwargs) -> "BPE")] - #[pyo3(text_signature = "(vocab, merges, **kwargs)")] + #[pyo3(signature = (vocab, merges, *, unk_token = None) -> "BPE")] fn from_file( - _cls: &Bound<'_, PyType>, - py: Python, + py: Python<'_>, vocab: &str, merges: &str, - kwargs: Option<&Bound<'_, PyDict>>, - ) -> PyResult> { - let (vocab, merges) = BPE::read_file(vocab, merges).map_err(|e| { - exceptions::PyException::new_err(format!("Error while reading BPE files: {e}")) - })?; - let vocab = vocab.into_iter().collect(); - Py::new( - py, - PyBPE::new( - py, - Some(PyVocab::Vocab(vocab)), - Some(PyMerges::Merges(merges)), - kwargs, - )?, - ) - } - - /// Clears the internal cache - #[pyo3(signature = () -> "None")] - #[pyo3(text_signature = "(self)")] - fn _clear_cache(self_: PyRef) -> PyResult<()> { - let super_ = self_.as_ref(); - let mut model = super_.model.write().map_err(|e| { - exceptions::PyException::new_err(format!("Error while clearing BPE cache: {e}")) - })?; - model.clear_cache(); - Ok(()) - } - - /// Resize the internal cache - #[pyo3(signature = (capacity) -> "None")] - #[pyo3(text_signature = "(self, capacity)")] - fn _resize_cache(self_: PyRef, capacity: usize) -> PyResult<()> { - let super_ = self_.as_ref(); - let mut model = super_.model.write().map_err(|e| { - exceptions::PyException::new_err(format!("Error while resizing BPE cache: {e}")) - })?; - model.resize_cache(capacity); - Ok(()) - } -} - -/// An implementation of the WordPiece algorithm -/// -/// Args: -/// vocab (:obj:`Dict[str, int]`, `optional`): -/// A dictionary of string keys and their ids :obj:`{"am": 0,...}` -/// -/// unk_token (:obj:`str`, `optional`): -/// The unknown token to be used by the model. -/// -/// max_input_chars_per_word (:obj:`int`, `optional`): -/// The maximum number of characters to authorize in a single word. -/// -/// Example:: -/// -/// >>> from tokenizers.models import WordPiece -/// >>> # Build an empty model (to be trained) -/// >>> model = WordPiece(unk_token="[UNK]") -/// >>> # Load from a vocabulary file -/// >>> model = WordPiece.from_file("vocab.txt") -/// -#[pyclass(extends=PyModel, module = "tokenizers.models", name = "WordPiece")] -pub struct PyWordPiece {} - -impl PyWordPiece { - fn with_builder( - mut builder: WordPieceBuilder, - kwargs: Option<&Bound<'_, PyDict>>, - ) -> PyResult> { - if let Some(kwargs) = kwargs { - for (key, val) in kwargs { - let key: String = key.extract()?; - match key.as_ref() { - "unk_token" => { - builder = builder.unk_token(val.extract()?); - } - "max_input_chars_per_word" => { - builder = builder.max_input_chars_per_word(val.extract()?); - } - "continuing_subword_prefix" => { - builder = builder.continuing_subword_prefix(val.extract()?); - } - _ => println!("Ignored unknown kwargs option {key}"), - } - } - } - - match builder.build() { - Err(e) => Err(exceptions::PyException::new_err(format!( - "Error while initializing WordPiece: {e}" - ))), - Ok(wordpiece) => Ok( - PyClassInitializer::::from(PyModel::from(wordpiece)) - .add_subclass(PyWordPiece {}), - ), + unk_token: Option, + ) -> PyResult> { + let mut builder = BPE::from_file(vocab, merges); + if let Some(unk) = unk_token { + builder = builder.unk_token(unk); } + let bpe = py.detach(|| builder.build()).map_err(to_pyerr)?; + wrap_model(py, bpe.into()) } } +/// The BERT model: greedily matches the longest vocabulary entry, marking +/// word continuations with a prefix ("##" by default). A piece longer than +/// `max_input_chars_per_word` becomes `unk_token` outright. +#[pyclass(frozen, extends = PyModel, name = "WordPiece", module = "tokenizers.models")] +pub struct PyWordPiece; + #[pymethods] impl PyWordPiece { - #[getter] - fn get_unk_token(self_: PyRef) -> String { - getter!(self_, WordPiece, unk_token.clone()) - } - - #[setter] - fn set_unk_token(self_: PyRef, unk_token: String) { - setter!(self_, WordPiece, unk_token, unk_token); - } - - #[getter] - fn get_continuing_subword_prefix(self_: PyRef) -> String { - getter!(self_, WordPiece, continuing_subword_prefix.clone()) - } - - #[setter] - fn set_continuing_subword_prefix(self_: PyRef, continuing_subword_prefix: String) { - setter!( - self_, - WordPiece, - continuing_subword_prefix, - continuing_subword_prefix - ); - } - - #[getter] - fn get_max_input_chars_per_word(self_: PyRef) -> usize { - getter!(self_, WordPiece, max_input_chars_per_word) - } - - #[setter] - fn set_max_input_chars_per_word(self_: PyRef, max: usize) { - setter!(self_, WordPiece, max_input_chars_per_word, max); - } - #[new] - #[pyo3( - signature = (vocab=None, **kwargs), - text_signature = "(self, vocab=None, unk_token='[UNK]', max_input_chars_per_word=100, continuing_subword_prefix='##')" - )] + #[pyo3(signature = (*, unk_token = String::from("[UNK]"), continuing_subword_prefix = String::from("##"), max_input_chars_per_word = 100))] fn new( - _py: Python<'_>, - vocab: Option, - kwargs: Option<&Bound<'_, PyDict>>, + unk_token: String, + continuing_subword_prefix: String, + max_input_chars_per_word: usize, ) -> PyResult> { - let mut builder = WordPiece::builder(); - - if let Some(vocab) = vocab { - match vocab { - PyVocab::Vocab(vocab) => { - let vocab: AHashMap<_, _> = vocab.into_iter().collect(); - builder = builder.vocab(vocab); - } - PyVocab::Filename(vocab_filename) => { - builder = builder.files(vocab_filename.to_string()); - } - } - } - - PyWordPiece::with_builder(builder, kwargs) - } - - /// Read a :obj:`vocab.txt` file - /// - /// This method provides a way to read and parse the content of a standard `vocab.txt` - /// file as used by the WordPiece Model, returning the relevant data structures. If you - /// want to instantiate some WordPiece models from memory, this method gives you the - /// expected input from the standard files. - /// - /// Args: - /// vocab (:obj:`str`): - /// The path to a :obj:`vocab.txt` file - /// - /// Returns: - /// :obj:`Dict[str, int]`: The vocabulary as a :obj:`dict` - #[staticmethod] - #[pyo3(text_signature = "(vocab)")] - fn read_file(vocab: &str) -> PyResult> { - let vocab = WordPiece::read_file(vocab).map_err(|e| { - exceptions::PyException::new_err(format!("Error while reading WordPiece file: {e}")) - })?; - Ok(vocab.into_iter().collect()) - } - - /// Instantiate a WordPiece model from the given file - /// - /// This method is roughly equivalent to doing:: - /// - /// vocab = WordPiece.read_file(vocab_filename) - /// wordpiece = WordPiece(vocab) - /// - /// If you don't need to keep the :obj:`vocab` values lying around, this method is - /// more optimized than manually calling :meth:`~tokenizers.models.WordPiece.read_file` to - /// initialize a :class:`~tokenizers.models.WordPiece` - /// - /// Args: - /// vocab (:obj:`str`): - /// The path to a :obj:`vocab.txt` file - /// - /// Returns: - /// :class:`~tokenizers.models.WordPiece`: An instance of WordPiece loaded from file - #[classmethod] - #[pyo3(signature = (vocab, **kwargs) -> "WordPiece")] - #[pyo3(text_signature = "(vocab, **kwargs)")] - fn from_file( - _cls: &Bound<'_, PyType>, - py: Python, - vocab: &str, - kwargs: Option<&Bound<'_, PyDict>>, - ) -> PyResult> { - let vocab = WordPiece::read_file(vocab).map_err(|e| { - exceptions::PyException::new_err(format!("Error while reading WordPiece file: {e}")) - })?; - let vocab = vocab.into_iter().collect(); - Py::new( - py, - PyWordPiece::new(py, Some(PyVocab::Vocab(vocab)), kwargs)?, - ) + let wp = WordPiece::builder() + .unk_token(unk_token) + .continuing_subword_prefix(continuing_subword_prefix) + .max_input_chars_per_word(max_input_chars_per_word) + .build() + .map_err(to_pyerr)?; + Ok(PyClassInitializer::from(PyModel { inner: wp.into() }).add_subclass(PyWordPiece)) } } -/// An implementation of the WordLevel algorithm -/// -/// Most simple tokenizer model based on mapping tokens to their corresponding id. -/// -/// Args: -/// vocab (:obj:`Dict[str, int]`, `optional`): -/// A dictionary of string keys and their ids :obj:`{"am": 0,...}` -/// -/// unk_token (:obj:`str`, `optional`): -/// The unknown token to be used by the model. -/// -/// Example:: -/// -/// >>> from tokenizers.models import WordLevel -/// >>> # Build from a vocabulary dictionary -/// >>> vocab = {"hello": 0, "world": 1, "": 2} -/// >>> model = WordLevel(vocab=vocab, unk_token="") -/// >>> # Load from file -/// >>> model = WordLevel.from_file("vocab.json", unk_token="") -/// -#[pyclass(extends=PyModel, module = "tokenizers.models", name = "WordLevel")] -pub struct PyWordLevel {} +/// The simplest model: one whole word, one id. Words outside the vocabulary +/// become `unk_token`. +#[pyclass(frozen, extends = PyModel, name = "WordLevel", module = "tokenizers.models")] +pub struct PyWordLevel; #[pymethods] impl PyWordLevel { - #[getter] - fn get_unk_token(self_: PyRef) -> String { - getter!(self_, WordLevel, unk_token.clone()) - } - - #[setter] - fn set_unk_token(self_: PyRef, unk_token: String) { - setter!(self_, WordLevel, unk_token, unk_token); - } - #[new] - #[pyo3( - signature = (vocab=None, unk_token = None), - text_signature = "(self, vocab=None, unk_token=None)" - )] - fn new( - _py: Python<'_>, - vocab: Option, - unk_token: Option, - ) -> PyResult> { - let mut builder = WordLevel::builder(); - - if let Some(vocab) = vocab { - match vocab { - PyVocab::Vocab(vocab) => { - let vocab = vocab.into_iter().collect(); - builder = builder.vocab(vocab); - } - PyVocab::Filename(vocab_filename) => { - builder = builder.files(vocab_filename.to_string()); - } - }; - } - if let Some(unk_token) = unk_token { - builder = builder.unk_token(unk_token); - } - - Ok(PyClassInitializer::::from(PyModel::from( - builder - .build() - .map_err(|e| exceptions::PyException::new_err(e.to_string()))?, - )) - .add_subclass(PyWordLevel {})) - } - - /// Read a :obj:`vocab.json` - /// - /// This method provides a way to read and parse the content of a vocabulary file, - /// returning the relevant data structures. If you want to instantiate some WordLevel models - /// from memory, this method gives you the expected input from the standard files. - /// - /// Args: - /// vocab (:obj:`str`): - /// The path to a :obj:`vocab.json` file - /// - /// Returns: - /// :obj:`Dict[str, int]`: The vocabulary as a :obj:`dict` - #[staticmethod] - #[pyo3(text_signature = "(vocab)")] - fn read_file(vocab: &str) -> PyResult> { - let vocab = WordLevel::read_file(vocab).map_err(|e| { - exceptions::PyException::new_err(format!("Error while reading WordLevel file: {e}")) - })?; - let vocab: HashMap<_, _> = vocab.into_iter().collect(); - Ok(vocab) - } - - /// Instantiate a WordLevel model from the given file - /// - /// This method is roughly equivalent to doing:: - /// - /// vocab = WordLevel.read_file(vocab_filename) - /// wordlevel = WordLevel(vocab) - /// - /// If you don't need to keep the :obj:`vocab` values lying around, this method is - /// more optimized than manually calling :meth:`~tokenizers.models.WordLevel.read_file` to - /// initialize a :class:`~tokenizers.models.WordLevel` - /// - /// Args: - /// vocab (:obj:`str`): - /// The path to a :obj:`vocab.json` file - /// - /// Returns: - /// :class:`~tokenizers.models.WordLevel`: An instance of WordLevel loaded from file - #[classmethod] - #[pyo3(signature = (vocab, unk_token = None)-> "WordLevel")] - #[pyo3(text_signature = "(vocab, unk_token=None)")] - fn from_file( - _cls: &Bound<'_, PyType>, - py: Python, - vocab: &str, - unk_token: Option, - ) -> PyResult> { - let vocab = WordLevel::read_file(vocab).map_err(|e| { - exceptions::PyException::new_err(format!("Error while reading WordLevel file: {e}")) - })?; - let vocab = vocab.into_iter().collect(); - Py::new( - py, - PyWordLevel::new(py, Some(PyVocab::Vocab(vocab)), unk_token)?, - ) + #[pyo3(signature = (*, unk_token = String::from("[UNK]")))] + fn new(unk_token: String) -> PyResult> { + let wl = WordLevel::builder() + .unk_token(unk_token) + .build() + .map_err(to_pyerr)?; + Ok(PyClassInitializer::from(PyModel { inner: wl.into() }).add_subclass(PyWordLevel)) } } -/// An implementation of the Unigram algorithm -/// -/// The Unigram algorithm is a subword tokenization algorithm based on unigram language -/// models, as used in SentencePiece. It learns a vocabulary by starting with a large -/// initial vocabulary and iteratively pruning it using the EM algorithm. -/// -/// Args: -/// vocab (:obj:`List[Tuple[str, float]]`, `optional`): -/// A list of vocabulary items and their log-probability scores, -/// e.g. ``[("am", -0.2442), ...]``. If not provided, an empty model is created. -/// -/// unk_id (:obj:`int`, `optional`): -/// The index of the unknown token in the vocabulary list. -/// -/// byte_fallback (:obj:`bool`, `optional`, defaults to :obj:`False`): -/// Whether to use SentencePiece byte fallback for characters not in the vocabulary. -/// -/// alpha (:obj:`float`, `optional`): -/// A float between 0 and 1 that represents the smoothing parameter (temperature) to use. -/// -/// nbest_size (:obj:`int`, `optional`): -/// An integer greater than 0 that represents the maximum number of best paths to consider. -/// If not set, it samples from the full lattice (i.e. all valid subword segmentations). -/// -/// Example:: -/// -/// >>> from tokenizers.models import Unigram -/// >>> # Build an empty model (to be trained) -/// >>> model = Unigram() -/// >>> # Build from a vocabulary list -/// >>> vocab = [("", 0.0), ("hello", -1.0), ("world", -1.5)] -/// >>> model = Unigram(vocab=vocab, unk_id=0) -/// -#[pyclass(extends=PyModel, module = "tokenizers.models", name = "Unigram")] -pub struct PyUnigram {} +/// The SentencePiece Unigram model: picks the most probable segmentation +/// under a learned piece vocabulary. Starts empty — train it, or load a +/// tokenizer.json. +#[pyclass(frozen, extends = PyModel, name = "Unigram", module = "tokenizers.models")] +pub struct PyUnigram; #[pymethods] impl PyUnigram { - #[getter] - fn get_alpha(self_: PyRef) -> Option { - getter!(self_, Unigram, alpha) - } - - #[setter] - fn set_alpha(self_: PyRef, alpha: Option) { - setter!(self_, Unigram, alpha, alpha); - } - - #[getter] - fn get_nbest_size(self_: PyRef) -> Option { - getter!(self_, Unigram, nbest_size) - } - - #[setter] - fn set_nbest_size(self_: PyRef, nbest_size: Option) { - setter!(self_, Unigram, nbest_size, nbest_size); - } - #[new] - #[pyo3( - signature = (vocab=None, unk_id=None, byte_fallback=None, alpha=None, nbest_size=None), - text_signature = "(self, vocab=None, unk_id=None, byte_fallback=None, alpha=None, nbest_size=None)" - )] - fn new( - vocab: Option>, - unk_id: Option, - byte_fallback: Option, - alpha: Option, - nbest_size: Option, - ) -> PyResult> { - match (vocab, unk_id, byte_fallback) { - (Some(vocab), unk_id, byte_fallback) => { - let mut model = Unigram::from(vocab, unk_id, byte_fallback.unwrap_or(false)) - .map_err(|e| { - exceptions::PyException::new_err(format!( - "Error while loading Unigram: {e}" - )) - })?; - model.alpha = alpha; - model.nbest_size = nbest_size; - Ok(PyClassInitializer::::from(PyModel::from(model)) - .add_subclass(PyUnigram {})) - } - (None, None, _) => { - let mut model = Unigram::default(); - model.alpha = alpha; - model.nbest_size = nbest_size; - Ok(PyClassInitializer::::from(PyModel::from(model)) - .add_subclass(PyUnigram {})) - } - _ => Err(exceptions::PyValueError::new_err( - "`vocab` and `unk_id` must be both specified", - )), - } - } - - /// Clears the internal cache - #[pyo3(signature = () -> "None")] - #[pyo3(text_signature = "(self)")] - fn _clear_cache(self_: PyRef) -> PyResult<()> { - let super_ = self_.as_ref(); - let mut model = super_.model.write().map_err(|e| { - exceptions::PyException::new_err(format!("Error while clearing Unigram cache: {e}")) - })?; - model.clear_cache(); - Ok(()) - } - - /// Resize the internal cache - #[pyo3(signature = (capacity) -> "None")] - #[pyo3(text_signature = "(self, capacity)")] - fn _resize_cache(self_: PyRef, capacity: usize) -> PyResult<()> { - let super_ = self_.as_ref(); - let mut model = super_.model.write().map_err(|e| { - exceptions::PyException::new_err(format!("Error while resizing Unigram cache: {e}")) - })?; - model.resize_cache(capacity); - Ok(()) + fn new() -> PyClassInitializer { + PyClassInitializer::from(PyModel { + inner: Unigram::default().into(), + }) + .add_subclass(PyUnigram) } } -/// Models Module +/// The algorithms that turn pre-tokenized pieces into token ids. #[pymodule(gil_used = false)] pub mod models { #[pymodule_export] - pub use super::PyBPE; - #[pymodule_export] - pub use super::PyModel; - #[pymodule_export] - pub use super::PyUnigram; - #[pymodule_export] - pub use super::PyWordLevel; - #[pymodule_export] - pub use super::PyWordPiece; -} - -#[cfg(test)] -mod test { - use crate::models::PyModel; - use pyo3::prelude::*; - use tk::models::ModelWrapper; - use tk::models::bpe::BPE; - - #[test] - fn get_subtype() { - Python::attach(|py| { - let py_model = PyModel::from(BPE::default()); - let py_bpe = py_model.get_as_subtype(py).unwrap(); - assert_eq!("BPE", py_bpe.bind(py).get_type().qualname().unwrap()); - }) - } - - #[test] - fn serialize() { - let rs_bpe = BPE::default(); - let rs_bpe_ser = serde_json::to_string(&rs_bpe).unwrap(); - let rs_wrapper: ModelWrapper = rs_bpe.into(); - let rs_wrapper_ser = serde_json::to_string(&rs_wrapper).unwrap(); - - let py_model = PyModel::from(rs_wrapper); - let py_ser = serde_json::to_string(&py_model).unwrap(); - assert_eq!(py_ser, rs_bpe_ser); - assert_eq!(py_ser, rs_wrapper_ser); - - let py_model: PyModel = serde_json::from_str(&rs_bpe_ser).unwrap(); - match *py_model.model.as_ref().read().unwrap() { - ModelWrapper::BPE(_) => (), - _ => panic!("Expected Bert postprocessor."), - }; - - let py_model: PyModel = serde_json::from_str(&rs_wrapper_ser).unwrap(); - match *py_model.model.as_ref().read().unwrap() { - ModelWrapper::BPE(_) => (), - _ => panic!("Expected Bert postprocessor."), - }; - } + pub use super::{PyBPE, PyModel, PyUnigram, PyWordLevel, PyWordPiece}; } diff --git a/bindings/python/src/normalizers.rs b/bindings/python/src/normalizers.rs index ef29681b7..36de6765f 100644 --- a/bindings/python/src/normalizers.rs +++ b/bindings/python/src/normalizers.rs @@ -1,1080 +1,226 @@ -use pyo3::exceptions::PyException; -use pyo3::types::*; -use pyo3::{exceptions, prelude::*}; -use std::sync::{Arc, RwLock}; - -use crate::error::ToPyResult; -use crate::utils::{PyNormalizedString, PyNormalizedStringRefMut, PyPattern}; -use serde::ser::SerializeStruct; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; -use tk::normalizers::{ - BertNormalizer, ByteLevel, Lowercase, NFC, NFD, NFKC, NFKD, Nmt, NormalizerWrapper, - Precompiled, Prepend, Replace, Strip, StripAccents, +use pyo3::prelude::*; +use tk_encode::normalizers::{ + BertNormalizer, Lowercase, NFC, NFD, NFKC, NFKD, NormalizerWrapper, Prepend, Replace, Sequence, + Strip, StripAccents, }; -use tk::{NormalizedString, Normalizer}; -use tokenizers as tk; - -/// Represents the different kind of NormalizedString we can receive from Python: -/// - Owned: Created in Python and owned by Python -/// - RefMut: A mutable reference to a NormalizedString owned by Rust -#[derive(FromPyObject)] -enum PyNormalizedStringMut<'p> { - Owned(PyRefMut<'p, PyNormalizedString>), - RefMut(PyNormalizedStringRefMut), -} -impl PyNormalizedStringMut<'_> { - /// Normalized the underlying `NormalizedString` using the provided normalizer - pub fn normalize_with(&mut self, normalizer: &N) -> PyResult<()> - where - N: Normalizer, - { - match self { - PyNormalizedStringMut::Owned(n) => normalizer.normalize(&mut n.normalized), - PyNormalizedStringMut::RefMut(n) => n.map_as_mut(|n| normalizer.normalize(n))?, - } - .map_err(|e| exceptions::PyException::new_err(format!("{e}"))) - } -} +use crate::error::to_pyerr; -/// Base class for all normalizers +/// Base class for all normalizers. /// -/// This class is not supposed to be instantiated directly. Instead, any implementation of a -/// Normalizer will return an instance of this class when instantiated. +/// A normalizer rewrites text before it is split: cleanup, case-folding, +/// Unicode normalization. Normalizers are immutable values — assigning one to +/// a tokenizer copies it. #[pyclass( - dict, - module = "tokenizers.normalizers", - name = "Normalizer", + frozen, subclass, - from_py_object + name = "Normalizer", + module = "tokenizers.normalizers" )] -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(transparent)] pub struct PyNormalizer { - pub(crate) normalizer: PyNormalizerTypeWrapper, -} - -impl PyNormalizer { - pub(crate) fn new(normalizer: PyNormalizerTypeWrapper) -> Self { - PyNormalizer { normalizer } - } - pub(crate) fn get_as_subtype(&self, py: Python<'_>) -> PyResult> { - let base = self.clone(); - Ok(match self.normalizer { - PyNormalizerTypeWrapper::Sequence(_) => Py::new(py, (PySequence {}, base))?.into_any(), - PyNormalizerTypeWrapper::Single(ref inner) => match &*inner - .as_ref() - .read() - .map_err(|_| PyException::new_err("RwLock synchronisation primitive is poisoned, cannot get subtype of PyNormalizer"))? - { - PyNormalizerWrapper::Custom(_) => { - Py::new(py, base)?.into_any() - } - PyNormalizerWrapper::Wrapped(inner) => match inner { - NormalizerWrapper::Sequence(_) => Py::new(py, (PySequence {}, base))? - .into_any(), - NormalizerWrapper::BertNormalizer(_) => { - Py::new(py, (PyBertNormalizer {}, base))?.into_any() - } - NormalizerWrapper::StripNormalizer(_) => Py::new(py, (PyStrip {}, base))? - .into_any(), - NormalizerWrapper::Prepend(_) => Py::new(py, (PyPrepend {}, base))? - .into_any(), - NormalizerWrapper::ByteLevel(_) => Py::new(py, (PyByteLevel {}, base))? - .into_any(), - NormalizerWrapper::StripAccents(_) => Py::new(py, (PyStripAccents {}, base))? - .into_any(), - NormalizerWrapper::NFC(_) => Py::new(py, (PyNFC {}, base))? - .into_any(), - NormalizerWrapper::NFD(_) => Py::new(py, (PyNFD {}, base))? - .into_any(), - NormalizerWrapper::NFKC(_) => Py::new(py, (PyNFKC {}, base))? - .into_any(), - NormalizerWrapper::NFKD(_) => Py::new(py, (PyNFKD {}, base))? - .into_any(), - NormalizerWrapper::Lowercase(_) => Py::new(py, (PyLowercase {}, base))? - .into_any(), - NormalizerWrapper::Precompiled(_) => Py::new(py, (PyPrecompiled {}, base))? - .into_any(), - NormalizerWrapper::Replace(_) => Py::new(py, (PyReplace {}, base))? - .into_any(), - NormalizerWrapper::Nmt(_) => Py::new(py, (PyNmt {}, base))? - .into_any(), - }, - }, - }) - } -} - -impl Normalizer for PyNormalizer { - fn normalize(&self, normalized: &mut NormalizedString) -> tk::Result<()> { - self.normalizer.normalize(normalized) - } + pub inner: NormalizerWrapper, } #[pymethods] impl PyNormalizer { - #[staticmethod] - #[pyo3(text_signature = "(normalizer)")] - fn custom(obj: Py) -> Self { - Self { - normalizer: PyNormalizerWrapper::Custom(CustomNormalizer::new(obj)).into(), - } - } - - fn __getstate__(&self, py: Python) -> PyResult> { - let data = serde_json::to_string(&self.normalizer).map_err(|e| { - exceptions::PyException::new_err(format!( - "Error while attempting to pickle Normalizer: {e}" - )) - })?; - Ok(PyBytes::new(py, data.as_bytes()).into()) - } - - fn __setstate__(&mut self, py: Python, state: Py) -> PyResult<()> { - match state.extract::<&[u8]>(py) { - Ok(s) => { - self.normalizer = serde_json::from_slice(s).map_err(|e| { - exceptions::PyException::new_err(format!( - "Error while attempting to unpickle Normalizer: {e}" - )) - })?; - Ok(()) - } - Err(e) => Err(e.into()), - } - } - - /// Normalize a :class:`~tokenizers.NormalizedString` in-place - /// - /// This method allows to modify a :class:`~tokenizers.NormalizedString` to - /// keep track of the alignment information. If you just want to see the result - /// of the normalization on a raw string, you can use - /// :meth:`~tokenizers.normalizers.Normalizer.normalize_str` - /// - /// Args: - /// normalized (:class:`~tokenizers.NormalizedString`): - /// The normalized string on which to apply this - /// :class:`~tokenizers.normalizers.Normalizer` - #[pyo3(text_signature = "(self, normalized)")] - fn normalize(&self, mut normalized: PyNormalizedStringMut) -> PyResult<()> { - normalized.normalize_with(&self.normalizer) - } - - /// Normalize the given string - /// - /// This method provides a way to visualize the effect of a - /// :class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment - /// information. If you need to get/convert offsets, you can use - /// :meth:`~tokenizers.normalizers.Normalizer.normalize` - /// - /// Args: - /// sequence (:obj:`str`): - /// A string to normalize - /// - /// Returns: - /// :obj:`str`: A string after normalization - #[pyo3(text_signature = "(self, sequence)")] - fn normalize_str(&self, sequence: &str) -> PyResult { - let mut normalized = NormalizedString::from(sequence); - ToPyResult(self.normalizer.normalize(&mut normalized)).into_py()?; - Ok(normalized.get().to_owned()) - } - - fn __repr__(&self) -> PyResult { - crate::utils::serde_pyo3::repr(self) - .map_err(|e| exceptions::PyException::new_err(e.to_string())) - } - - fn __str__(&self) -> PyResult { - crate::utils::serde_pyo3::to_string(self) - .map_err(|e| exceptions::PyException::new_err(e.to_string())) - } -} - -macro_rules! getter { - ($self: ident, $variant: ident, $name: ident) => {{ - let super_ = $self.as_ref(); - if let PyNormalizerTypeWrapper::Single(ref norm) = super_.normalizer { - let wrapper = norm.read().expect( - "RwLock synchronisation primitive is poisoned, cannot get subtype of PyNormalizer", - ); - if let PyNormalizerWrapper::Wrapped(NormalizerWrapper::$variant(o)) = (&*wrapper) { - o.$name.clone() - } else { - unreachable!() - } - } else { - unreachable!() - } - }}; -} - -macro_rules! setter { - ($self: ident, $variant: ident, $name: ident, $value: expr) => {{ - let super_ = $self.as_ref(); - if let PyNormalizerTypeWrapper::Single(ref norm) = super_.normalizer { - let mut wrapper = norm.write().expect( - "RwLock synchronisation primitive is poisoned, cannot get subtype of PyNormalizer", - ); - if let PyNormalizerWrapper::Wrapped(NormalizerWrapper::$variant(ref mut o)) = *wrapper { - o.$name = $value; + fn __repr__(&self) -> String { + crate::component_repr(&self.inner) + } +} + +pub fn wrap_normalizer(py: Python<'_>, inner: NormalizerWrapper) -> PyResult> { + let base = PyNormalizer { + inner: inner.clone(), + }; + let init = PyClassInitializer::from(base); + let obj = match inner { + NormalizerWrapper::BertNormalizer(_) => { + Bound::new(py, init.add_subclass(PyBertNormalizer))?.into_super() + } + NormalizerWrapper::StripNormalizer(_) => { + Bound::new(py, init.add_subclass(PyStrip))?.into_super() + } + NormalizerWrapper::StripAccents(_) => { + Bound::new(py, init.add_subclass(PyStripAccents))?.into_super() + } + NormalizerWrapper::NFC(_) => Bound::new(py, init.add_subclass(PyNFC))?.into_super(), + NormalizerWrapper::NFD(_) => Bound::new(py, init.add_subclass(PyNFD))?.into_super(), + NormalizerWrapper::NFKC(_) => Bound::new(py, init.add_subclass(PyNFKC))?.into_super(), + NormalizerWrapper::NFKD(_) => Bound::new(py, init.add_subclass(PyNFKD))?.into_super(), + NormalizerWrapper::Sequence(_) => { + Bound::new(py, init.add_subclass(PySequence))?.into_super() + } + NormalizerWrapper::Lowercase(_) => { + Bound::new(py, init.add_subclass(PyLowercase))?.into_super() + } + NormalizerWrapper::Replace(_) => Bound::new(py, init.add_subclass(PyReplace))?.into_super(), + NormalizerWrapper::Prepend(_) => Bound::new(py, init.add_subclass(PyPrepend))?.into_super(), + // Loadable from tokenizer.json but not constructible from Python: exposed as the base class. + NormalizerWrapper::Nmt(_) + | NormalizerWrapper::Precompiled(_) + | NormalizerWrapper::ByteLevel(_) => Bound::new(py, init)?, + }; + Ok(obj.unbind()) +} + +macro_rules! unit_normalizer { + ($pyname:ident, $name:literal, $inner:expr, $doc:literal) => { + #[doc = $doc] + #[pyclass(frozen, extends = PyNormalizer, name = $name, module = "tokenizers.normalizers")] + pub struct $pyname; + + #[pymethods] + impl $pyname { + #[new] + fn new() -> PyClassInitializer { + PyClassInitializer::from(PyNormalizer { + inner: $inner.into(), + }) + .add_subclass($pyname) } } - }}; -} + }; +} + +unit_normalizer!( + PyNFC, + "NFC", + NFC, + "Unicode NFC: recombines split characters (e + ´ becomes é)." +); +unit_normalizer!( + PyNFD, + "NFD", + NFD, + "Unicode NFD: splits characters into base + accents (é becomes e + ´)." +); +unit_normalizer!( + PyNFKC, + "NFKC", + NFKC, + "Unicode NFKC: NFC, plus compatibility replacements (fi becomes fi)." +); +unit_normalizer!( + PyNFKD, + "NFKD", + NFKD, + "Unicode NFKD: NFD, plus compatibility replacements (fi becomes fi)." +); +unit_normalizer!( + PyLowercase, + "Lowercase", + Lowercase, + "Lowercases everything." +); +unit_normalizer!( + PyStripAccents, + "StripAccents", + StripAccents, + "Removes accents (é becomes e). Only works on decomposed text: put NFD before it." +); + +/// Removes whitespace at the start and/or end of the text. +#[pyclass(frozen, extends = PyNormalizer, name = "Strip", module = "tokenizers.normalizers")] +pub struct PyStrip; -/// BertNormalizer -/// -/// Takes care of normalizing raw text before giving it to a Bert model. -/// This includes cleaning the text, handling accents, chinese chars and lowercasing -/// -/// Args: -/// clean_text (:obj:`bool`, `optional`, defaults to :obj:`True`): -/// Whether to clean the text, by removing any control characters -/// and replacing all whitespaces by the classic one. -/// -/// handle_chinese_chars (:obj:`bool`, `optional`, defaults to :obj:`True`): -/// Whether to handle chinese chars by putting spaces around them. -/// -/// strip_accents (:obj:`bool`, `optional`): -/// Whether to strip all accents. If this option is not specified (ie == None), -/// then it will be determined by the value for `lowercase` (as in the original Bert). -/// -/// lowercase (:obj:`bool`, `optional`, defaults to :obj:`True`): -/// Whether to lowercase. -/// -/// Example:: -/// -/// >>> from tokenizers.normalizers import BertNormalizer -/// >>> normalizer = BertNormalizer(lowercase=True) -/// >>> normalizer.normalize_str("Héllo WORLD") -/// 'hello world' -/// -#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name = "BertNormalizer")] -pub struct PyBertNormalizer {} #[pymethods] -impl PyBertNormalizer { - #[getter] - fn get_clean_text(self_: PyRef) -> bool { - getter!(self_, BertNormalizer, clean_text) - } - - #[setter] - fn set_clean_text(self_: PyRef, clean_text: bool) { - setter!(self_, BertNormalizer, clean_text, clean_text); - } - - #[getter] - fn get_handle_chinese_chars(self_: PyRef) -> bool { - getter!(self_, BertNormalizer, handle_chinese_chars) - } - - #[setter] - fn set_handle_chinese_chars(self_: PyRef, handle_chinese_chars: bool) { - setter!( - self_, - BertNormalizer, - handle_chinese_chars, - handle_chinese_chars - ); - } - - #[getter] - fn get_strip_accents(self_: PyRef) -> Option { - getter!(self_, BertNormalizer, strip_accents) - } - - #[setter] - fn set_strip_accents(self_: PyRef, strip_accents: Option) { - setter!(self_, BertNormalizer, strip_accents, strip_accents); - } - - #[getter] - fn get_lowercase(self_: PyRef) -> bool { - getter!(self_, BertNormalizer, lowercase) - } - - #[setter] - fn set_lowercase(self_: PyRef, lowercase: bool) { - setter!(self_, BertNormalizer, lowercase, lowercase) - } - - #[new] - #[pyo3(signature = ( - clean_text = true, - handle_chinese_chars = true, - strip_accents = None, - lowercase = true - ), - text_signature = "(self, clean_text=True, handle_chinese_chars=True, strip_accents=None, lowercase=True)")] - fn new( - clean_text: bool, - handle_chinese_chars: bool, - strip_accents: Option, - lowercase: bool, - ) -> PyClassInitializer { - let normalizer = - BertNormalizer::new(clean_text, handle_chinese_chars, strip_accents, lowercase); - PyClassInitializer::::from(PyNormalizer::from(normalizer)) - .add_subclass(PyBertNormalizer {}) - } -} - -/// NFD Unicode Normalizer -/// -/// Applies Unicode NFD (Canonical Decomposition) normalization. Decomposes characters into -/// their canonical components. For example, accented characters like ``é`` (U+00E9) are -/// decomposed into ``e`` (U+0065) + combining accent (U+0301). -/// -/// This is often used as a first step before stripping accents with -/// :class:`~tokenizers.normalizers.StripAccents`. -/// -/// Example:: -/// -/// >>> from tokenizers.normalizers import NFD -/// >>> normalizer = NFD() -/// >>> normalizer.normalize_str("Héllo") -/// 'He\u0301llo' -/// -#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name = "NFD")] -pub struct PyNFD {} -#[pymethods] -impl PyNFD { - #[new] - #[pyo3(text_signature = "(self)")] - fn new() -> PyClassInitializer { - PyClassInitializer::from(PyNormalizer::new(NFD.into())).add_subclass(PyNFD {}) - } -} - -/// NFKD Unicode Normalizer -/// -/// Applies Unicode NFKD (Compatibility Decomposition) normalization. Like NFD but also -/// decomposes compatibility characters. For example, the ligature ``fi`` (U+FB01) is -/// decomposed into ``f`` + ``i``. -/// -/// Example:: -/// -/// >>> from tokenizers.normalizers import NFKD -/// >>> normalizer = NFKD() -/// >>> normalizer.normalize_str("fine") -/// 'fine' -/// -#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name = "NFKD")] -pub struct PyNFKD {} -#[pymethods] -impl PyNFKD { - #[new] - #[pyo3(text_signature = "(self)")] - fn new() -> PyClassInitializer { - PyClassInitializer::::from(PyNormalizer::from(NFKD)).add_subclass(PyNFKD {}) - } -} - -/// NFC Unicode Normalizer -/// -/// Applies Unicode NFC (Canonical Decomposition, followed by Canonical Composition) -/// normalization. First decomposes characters, then recomposes them using canonical -/// composition rules. This produces the canonical composed form. -/// -/// Example:: -/// -/// >>> from tokenizers.normalizers import NFC -/// >>> normalizer = NFC() -/// >>> normalizer.normalize_str("e\u0301") # 'e' + combining accent -/// 'é' -/// -#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name = "NFC")] -pub struct PyNFC {} -#[pymethods] -impl PyNFC { - #[new] - #[pyo3(text_signature = "(self)")] - fn new() -> PyClassInitializer { - PyClassInitializer::::from(PyNormalizer::from(NFC)).add_subclass(PyNFC {}) - } -} - -/// NFKC Unicode Normalizer -/// -/// Applies Unicode NFKC (Compatibility Decomposition, followed by Canonical Composition) -/// normalization. Like NFC but also maps compatibility characters to their canonical -/// equivalents. This is the normalization used by Python's :func:`str.casefold` and -/// by many NLP pipelines. -/// -/// Example:: -/// -/// >>> from tokenizers.normalizers import NFKC -/// >>> normalizer = NFKC() -/// >>> normalizer.normalize_str("fine caf\u00e9") -/// 'fine café' -/// -#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name = "NFKC")] -pub struct PyNFKC {} -#[pymethods] -impl PyNFKC { +impl PyStrip { #[new] - #[pyo3(text_signature = "(self)")] - fn new() -> PyClassInitializer { - PyClassInitializer::::from(PyNormalizer::from(NFKC)).add_subclass(PyNFKC {}) + #[pyo3(signature = (*, left = true, right = true))] + fn new(left: bool, right: bool) -> PyClassInitializer { + PyClassInitializer::from(PyNormalizer { + inner: Strip::new(left, right).into(), + }) + .add_subclass(PyStrip) } } -/// Allows concatenating multiple other Normalizer as a Sequence. -/// All the normalizers run in sequence in the given order -/// -/// Args: -/// normalizers (:obj:`List[Normalizer]`): -/// A list of Normalizer to be run as a sequence -/// -/// Example:: -/// -/// >>> from tokenizers.normalizers import NFD, Lowercase, StripAccents, Sequence -/// >>> normalizer = Sequence([NFD(), Lowercase(), StripAccents()]) -/// >>> normalizer.normalize_str("Héllo Wörld") -/// 'hello world' -/// -#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name = "Sequence")] -pub struct PySequence {} +/// Replaces every occurrence of `pattern` with `content`. With `regex=True` +/// the pattern is a regular expression. +#[pyclass(frozen, extends = PyNormalizer, name = "Replace", module = "tokenizers.normalizers")] +pub struct PyReplace; #[pymethods] -impl PySequence { +impl PyReplace { #[new] - #[pyo3(signature = (normalizers), text_signature = "(self, normalizers)")] - fn new(normalizers: &Bound<'_, PyList>) -> PyResult> { - let mut sequence = Vec::with_capacity(normalizers.len()); - for n in normalizers.iter() { - let normalizer: PyRef = n.extract()?; - match &normalizer.normalizer { - PyNormalizerTypeWrapper::Sequence(inner) => sequence.extend(inner.iter().cloned()), - PyNormalizerTypeWrapper::Single(inner) => sequence.push(inner.clone()), - } - } - Ok( - PyClassInitializer::from(PyNormalizer::new(PyNormalizerTypeWrapper::Sequence( - sequence, - ))) - .add_subclass(PySequence {}), - ) - } - - fn __getnewargs__<'p>(&self, py: Python<'p>) -> PyResult> { - PyTuple::new(py, [PyList::empty(py)]) - } - - fn __len__(self_: PyRef<'_, Self>) -> usize { - match &self_.as_ref().normalizer { - PyNormalizerTypeWrapper::Sequence(inner) => inner.len(), - PyNormalizerTypeWrapper::Single(_) => 1, - } - } - - fn __getitem__(self_: PyRef<'_, Self>, py: Python<'_>, index: usize) -> PyResult> { - match &self_.as_ref().normalizer { - PyNormalizerTypeWrapper::Sequence(inner) => match inner.get(index) { - Some(item) => PyNormalizer::new(PyNormalizerTypeWrapper::Single(item.clone())) - .get_as_subtype(py), - _ => Err(PyErr::new::( - "Index not found", - )), - }, - PyNormalizerTypeWrapper::Single(inner) => { - PyNormalizer::new(PyNormalizerTypeWrapper::Single(inner.clone())).get_as_subtype(py) - } - } - } - - fn __setitem__(self_: PyRef<'_, Self>, index: usize, value: Bound<'_, PyAny>) -> PyResult<()> { - let norm: PyNormalizer = value.extract()?; - let PyNormalizerTypeWrapper::Single(norm) = norm.normalizer else { - return Err(PyException::new_err("normalizer should not be a sequence")); - }; - match &self_.as_ref().normalizer { - PyNormalizerTypeWrapper::Sequence(inner) => match inner.get(index) { - Some(item) => { - *item - .write() - .map_err(|_| PyException::new_err("RwLock synchronisation primitive is poisoned, cannot get subtype of PyNormalizer"))? = norm - .read() - .map_err(|_| PyException::new_err("RwLock synchronisation primitive is poisoned, cannot get subtype of PyNormalizer"))? - .clone(); - } - _ => { - return Err(PyErr::new::( - "Index not found", - )); - } - }, - PyNormalizerTypeWrapper::Single(_) => { - return Err(PyException::new_err("normalizer is not a sequence")); - } + #[pyo3(signature = (pattern, content, *, regex = false))] + fn new(pattern: &str, content: &str, regex: bool) -> PyResult> { + use tk_encode::normalizers::replace::ReplacePattern; + let pattern = if regex { + ReplacePattern::Regex(pattern.to_owned()) + } else { + ReplacePattern::String(pattern.to_owned()) }; - Ok(()) + let replace = Replace::new(pattern, content).map_err(to_pyerr)?; + Ok(PyClassInitializer::from(PyNormalizer { + inner: replace.into(), + }) + .add_subclass(PyReplace)) } } -/// Lowercase Normalizer -/// -/// Converts all text to lowercase using Unicode-aware lowercasing. This is equivalent -/// to calling :meth:`str.lower` on the input. -/// -/// Example:: -/// -/// >>> from tokenizers.normalizers import Lowercase -/// >>> normalizer = Lowercase() -/// >>> normalizer.normalize_str("Hello World") -/// 'hello world' -/// -#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name = "Lowercase")] -pub struct PyLowercase {} -#[pymethods] -impl PyLowercase { - #[new] - #[pyo3(text_signature = "(self)")] - fn new() -> PyClassInitializer { - PyClassInitializer::::from(PyNormalizer::from(Lowercase)) - .add_subclass(PyLowercase {}) - } -} +/// Puts a fixed string in front of the text (SentencePiece prepends "▁"). +#[pyclass(frozen, extends = PyNormalizer, name = "Prepend", module = "tokenizers.normalizers")] +pub struct PyPrepend; -/// Strip normalizer -/// -/// Removes leading and/or trailing whitespace from the input string. -/// -/// Args: -/// left (:obj:`bool`, defaults to :obj:`True`): -/// Whether to strip leading (left) whitespace. -/// -/// right (:obj:`bool`, defaults to :obj:`True`): -/// Whether to strip trailing (right) whitespace. -/// -/// Example:: -/// -/// >>> from tokenizers.normalizers import Strip -/// >>> normalizer = Strip() -/// >>> normalizer.normalize_str(" hello world ") -/// 'hello world' -/// >>> Strip(right=False).normalize_str(" hello ") -/// 'hello ' -/// -#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name = "Strip")] -pub struct PyStrip {} -#[pymethods] -impl PyStrip { - #[getter] - fn get_left(self_: PyRef) -> bool { - getter!(self_, StripNormalizer, strip_left) - } - - #[setter] - fn set_left(self_: PyRef, left: bool) { - setter!(self_, StripNormalizer, strip_left, left) - } - - #[getter] - fn get_right(self_: PyRef) -> bool { - getter!(self_, StripNormalizer, strip_right) - } - - #[setter] - fn set_right(self_: PyRef, right: bool) { - setter!(self_, StripNormalizer, strip_right, right) - } - - #[new] - #[pyo3(signature = (left = true, right = true), text_signature = "(self, left=True, right=True)")] - fn new(left: bool, right: bool) -> PyClassInitializer { - PyClassInitializer::::from(PyNormalizer::from(Strip::new(left, right))) - .add_subclass(PyStrip {}) - } -} - -/// Prepend normalizer -/// -/// Prepends a given string to the beginning of the input. This is typically used to -/// add a meta-symbol such as ``▁`` (U+2581) at the start of each sequence, which is -/// the convention used by SentencePiece-based models to indicate that a token appears -/// at the start of a word. -/// -/// Args: -/// prepend (:obj:`str`, defaults to :obj:`"▁"`): -/// The string to prepend to the input. -/// -/// Example:: -/// -/// >>> from tokenizers.normalizers import Prepend -/// >>> normalizer = Prepend("▁") -/// >>> normalizer.normalize_str("hello") -/// '▁hello' -/// -#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name = "Prepend")] -pub struct PyPrepend {} #[pymethods] impl PyPrepend { - #[getter] - fn get_prepend(self_: PyRef) -> String { - getter!(self_, Prepend, prepend) - } - - #[setter] - fn set_prepend(self_: PyRef, prepend: String) { - setter!(self_, Prepend, prepend, prepend) - } - #[new] - #[pyo3(signature = (prepend="▁".to_string()), text_signature = "(self, prepend)")] fn new(prepend: String) -> PyClassInitializer { - PyClassInitializer::::from(PyNormalizer::from(Prepend::new(prepend))) - .add_subclass(PyPrepend {}) + PyClassInitializer::from(PyNormalizer { + inner: Prepend::new(prepend).into(), + }) + .add_subclass(PyPrepend) } } -/// Bytelevel Normalizer -/// -/// Converts all bytes in the input to their Unicode representation using the GPT-2 -/// byte-to-unicode mapping. Every byte value (0–255) is mapped to a unique visible -/// character so that any arbitrary binary input can be tokenized without needing a -/// special unknown token. -/// -/// This normalizer is used together with the -/// :class:`~tokenizers.pre_tokenizers.ByteLevel` pre-tokenizer and -/// :class:`~tokenizers.decoders.ByteLevel` decoder. -/// -/// Example:: -/// -/// >>> from tokenizers.normalizers import ByteLevel -/// >>> normalizer = ByteLevel() -/// >>> normalizer.normalize_str("hello\nworld") -/// 'helloĊworld' -/// -#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name = "ByteLevel")] -pub struct PyByteLevel {} -#[pymethods] -impl PyByteLevel { - #[new] - #[pyo3(text_signature = "(self)")] - fn new() -> PyClassInitializer { - PyClassInitializer::::from(PyNormalizer::from(ByteLevel::new())) - .add_subclass(PyByteLevel {}) - } -} +/// The BERT cleanup: removes control characters, puts spaces around CJK +/// characters, and optionally strips accents and lowercases. +/// `strip_accents=None` means "follow the lowercase setting", like the +/// original BERT. +#[pyclass(frozen, extends = PyNormalizer, name = "BertNormalizer", module = "tokenizers.normalizers")] +pub struct PyBertNormalizer; -/// StripAccents normalizer -/// -/// Strips all accent marks (combining diacritical characters) from the input. This -/// normalizer should typically be used after applying :class:`~tokenizers.normalizers.NFD` -/// or :class:`~tokenizers.normalizers.NFKD` decomposition, which separates base -/// characters from their combining accents. -/// -/// Example:: -/// -/// >>> from tokenizers.normalizers import NFD, StripAccents, Sequence -/// >>> normalizer = Sequence([NFD(), StripAccents()]) -/// >>> normalizer.normalize_str("café") -/// 'cafe' -/// -#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name = "StripAccents")] -pub struct PyStripAccents {} #[pymethods] -impl PyStripAccents { - #[new] - #[pyo3(text_signature = "(self)")] - fn new() -> PyClassInitializer { - PyClassInitializer::::from(PyNormalizer::from(StripAccents)) - .add_subclass(PyStripAccents {}) - } -} - -/// Nmt normalizer -/// -/// Normalizer used in the Google NMT pipeline. It handles various text cleaning tasks -/// including removing control characters, normalizing whitespace, and replacing certain -/// Unicode characters. This is equivalent to the normalization done in the original -/// SentencePiece NMT preprocessing. -/// -/// Example:: -/// -/// >>> from tokenizers.normalizers import Nmt -/// >>> normalizer = Nmt() -/// >>> normalizer.normalize_str("Hello\x00World") -/// 'Hello World' -/// -#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name = "Nmt")] -pub struct PyNmt {} -#[pymethods] -impl PyNmt { +impl PyBertNormalizer { #[new] - #[pyo3(text_signature = "(self)")] - fn new() -> PyClassInitializer { - PyClassInitializer::::from(PyNormalizer::from(Nmt)).add_subclass(PyNmt {}) + #[pyo3(signature = (*, clean_text = true, handle_chinese_chars = true, strip_accents = None, lowercase = true))] + fn new( + clean_text: bool, + handle_chinese_chars: bool, + strip_accents: Option, + lowercase: bool, + ) -> PyClassInitializer { + let inner = BertNormalizer::new(clean_text, handle_chinese_chars, strip_accents, lowercase); + PyClassInitializer::from(PyNormalizer { + inner: inner.into(), + }) + .add_subclass(PyBertNormalizer) } } -/// Precompiled normalizer -/// -/// A normalizer that uses a precompiled character map built from a SentencePiece model. -/// This normalizer is automatically extracted from SentencePiece ``.model`` files and -/// should not be constructed manually — it is used internally for full compatibility -/// with SentencePiece-based tokenizers. -/// -/// Args: -/// precompiled_charsmap (:obj:`bytes`): -/// The raw bytes of the precompiled character map, as found inside a -/// SentencePiece ``.model`` file. -/// -#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name = "Precompiled")] -pub struct PyPrecompiled {} -#[pymethods] -impl PyPrecompiled { - #[new] - #[pyo3(text_signature = "(self, precompiled_charsmap)")] - fn new(precompiled_charsmap: Vec) -> PyResult> { - // let precompiled_charsmap: Vec = FromPyObject::extract(py_precompiled_charsmap)?; - Ok(PyClassInitializer::::from(PyNormalizer::from( - Precompiled::from(&precompiled_charsmap).map_err(|e| { - exceptions::PyException::new_err(format!( - "Error while attempting to build Precompiled normalizer: {e}" - )) - })?, - )) - .add_subclass(PyPrecompiled {})) - } -} +/// Runs several normalizers in order. +#[pyclass(frozen, extends = PyNormalizer, name = "Sequence", module = "tokenizers.normalizers")] +pub struct PySequence; -/// Replace normalizer -/// -/// Replaces occurrences of a pattern in the input string with the given content. -/// The pattern can be either a plain string or a regular expression wrapped in -/// :class:`~tokenizers.Regex`. -/// -/// Args: -/// pattern (:obj:`str` or :class:`~tokenizers.Regex`): -/// The pattern to search for. Use a plain string for literal replacement, -/// or wrap a regex pattern in :class:`~tokenizers.Regex` for regex replacement. -/// -/// content (:obj:`str`): -/// The string to replace each match with. -/// -/// Example:: -/// -/// >>> from tokenizers import Regex -/// >>> from tokenizers.normalizers import Replace -/// >>> # Replace a literal string -/// >>> Replace(".", " ").normalize_str("hello.world") -/// 'hello world' -/// >>> # Replace using a regex -/// >>> Replace(Regex(r"\s+"), " ").normalize_str("hello world") -/// 'hello world' -/// -#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name = "Replace")] -pub struct PyReplace {} #[pymethods] -impl PyReplace { +impl PySequence { #[new] - #[pyo3(text_signature = "(self, pattern, content)")] - fn new(pattern: PyPattern, content: String) -> PyResult> { - Ok(PyClassInitializer::::from(PyNormalizer::from( - ToPyResult(Replace::new(pattern, content)).into_py()?, - )) - .add_subclass(PyReplace {})) - } - - #[getter] - fn get_pattern(_self: PyRef) -> PyResult<()> { - Err(PyException::new_err("Cannot get pattern")) - } - - #[setter] - fn set_pattern(_self: PyRef, _pattern: PyPattern) -> PyResult<()> { - Err(PyException::new_err( - "Cannot set pattern, please instantiate a new replace pattern instead", - )) - } - - #[getter] - fn get_content(self_: PyRef) -> String { - getter!(self_, Replace, content) - } - - #[setter] - fn set_content(self_: PyRef, content: String) { - setter!(self_, Replace, content, content) - } -} - -#[derive(Clone, Debug)] -pub(crate) struct CustomNormalizer { - inner: Py, -} -impl CustomNormalizer { - pub fn new(inner: Py) -> Self { - Self { inner } - } -} - -impl tk::tokenizer::Normalizer for CustomNormalizer { - fn normalize(&self, normalized: &mut NormalizedString) -> tk::Result<()> { - Python::attach(|py| { - let normalized = PyNormalizedStringRefMut::new(normalized); - let py_normalized = self.inner.bind(py); - py_normalized.call_method("normalize", (normalized.get().clone(),), None)?; - Ok(()) + fn new(normalizers: Vec>) -> PyClassInitializer { + let inner: Vec = normalizers.iter().map(|n| n.inner.clone()).collect(); + PyClassInitializer::from(PyNormalizer { + inner: Sequence::new(inner).into(), }) + .add_subclass(PySequence) } } -impl Serialize for CustomNormalizer { - fn serialize(&self, _serializer: S) -> Result - where - S: Serializer, - { - Err(serde::ser::Error::custom( - "Custom Normalizer cannot be serialized", - )) - } -} - -impl<'de> Deserialize<'de> for CustomNormalizer { - fn deserialize(_deserializer: D) -> Result - where - D: Deserializer<'de>, - { - Err(serde::de::Error::custom( - "Custom Normalizer cannot be deserialized", - )) - } -} - -#[derive(Clone, Debug, Deserialize)] -#[serde(untagged)] -pub(crate) enum PyNormalizerWrapper { - Custom(CustomNormalizer), - Wrapped(NormalizerWrapper), -} - -impl Serialize for PyNormalizerWrapper { - fn serialize(&self, serializer: S) -> Result<::Ok, ::Error> - where - S: Serializer, - { - match self { - PyNormalizerWrapper::Wrapped(inner) => inner.serialize(serializer), - PyNormalizerWrapper::Custom(inner) => inner.serialize(serializer), - } - } -} - -#[derive(Debug, Clone)] -pub(crate) enum PyNormalizerTypeWrapper { - Sequence(Vec>>), - Single(Arc>), -} - -/// XXX: we need to manually implement deserialize here because of the structure of the -/// PyNormalizerTypeWrapper enum. Given the underlying PyNormalizerWrapper can contain a Sequence, -/// default deserialization will give us a PyNormalizerTypeWrapper::Single(Sequence) when we'd like -/// it to be PyNormalizerTypeWrapper::Sequence(// ...). -impl<'de> Deserialize<'de> for PyNormalizerTypeWrapper { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let wrapper = NormalizerWrapper::deserialize(deserializer)?; - let py_wrapper: PyNormalizerWrapper = wrapper.into(); - Ok(py_wrapper.into()) - } -} - -impl Serialize for PyNormalizerTypeWrapper { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - match self { - PyNormalizerTypeWrapper::Sequence(seq) => { - let mut ser = serializer.serialize_struct("Sequence", 2)?; - ser.serialize_field("type", "Sequence")?; - ser.serialize_field("normalizers", seq)?; - ser.end() - } - PyNormalizerTypeWrapper::Single(inner) => inner.serialize(serializer), - } - } -} - -impl From for PyNormalizerWrapper -where - I: Into, -{ - fn from(norm: I) -> Self { - PyNormalizerWrapper::Wrapped(norm.into()) - } -} - -impl From for PyNormalizerTypeWrapper -where - I: Into, -{ - fn from(norm: I) -> Self { - let norm = norm.into(); - match norm { - PyNormalizerWrapper::Wrapped(NormalizerWrapper::Sequence(seq)) => { - PyNormalizerTypeWrapper::Sequence( - seq.into_iter() - .map(|e| Arc::new(RwLock::new(PyNormalizerWrapper::Wrapped(e.clone())))) - .collect(), - ) - } - _ => PyNormalizerTypeWrapper::Single(Arc::new(RwLock::new(norm))), - } - } -} - -impl From for PyNormalizer -where - I: Into, -{ - fn from(norm: I) -> Self { - PyNormalizer { - normalizer: norm.into().into(), - } - } -} - -impl Normalizer for PyNormalizerTypeWrapper { - fn normalize(&self, normalized: &mut NormalizedString) -> tk::Result<()> { - match self { - PyNormalizerTypeWrapper::Single(inner) => inner - .read() - .map_err(|_| PyException::new_err("RwLock synchronisation primitive is poisoned, cannot get subtype of PyNormalizer"))? - .normalize(normalized), - PyNormalizerTypeWrapper::Sequence(inner) => inner.iter().try_for_each(|n| { - n.read() - .map_err(|_| PyException::new_err("RwLock synchronisation primitive is poisoned, cannot get subtype of PyNormalizer"))? - .normalize(normalized) - }), - } - } -} - -impl Normalizer for PyNormalizerWrapper { - fn normalize(&self, normalized: &mut NormalizedString) -> tk::Result<()> { - match self { - PyNormalizerWrapper::Wrapped(inner) => inner.normalize(normalized), - PyNormalizerWrapper::Custom(inner) => inner.normalize(normalized), - } - } -} - -/// Normalizers Module +/// Text cleanup that runs before the text is split. #[pymodule(gil_used = false)] pub mod normalizers { #[pymodule_export] - pub use super::PyBertNormalizer; - #[pymodule_export] - pub use super::PyByteLevel; - #[pymodule_export] - pub use super::PyLowercase; - #[pymodule_export] - pub use super::PyNFC; - #[pymodule_export] - pub use super::PyNFD; - #[pymodule_export] - pub use super::PyNFKC; - #[pymodule_export] - pub use super::PyNFKD; - #[pymodule_export] - pub use super::PyNmt; - #[pymodule_export] - pub use super::PyNormalizer; - #[pymodule_export] - pub use super::PyPrecompiled; - #[pymodule_export] - pub use super::PyPrepend; - #[pymodule_export] - pub use super::PyReplace; - #[pymodule_export] - pub use super::PySequence; - #[pymodule_export] - pub use super::PyStrip; - #[pymodule_export] - pub use super::PyStripAccents; -} - -#[cfg(test)] -mod test { - use pyo3::prelude::*; - use tk::normalizers::NormalizerWrapper; - use tk::normalizers::unicode::{NFC, NFKC}; - use tk::normalizers::utils::Sequence; - - use crate::normalizers::{PyNormalizer, PyNormalizerTypeWrapper, PyNormalizerWrapper}; - - #[test] - fn get_subtype() { - Python::attach(|py| { - let py_norm = PyNormalizer::new(NFC.into()); - let py_nfc = py_norm.get_as_subtype(py).unwrap(); - assert_eq!("NFC", py_nfc.bind(py).get_type().qualname().unwrap()); - }) - } - - #[test] - fn serialize() { - let py_wrapped: PyNormalizerWrapper = NFKC.into(); - let py_ser = serde_json::to_string(&py_wrapped).unwrap(); - let rs_wrapped = NormalizerWrapper::NFKC(NFKC); - let rs_ser = serde_json::to_string(&rs_wrapped).unwrap(); - assert_eq!(py_ser, rs_ser); - let py_norm: PyNormalizer = serde_json::from_str(&rs_ser).unwrap(); - match py_norm.normalizer { - PyNormalizerTypeWrapper::Single(inner) => match *inner.as_ref().read().unwrap() { - PyNormalizerWrapper::Wrapped(NormalizerWrapper::NFKC(_)) => {} - _ => panic!("Expected NFKC"), - }, - _ => panic!("Expected wrapped, not sequence."), - } - - let py_seq: PyNormalizerWrapper = Sequence::new(vec![NFC.into(), NFKC.into()]).into(); - let py_wrapper_ser = serde_json::to_string(&py_seq).unwrap(); - let rs_wrapped = NormalizerWrapper::Sequence(Sequence::new(vec![NFC.into(), NFKC.into()])); - let rs_ser = serde_json::to_string(&rs_wrapped).unwrap(); - assert_eq!(py_wrapper_ser, rs_ser); - - let py_seq = PyNormalizer::new(py_seq.into()); - let py_ser = serde_json::to_string(&py_seq).unwrap(); - assert_eq!(py_wrapper_ser, py_ser); - - let rs_seq = Sequence::new(vec![NFC.into(), NFKC.into()]); - let rs_ser = serde_json::to_string(&rs_seq).unwrap(); - assert_eq!(py_wrapper_ser, rs_ser); - } - - #[test] - fn deserialize_sequence() { - let string = r#"{"type": "NFKC"}"#; - let normalizer: PyNormalizer = serde_json::from_str(string).unwrap(); - match normalizer.normalizer { - PyNormalizerTypeWrapper::Single(inner) => match *inner.as_ref().read().unwrap() { - PyNormalizerWrapper::Wrapped(NormalizerWrapper::NFKC(_)) => {} - _ => panic!("Expected NFKC"), - }, - _ => panic!("Expected wrapped, not sequence."), - } - - let sequence_string = format!(r#"{{"type": "Sequence", "normalizers": [{string}]}}"#); - let normalizer: PyNormalizer = serde_json::from_str(&sequence_string).unwrap(); - - match normalizer.normalizer { - PyNormalizerTypeWrapper::Sequence(inner) => { - assert_eq!(inner.len(), 1); - match *inner[0].as_ref().read().unwrap() { - PyNormalizerWrapper::Wrapped(NormalizerWrapper::NFKC(_)) => {} - _ => panic!("Expected NFKC"), - }; - } - _ => panic!("Expected sequence"), - }; - } + pub use super::{ + PyBertNormalizer, PyLowercase, PyNFC, PyNFD, PyNFKC, PyNFKD, PyNormalizer, PyPrepend, + PyReplace, PySequence, PyStrip, PyStripAccents, + }; } diff --git a/bindings/python/src/pre_tokenizers.rs b/bindings/python/src/pre_tokenizers.rs index 1e11185f1..46f1fd1cb 100644 --- a/bindings/python/src/pre_tokenizers.rs +++ b/bindings/python/src/pre_tokenizers.rs @@ -1,1141 +1,283 @@ -use std::sync::{Arc, RwLock}; - -use pyo3::exceptions; -use pyo3::exceptions::PyException; +use pyo3::exceptions::PyValueError; use pyo3::prelude::*; -use pyo3::types::*; -use serde::ser::SerializeStruct; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; - -use tk::normalizer::SplitDelimiterBehavior; -use tk::pre_tokenizers::PreTokenizerWrapper; -use tk::pre_tokenizers::bert::BertPreTokenizer; -use tk::pre_tokenizers::byte_level::ByteLevel; -use tk::pre_tokenizers::delimiter::CharDelimiterSplit; -use tk::pre_tokenizers::digits::Digits; -use tk::pre_tokenizers::fixed_length::FixedLength; -use tk::pre_tokenizers::metaspace::{Metaspace, PrependScheme}; -use tk::pre_tokenizers::punctuation::Punctuation; -use tk::pre_tokenizers::split::Split; -use tk::pre_tokenizers::unicode_scripts::UnicodeScripts; -use tk::pre_tokenizers::whitespace::{Whitespace, WhitespaceSplit}; -use tk::tokenizer::Offsets; -use tk::{PreTokenizedString, PreTokenizer}; -use tokenizers as tk; - -use super::error::ToPyResult; -use super::utils::*; - -/// Base class for all pre-tokenizers -/// -/// This class is not supposed to be instantiated directly. Instead, any implementation of a -/// PreTokenizer will return an instance of this class when instantiated. +use tk_encode::pre_tokenizers::PreTokenizerWrapper; +use tk_encode::pre_tokenizers::bert::BertPreTokenizer; +use tk_encode::pre_tokenizers::byte_level::ByteLevel; +use tk_encode::pre_tokenizers::delimiter::CharDelimiterSplit; +use tk_encode::pre_tokenizers::digits::Digits; +use tk_encode::pre_tokenizers::fixed_length::FixedLength; +use tk_encode::pre_tokenizers::punctuation::Punctuation; +use tk_encode::pre_tokenizers::sequence::Sequence; +use tk_encode::pre_tokenizers::split::{Split, SplitPattern}; +use tk_encode::pre_tokenizers::unicode_scripts::UnicodeScripts; +use tk_encode::pre_tokenizers::whitespace::{Whitespace, WhitespaceSplit}; +use tk_encode::tokenizer::SplitDelimiterBehavior; + +use crate::error::to_pyerr; + +pub fn parse_behavior(s: &str) -> PyResult { + match s { + "removed" => Ok(SplitDelimiterBehavior::Removed), + "isolated" => Ok(SplitDelimiterBehavior::Isolated), + "merged_with_previous" => Ok(SplitDelimiterBehavior::MergedWithPrevious), + "merged_with_next" => Ok(SplitDelimiterBehavior::MergedWithNext), + "contiguous" => Ok(SplitDelimiterBehavior::Contiguous), + other => Err(PyValueError::new_err(format!( + "unknown behavior {other:?}; expected one of: removed, isolated, \ + merged_with_previous, merged_with_next, contiguous" + ))), + } +} + +/// Base class for all pre-tokenizers. +/// +/// A pre-tokenizer cuts text into pieces (usually words); the model then turns +/// each piece into token ids. Pre-tokenizers are immutable values — assigning +/// one to a tokenizer copies it. Only pre-tokenizers the encode pipeline can +/// run are constructible here; `Metaspace` is not available yet. #[pyclass( - dict, - module = "tokenizers.pre_tokenizers", - name = "PreTokenizer", + frozen, subclass, - from_py_object + name = "PreTokenizer", + module = "tokenizers.pre_tokenizers" )] -#[derive(Clone, Serialize, Deserialize)] -#[serde(transparent)] pub struct PyPreTokenizer { - pub(crate) pretok: PyPreTokenizerTypeWrapper, + pub inner: PreTokenizerWrapper, } +#[pymethods] impl PyPreTokenizer { - #[allow(dead_code)] - pub(crate) fn new(pretok: PyPreTokenizerTypeWrapper) -> Self { - PyPreTokenizer { pretok } - } - - pub(crate) fn get_as_subtype(&self, py: Python<'_>) -> PyResult> { - let base = self.clone(); - Ok(match self.pretok { - PyPreTokenizerTypeWrapper::Sequence(_) => Py::new(py, (PySequence {}, base))? - .into_any(), - PyPreTokenizerTypeWrapper::Single(ref inner) => { - match &*inner - .as_ref() - .read() - .map_err(|_| PyException::new_err("RwLock synchronisation primitive is poisoned, cannot get subtype of PyPreTokenizer"))? - { - PyPreTokenizerWrapper::Custom(_) => { - Py::new(py, base)?.into_pyobject(py)?.into_any().into() - } - PyPreTokenizerWrapper::Wrapped(inner) => match inner { - PreTokenizerWrapper::Whitespace(_) => Py::new(py, (PyWhitespace {}, base))? - .into_any(), - PreTokenizerWrapper::Split(_) => Py::new(py, (PySplit {}, base))? - .into_any(), - PreTokenizerWrapper::Punctuation(_) => { - Py::new(py, (PyPunctuation {}, base))? - .into_any() - } - PreTokenizerWrapper::Sequence(_) => Py::new(py, (PySequence {}, base))? - .into_any(), - PreTokenizerWrapper::Metaspace(_) => Py::new(py, (PyMetaspace {}, base))? - .into_any(), - PreTokenizerWrapper::Delimiter(_) => { - Py::new(py, (PyCharDelimiterSplit {}, base))? - .into_any() - } - PreTokenizerWrapper::WhitespaceSplit(_) => { - Py::new(py, (PyWhitespaceSplit {}, base))? - .into_any() - } - PreTokenizerWrapper::ByteLevel(_) => Py::new(py, (PyByteLevel {}, base))? - .into_any(), - PreTokenizerWrapper::BertPreTokenizer(_) => { - Py::new(py, (PyBertPreTokenizer {}, base))? - .into_any() - } - PreTokenizerWrapper::Digits(_) => Py::new(py, (PyDigits {}, base))? - .into_any(), - PreTokenizerWrapper::UnicodeScripts(_) => { - Py::new(py, (PyUnicodeScripts {}, base))? - .into_any() - } - PreTokenizerWrapper::FixedLength(_) => { - Py::new(py, (PyFixedLength {}, base))? - .into_any() - } - }, - } - } - }) - } -} - -impl PreTokenizer for PyPreTokenizer { - fn pre_tokenize(&self, normalized: &mut PreTokenizedString) -> tk::Result<()> { - self.pretok.pre_tokenize(normalized) + fn __repr__(&self) -> String { + crate::component_repr(&self.inner) } } -#[pymethods] -impl PyPreTokenizer { - #[staticmethod] - #[pyo3(text_signature = "(pretok)")] - fn custom(pretok: Py) -> Self { - PyPreTokenizer { - pretok: PyPreTokenizerWrapper::Custom(CustomPreTokenizer::new(pretok)).into(), +pub fn wrap_pre_tokenizer( + py: Python<'_>, + inner: PreTokenizerWrapper, +) -> PyResult> { + let base = PyPreTokenizer { + inner: inner.clone(), + }; + let init = PyClassInitializer::from(base); + let obj = match inner { + PreTokenizerWrapper::BertPreTokenizer(_) => { + Bound::new(py, init.add_subclass(PyBertPreTokenizer))?.into_super() } - } - - fn __getstate__(&self, py: Python) -> PyResult> { - let data = serde_json::to_string(&self.pretok).map_err(|e| { - exceptions::PyException::new_err(format!( - "Error while attempting to pickle PreTokenizer: {e}" - )) - })?; - Ok(PyBytes::new(py, data.as_bytes()).into()) - } - - fn __setstate__(&mut self, py: Python, state: Py) -> PyResult<()> { - match state.extract::<&[u8]>(py) { - Ok(s) => { - let unpickled = serde_json::from_slice(s).map_err(|e| { - exceptions::PyException::new_err(format!( - "Error while attempting to unpickle PreTokenizer: {e}" - )) - })?; - self.pretok = unpickled; - Ok(()) - } - Err(e) => Err(e.into()), + PreTokenizerWrapper::ByteLevel(_) => { + Bound::new(py, init.add_subclass(PyByteLevel))?.into_super() } - } - - /// Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place - /// - /// This method allows to modify a :class:`~tokenizers.PreTokenizedString` to - /// keep track of the pre-tokenization, and leverage the capabilities of the - /// :class:`~tokenizers.PreTokenizedString`. If you just want to see the result of - /// the pre-tokenization of a raw string, you can use - /// :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str` - /// - /// Args: - /// pretok (:class:`~tokenizers.PreTokenizedString): - /// The pre-tokenized string on which to apply this - /// :class:`~tokenizers.pre_tokenizers.PreTokenizer` - #[pyo3(text_signature = "(self, pretok)")] - fn pre_tokenize(&self, pretok: &mut PyPreTokenizedString) -> PyResult<()> { - ToPyResult(self.pretok.pre_tokenize(&mut pretok.pretok)).into() - } - - /// Pre tokenize the given string - /// - /// This method provides a way to visualize the effect of a - /// :class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the - /// alignment, nor does it provide all the capabilities of the - /// :class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use - /// :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize` - /// - /// Args: - /// sequence (:obj:`str`): - /// A string to pre-tokeize - /// - /// Returns: - /// :obj:`List[Tuple[str, Offsets]]`: - /// A list of tuple with the pre-tokenized parts and their offsets - #[pyo3(text_signature = "(self, sequence)")] - fn pre_tokenize_str(&self, s: &str) -> PyResult> { - let mut pretokenized = tk::tokenizer::PreTokenizedString::from(s); - - ToPyResult(self.pretok.pre_tokenize(&mut pretokenized)).into_py()?; - - Ok(pretokenized - .get_splits(tk::OffsetReferential::Original, tk::OffsetType::Char) - .into_iter() - .map(|(s, o, _)| (s.to_owned(), o)) - .collect()) - } - - fn __repr__(&self) -> PyResult { - crate::utils::serde_pyo3::repr(self) - .map_err(|e| exceptions::PyException::new_err(e.to_string())) - } - - fn __str__(&self) -> PyResult { - crate::utils::serde_pyo3::to_string(self) - .map_err(|e| exceptions::PyException::new_err(e.to_string())) - } -} - -macro_rules! getter { - ($self: ident, $variant: ident, $($name: tt)+) => {{ - let super_ = $self.as_ref(); - if let PyPreTokenizerTypeWrapper::Single(ref single) = super_.pretok { - if let PyPreTokenizerWrapper::Wrapped(PreTokenizerWrapper::$variant(ref pretok)) = - *single.read().expect("RwLock synchronisation primitive is poisoned, cannot get subtype of PyPreTokenizer") { - pretok.$($name)+ - } else { - unreachable!() - } - } else { - unreachable!() + PreTokenizerWrapper::Delimiter(_) => { + Bound::new(py, init.add_subclass(PyCharDelimiterSplit))?.into_super() } - }}; -} - -macro_rules! setter { - ($self: ident, $variant: ident, $name: ident, $value: expr) => {{ - let super_ = $self.as_ref(); - if let PyPreTokenizerTypeWrapper::Single(ref single) = super_.pretok { - if let PyPreTokenizerWrapper::Wrapped(PreTokenizerWrapper::$variant(ref mut pretok)) = - *single.write().expect("RwLock synchronisation primitive is poisoned, cannot get subtype of PyPreTokenizer") - { - pretok.$name = $value; - } + PreTokenizerWrapper::Whitespace(_) => { + Bound::new(py, init.add_subclass(PyWhitespace))?.into_super() } - }}; - ($self: ident, $variant: ident, @$name: ident, $value: expr) => {{ - let super_ = $self.as_ref(); - if let PyPreTokenizerTypeWrapper::Single(ref single) = super_.pretok { - if let PyPreTokenizerWrapper::Wrapped(PreTokenizerWrapper::$variant(ref mut pretok)) = - *single.write().expect("RwLock synchronisation primitive is poisoned, cannot get subtype of PyPreTokenizer") - { - pretok.$name($value); - } + PreTokenizerWrapper::WhitespaceSplit(_) => { + Bound::new(py, init.add_subclass(PyWhitespaceSplit))?.into_super() + } + PreTokenizerWrapper::Sequence(_) => { + Bound::new(py, init.add_subclass(PySequence))?.into_super() + } + PreTokenizerWrapper::Split(_) => Bound::new(py, init.add_subclass(PySplit))?.into_super(), + PreTokenizerWrapper::Punctuation(_) => { + Bound::new(py, init.add_subclass(PyPunctuation))?.into_super() } - }}; + PreTokenizerWrapper::Digits(_) => Bound::new(py, init.add_subclass(PyDigits))?.into_super(), + PreTokenizerWrapper::UnicodeScripts(_) => { + Bound::new(py, init.add_subclass(PyUnicodeScripts))?.into_super() + } + PreTokenizerWrapper::FixedLength(_) => { + Bound::new(py, init.add_subclass(PyFixedLength))?.into_super() + } + // Loadable from tokenizer.json but not constructible from Python (and + // rejected by the pipeline at compile time): exposed as the base class. + PreTokenizerWrapper::Metaspace(_) => Bound::new(py, init)?, + }; + Ok(obj.unbind()) } -/// ByteLevel PreTokenizer -/// -/// This pre-tokenizer takes care of replacing all bytes of the given string -/// with a corresponding representation, as well as splitting into words. -/// -/// Args: -/// add_prefix_space (:obj:`bool`, `optional`, defaults to :obj:`True`): -/// Whether to add a space to the first word if there isn't already one. This -/// lets us treat `hello` exactly like `say hello`. -/// use_regex (:obj:`bool`, `optional`, defaults to :obj:`True`): -/// Set this to :obj:`False` to prevent this `pre_tokenizer` from using -/// the GPT2 specific regexp for spliting on whitespace. -/// -/// Example:: -/// -/// >>> from tokenizers.pre_tokenizers import ByteLevel -/// >>> pre_tokenizer = ByteLevel() -/// >>> pre_tokenizer.pre_tokenize_str("Hello my friend, how is it going?") -/// [('ĠHello', (0, 5)), ('Ġmy', (5, 8)), ('Ġfriend,', (8, 15)), ('Ġhow', (15, 19)), ('Ġis', (19, 22)), ('Ġit', (22, 25)), ('Ġgoing?', (25, 32))] -/// -#[pyclass(extends=PyPreTokenizer, module = "tokenizers.pre_tokenizers", name = "ByteLevel")] -pub struct PyByteLevel {} -#[pymethods] -impl PyByteLevel { - #[getter] - fn get_add_prefix_space(self_: PyRef) -> bool { - getter!(self_, ByteLevel, add_prefix_space) - } - - #[setter] - fn set_add_prefix_space(self_: PyRef, add_prefix_space: bool) { - setter!(self_, ByteLevel, add_prefix_space, add_prefix_space); - } - - #[getter] - fn get_use_regex(self_: PyRef) -> bool { - getter!(self_, ByteLevel, use_regex) - } - - #[setter] - fn set_use_regex(self_: PyRef, use_regex: bool) { - setter!(self_, ByteLevel, use_regex, use_regex); - } - - #[getter] - fn get_trim_offsets(self_: PyRef) -> bool { - getter!(self_, ByteLevel, trim_offsets) - } +macro_rules! unit_pre_tokenizer { + ($pyname:ident, $name:literal, $inner:expr, $doc:literal) => { + #[doc = $doc] + #[pyclass(frozen, extends = PyPreTokenizer, name = $name, module = "tokenizers.pre_tokenizers")] + pub struct $pyname; - #[setter] - fn set_trim_offsets(self_: PyRef, trim_offsets: bool) { - setter!(self_, ByteLevel, trim_offsets, trim_offsets) - } - - #[new] - #[pyo3( - signature = (add_prefix_space = true, trim_offsets = true, use_regex = true, **_kwargs), - text_signature = "(self, add_prefix_space=True, trim_offsets=True, use_regex=True)" - )] - fn new( - add_prefix_space: bool, - trim_offsets: bool, - use_regex: bool, - _kwargs: Option<&Bound<'_, PyDict>>, - ) -> PyClassInitializer { - PyClassInitializer::::from(PyPreTokenizer::from( - ByteLevel::default() - .add_prefix_space(add_prefix_space) - .trim_offsets(trim_offsets) - .use_regex(use_regex), - )) - .add_subclass(PyByteLevel {}) - } - - /// Returns the alphabet used by this PreTokenizer. - /// - /// Since the ByteLevel works as its name suggests, at the byte level, it - /// encodes each byte value to a unique visible character. This means that there is a - /// total of 256 different characters composing this alphabet. - /// - /// Returns: - /// :obj:`List[str]`: A list of characters that compose the alphabet - #[staticmethod] - #[pyo3(text_signature = "()")] - fn alphabet() -> Vec { - ByteLevel::alphabet() - .into_iter() - .map(|c| c.to_string()) - .collect() - } + #[pymethods] + impl $pyname { + #[new] + fn new() -> PyClassInitializer { + PyClassInitializer::from(PyPreTokenizer { inner: $inner.into() }).add_subclass($pyname) + } + } + }; } -/// This pre-tokenizer splits on word boundaries according to the ``\w+|[^\w\s]+`` -/// regex pattern. It splits on word characters or characters that aren't words or -/// whitespaces (punctuation such as hyphens, apostrophes, commas, etc.). -/// -/// Example:: -/// -/// >>> from tokenizers.pre_tokenizers import Whitespace -/// >>> pre_tokenizer = Whitespace() -/// >>> pre_tokenizer.pre_tokenize_str("Hello, world! Let's tokenize.") -/// [('Hello', (0, 5)), (',', (5, 6)), ('world', (7, 12)), ('!', (12, 13)), ('Let', (14, 17)), ("'", (17, 18)), ('s', (18, 19)), ('tokenize', (20, 28)), ('.', (28, 29))] -/// -#[pyclass(extends=PyPreTokenizer, module = "tokenizers.pre_tokenizers", name = "Whitespace")] -pub struct PyWhitespace {} -#[pymethods] -impl PyWhitespace { - #[new] - #[pyo3(text_signature = "(self)")] - fn new() -> PyClassInitializer { - PyClassInitializer::::from(PyPreTokenizer::from(Whitespace {})) - .add_subclass(PyWhitespace {}) - } -} +unit_pre_tokenizer!( + PyWhitespace, + "Whitespace", + Whitespace, + "Splits into runs of letters/digits/underscore or runs of other symbols (the pattern `\\w+|[^\\w\\s]+`)." +); +unit_pre_tokenizer!( + PyWhitespaceSplit, + "WhitespaceSplit", + WhitespaceSplit, + "Splits on whitespace only." +); +unit_pre_tokenizer!( + PyBertPreTokenizer, + "BertPreTokenizer", + BertPreTokenizer, + "The BERT split: on whitespace, and each punctuation character becomes its own piece." +); +unit_pre_tokenizer!( + PyUnicodeScripts, + "UnicodeScripts", + UnicodeScripts, + "Splits where the script changes (Latin to Han, for example), so a piece never mixes alphabets." +); + +/// GPT-2 style byte-level splitting: cuts with the GPT-2 regex unless +/// `use_regex=False`. The pipeline does not support `add_prefix_space`, so it +/// is always off. +#[pyclass(frozen, extends = PyPreTokenizer, name = "ByteLevel", module = "tokenizers.pre_tokenizers")] +pub struct PyByteLevel; -/// This pre-tokenizer simply splits on whitespace. Works like :meth:`str.split` with no -/// arguments — it splits on any whitespace and discards the whitespace tokens. Unlike -/// :class:`~tokenizers.pre_tokenizers.Whitespace`, it does not split on punctuation. -/// -/// Example:: -/// -/// >>> from tokenizers.pre_tokenizers import WhitespaceSplit -/// >>> pre_tokenizer = WhitespaceSplit() -/// >>> pre_tokenizer.pre_tokenize_str("Hello, world! How are you?") -/// [('Hello,', (0, 6)), ('world!', (7, 13)), ('How', (14, 17)), ('are', (18, 21)), ('you?', (22, 26))] -/// -#[pyclass(extends=PyPreTokenizer, module = "tokenizers.pre_tokenizers", name = "WhitespaceSplit")] -pub struct PyWhitespaceSplit {} #[pymethods] -impl PyWhitespaceSplit { +impl PyByteLevel { #[new] - #[pyo3(text_signature = "(self)")] - fn new() -> PyClassInitializer { - PyClassInitializer::::from(PyPreTokenizer::from(WhitespaceSplit)) - .add_subclass(PyWhitespaceSplit {}) + #[pyo3(signature = (*, use_regex = true))] + fn new(use_regex: bool) -> PyClassInitializer { + PyClassInitializer::from(PyPreTokenizer { + inner: ByteLevel::new(false, true, use_regex).into(), + }) + .add_subclass(PyByteLevel) } } -/// Split PreTokenizer -/// -/// This versatile pre-tokenizer splits using the provided pattern and -/// according to the provided behavior. The pattern can be inverted by -/// making use of the invert flag. -/// -/// Args: -/// pattern (:obj:`str` or :class:`~tokenizers.Regex`): -/// A pattern used to split the string. Usually a string or a regex built with `tokenizers.Regex`. -/// If you want to use a regex pattern, it has to be wrapped around a `tokenizers.Regex`, -/// otherwise we consider is as a string pattern. For example `pattern="|"` -/// means you want to split on `|` (imagine a csv file for example), while -/// `pattern=tokenizers.Regex("1|2")` means you split on either '1' or '2'. -/// behavior (:class:`~tokenizers.SplitDelimiterBehavior`): -/// The behavior to use when splitting. -/// Choices: "removed", "isolated", "merged_with_previous", "merged_with_next", -/// "contiguous" -/// -/// invert (:obj:`bool`, `optional`, defaults to :obj:`False`): -/// Whether to invert the pattern. -/// -/// Example:: -/// -/// >>> from tokenizers import Regex -/// >>> from tokenizers.pre_tokenizers import Split -/// >>> # Split on commas, removing them -/// >>> pre_tokenizer = Split(",", behavior="removed") -/// >>> pre_tokenizer.pre_tokenize_str("one,two,three") -/// [('one', (0, 3)), ('two', (4, 7)), ('three', (8, 13))] -/// >>> # Split using a regex, keeping the delimiter isolated -/// >>> Split(Regex(r"\s+"), behavior="isolated").pre_tokenize_str("hello world") -/// [('hello', (0, 5)), (' ', (5, 8)), ('world', (8, 13))] -/// -#[pyclass(extends=PyPreTokenizer, module = "tokenizers.pre_tokenizers", name = "Split")] -pub struct PySplit {} -#[pymethods] -impl PySplit { - #[new] - #[pyo3(signature = (pattern, behavior, invert = false), text_signature = "(self, pattern, behavior, invert=False)")] - fn new( - pattern: PyPattern, - behavior: PySplitDelimiterBehavior, - invert: bool, - ) -> PyResult> { - Ok( - PyClassInitializer::::from(PyPreTokenizer::from( - ToPyResult(Split::new(pattern, behavior.into(), invert)).into_py()?, - )) - .add_subclass(PySplit {}), - ) - } - - fn __getnewargs__<'p>(&self, py: Python<'p>) -> PyResult> { - PyTuple::new(py, [" ", "removed"]) - } - - #[getter] - fn get_pattern(_self: PyRef) -> PyResult<()> { - Err(PyException::new_err("Cannot get pattern")) - } - - #[setter] - fn set_pattern(_self: PyRef, _pattern: PyPattern) -> PyResult<()> { - Err(PyException::new_err( - "Cannot set pattern, please instantiate a new split pattern instead", - )) - } - - #[getter] - fn get_behavior(self_: PyRef) -> String { - getter!(self_, Split, behavior).to_string().to_lowercase() - } - - #[setter] - fn set_behavior(self_: PyRef, behavior: String) -> PyResult<()> { - let behavior = match behavior.as_ref() { - "removed" => SplitDelimiterBehavior::Removed, - "isolated" => SplitDelimiterBehavior::Isolated, - "merged_with_previous" => SplitDelimiterBehavior::MergedWithPrevious, - "merged_with_next" => SplitDelimiterBehavior::MergedWithNext, - "contiguous" => SplitDelimiterBehavior::Contiguous, - _ => { - return Err(exceptions::PyValueError::new_err( - "Wrong value for SplitDelimiterBehavior, expected one of: \ - `removed, isolated, merged_with_previous, merged_with_next, contiguous`", - )); - } - }; - setter!(self_, Split, behavior, behavior); - Ok(()) - } - - #[getter] - fn get_invert(self_: PyRef) -> bool { - getter!(self_, Split, invert) - } - - #[setter] - fn set_invert(self_: PyRef, invert: bool) { - setter!(self_, Split, invert, invert) - } -} +/// Splits on one fixed character, dropping it. +#[pyclass(frozen, extends = PyPreTokenizer, name = "CharDelimiterSplit", module = "tokenizers.pre_tokenizers")] +pub struct PyCharDelimiterSplit; -/// This pre-tokenizer simply splits on the provided char. Works like :meth:`str.split` -/// with a single-character delimiter. -/// -/// Args: -/// delimiter (:obj:`str`): -/// The single character that will be used to split the input. The delimiter -/// is removed from the output. -/// -/// Example:: -/// -/// >>> from tokenizers.pre_tokenizers import CharDelimiterSplit -/// >>> pre_tokenizer = CharDelimiterSplit("x") -/// >>> pre_tokenizer.pre_tokenize_str("helloxthere") -/// [('hello', (0, 5)), ('there', (6, 11))] -/// -#[pyclass(extends=PyPreTokenizer, module = "tokenizers.pre_tokenizers", name = "CharDelimiterSplit")] -pub struct PyCharDelimiterSplit {} #[pymethods] impl PyCharDelimiterSplit { - #[getter] - fn get_delimiter(self_: PyRef) -> String { - getter!(self_, Delimiter, delimiter.to_string()) - } - - #[setter] - fn set_delimiter(self_: PyRef, delimiter: char) { - setter!(self_, Delimiter, delimiter, delimiter); - } - #[new] - #[pyo3(signature = (delimiter), text_signature = "(self, delimiter)")] - pub fn new(delimiter: char) -> PyResult> { - Ok( - PyClassInitializer::::from(PyPreTokenizer::from( - CharDelimiterSplit::new(delimiter), - )) - .add_subclass(PyCharDelimiterSplit {}), - ) - } - - fn __getnewargs__<'p>(&self, py: Python<'p>) -> PyResult> { - PyTuple::new(py, [" "]) + fn new(delimiter: char) -> PyClassInitializer { + PyClassInitializer::from(PyPreTokenizer { + inner: CharDelimiterSplit::new(delimiter).into(), + }) + .add_subclass(PyCharDelimiterSplit) } } -/// BertPreTokenizer -/// -/// This pre-tokenizer splits tokens on whitespace and punctuation. Each occurrence of -/// a punctuation character will be treated as a separate token. This is the pre-tokenizer -/// used by the original BERT model. -/// -/// Example:: -/// -/// >>> from tokenizers.pre_tokenizers import BertPreTokenizer -/// >>> pre_tokenizer = BertPreTokenizer() -/// >>> pre_tokenizer.pre_tokenize_str("Hello, I'm a single sentence!") -/// [('Hello', (0, 5)), (',', (5, 6)), ('I', (7, 8)), ("'", (8, 9)), ('m', (9, 10)), ('a', (11, 12)), ('single', (13, 19)), ('sentence', (20, 28)), ('!', (28, 29))] -/// -#[pyclass(extends=PyPreTokenizer, module = "tokenizers.pre_tokenizers", name = "BertPreTokenizer")] -pub struct PyBertPreTokenizer {} +/// Separates digits from everything else. With `individual_digits=True`, +/// every digit becomes its own piece. +#[pyclass(frozen, extends = PyPreTokenizer, name = "Digits", module = "tokenizers.pre_tokenizers")] +pub struct PyDigits; + #[pymethods] -impl PyBertPreTokenizer { +impl PyDigits { #[new] - #[pyo3(text_signature = "(self)")] - fn new() -> PyClassInitializer { - PyClassInitializer::::from(PyPreTokenizer::from(BertPreTokenizer)) - .add_subclass(PyBertPreTokenizer {}) + #[pyo3(signature = (*, individual_digits = false))] + fn new(individual_digits: bool) -> PyClassInitializer { + PyClassInitializer::from(PyPreTokenizer { + inner: Digits::new(individual_digits).into(), + }) + .add_subclass(PyDigits) } } -/// This pre-tokenizer simply splits on punctuation as individual characters. -/// -/// Args: -/// behavior (:class:`~tokenizers.SplitDelimiterBehavior`): -/// The behavior to use when splitting. -/// Choices: "removed", "isolated" (default), "merged_with_previous", "merged_with_next", -/// "contiguous" -/// -/// Example:: -/// -/// >>> from tokenizers.pre_tokenizers import Punctuation -/// >>> pre_tokenizer = Punctuation() -/// >>> pre_tokenizer.pre_tokenize_str("Hello, how are you?") -/// [('Hello', (0, 5)), (',', (5, 6)), ('how', (7, 10)), ('are', (11, 14)), ('you', (15, 18)), ('?', (18, 19))] -/// -#[pyclass(extends=PyPreTokenizer, module = "tokenizers.pre_tokenizers", name = "Punctuation")] -pub struct PyPunctuation {} +/// Cuts the text into pieces of exactly `length` characters (the last one may +/// be shorter). +#[pyclass(frozen, extends = PyPreTokenizer, name = "FixedLength", module = "tokenizers.pre_tokenizers")] +pub struct PyFixedLength; + #[pymethods] -impl PyPunctuation { +impl PyFixedLength { #[new] - #[pyo3( signature = (behavior = PySplitDelimiterBehavior(SplitDelimiterBehavior::Isolated)), text_signature = "(self, behavior=\"isolated\")")] - fn new(behavior: PySplitDelimiterBehavior) -> PyClassInitializer { - PyClassInitializer::::from(PyPreTokenizer::from(Punctuation::new( - behavior.into(), - ))) - .add_subclass(PyPunctuation {}) - } - - #[getter] - fn get_behavior(self_: PyRef) -> String { - getter!(self_, Punctuation, behavior) - .to_string() - .to_lowercase() - } - - #[setter] - fn set_behavior(self_: PyRef, behavior: String) -> PyResult<()> { - let behavior = match behavior.as_ref() { - "removed" => SplitDelimiterBehavior::Removed, - "isolated" => SplitDelimiterBehavior::Isolated, - "merged_with_previous" => SplitDelimiterBehavior::MergedWithPrevious, - "merged_with_next" => SplitDelimiterBehavior::MergedWithNext, - "contiguous" => SplitDelimiterBehavior::Contiguous, - _ => { - return Err(exceptions::PyValueError::new_err( - "Wrong value for SplitDelimiterBehavior, expected one of: \ - `removed, isolated, merged_with_previous, merged_with_next, contiguous`", - )); - } - }; - setter!(self_, Punctuation, behavior, behavior); - Ok(()) + #[pyo3(signature = (*, length = 5))] + fn new(length: usize) -> PyClassInitializer { + PyClassInitializer::from(PyPreTokenizer { + inner: FixedLength::new(length).into(), + }) + .add_subclass(PyFixedLength) } } -/// This pre-tokenizer composes other pre-tokenizers and applies them in sequence. -/// Each pre-tokenizer in the list is applied to the output of the previous one, -/// allowing complex tokenization strategies to be built by chaining simpler components. -/// -/// Args: -/// pretokenizers (:obj:`List[PreTokenizer]`): -/// A list of :class:`~tokenizers.pre_tokenizers.PreTokenizer` to be applied -/// in sequence. -/// -/// Example:: -/// -/// >>> from tokenizers.pre_tokenizers import Punctuation, Whitespace, Sequence -/// >>> pre_tokenizer = Sequence([Whitespace(), Punctuation()]) -/// >>> pre_tokenizer.pre_tokenize_str("Hello, world!") -/// [('Hello', (0, 5)), (',', (5, 6)), ('world', (7, 12)), ('!', (12, 13))] -/// -#[pyclass(extends=PyPreTokenizer, module = "tokenizers.pre_tokenizers", name = "Sequence")] -pub struct PySequence {} +/// Splits on punctuation. `behavior` says what happens to the punctuation +/// itself — see `Split` for the options. +#[pyclass(frozen, extends = PyPreTokenizer, name = "Punctuation", module = "tokenizers.pre_tokenizers")] +pub struct PyPunctuation; + #[pymethods] -impl PySequence { +impl PyPunctuation { #[new] - #[pyo3(text_signature = "(self, pretokenizers)")] - fn new(pre_tokenizers: &Bound<'_, PyList>) -> PyResult> { - let mut sequence = Vec::with_capacity(pre_tokenizers.len()); - for n in pre_tokenizers.iter() { - let pretokenizer: PyRef = n.extract()?; - match &pretokenizer.pretok { - PyPreTokenizerTypeWrapper::Sequence(inner) => { - sequence.extend(inner.iter().cloned()) - } - PyPreTokenizerTypeWrapper::Single(inner) => sequence.push(inner.clone()), - } - } - Ok( - PyClassInitializer::::from(PyPreTokenizer::new( - PyPreTokenizerTypeWrapper::Sequence(sequence), - )) - .add_subclass(PySequence {}), - ) - } - - fn __getnewargs__<'p>(&self, py: Python<'p>) -> PyResult> { - PyTuple::new(py, [PyList::empty(py)]) - } - - fn __getitem__(self_: PyRef<'_, Self>, py: Python<'_>, index: usize) -> PyResult> { - match &self_.as_ref().pretok { - PyPreTokenizerTypeWrapper::Sequence(inner) => match inner.get(index) { - Some(item) => PyPreTokenizer::new(PyPreTokenizerTypeWrapper::Single(item.clone())) - .get_as_subtype(py), - _ => Err(PyErr::new::( - "Index not found", - )), - }, - _ => Err(PyErr::new::( - "This processor is not a Sequence, it does not support __getitem__", - )), - } - } - - fn __setitem__(self_: PyRef<'_, Self>, index: usize, value: Bound<'_, PyAny>) -> PyResult<()> { - let pretok: PyPreTokenizer = value.extract()?; - let PyPreTokenizerTypeWrapper::Single(pretok) = pretok.pretok else { - return Err(PyException::new_err( - "pre tokenizer should not be a sequence", - )); - }; - match &self_.as_ref().pretok { - PyPreTokenizerTypeWrapper::Sequence(inner) => match inner.get(index) { - Some(item) => { - *item - .write() - .map_err(|_| PyException::new_err("RwLock synchronisation primitive is poisoned, cannot get subtype of PyPreTokenizer"))? = (*pretok - .read() - .map_err(|_| PyException::new_err("RwLock synchronisation primitive is poisoned, cannot get subtype of PyPreTokenizer"))?) - .clone(); - } - _ => { - return Err(PyErr::new::( - "Index not found", - )); - } - }, - PyPreTokenizerTypeWrapper::Single(_) => { - return Err(PyException::new_err("pre tokenizer is not a sequence")); - } - }; - Ok(()) + #[pyo3(signature = (behavior = String::from("isolated")))] + fn new(behavior: String) -> PyResult> { + Ok(PyClassInitializer::from(PyPreTokenizer { + inner: Punctuation::new(parse_behavior(&behavior)?).into(), + }) + .add_subclass(PyPunctuation)) } } -pub(crate) fn from_string(string: String) -> Result { - let scheme = match string.as_str() { - "first" => PrependScheme::First, - "never" => PrependScheme::Never, - "always" => PrependScheme::Always, - _ => { - return Err(exceptions::PyValueError::new_err(format!( - "{string} is an unknown variant, should be one of ['first', 'never', 'always']" - ))); - } - }; - Ok(scheme) -} +/// Splits on a pattern: a literal string, or a regular expression with +/// `regex=True`. `behavior` says what to do with each match — "removed" drops +/// it, "isolated" keeps it as its own piece, "merged_with_previous" / +/// "merged_with_next" glue it to a neighbor, "contiguous" merges runs of +/// matches. `invert=True` keeps the matches and splits everything else. +#[pyclass(frozen, extends = PyPreTokenizer, name = "Split", module = "tokenizers.pre_tokenizers")] +pub struct PySplit; -/// Metaspace pre-tokenizer -/// -/// This pre-tokenizer replaces any whitespace by the provided replacement character. -/// It then tries to split on these spaces. -/// -/// Args: -/// replacement (:obj:`str`, `optional`, defaults to :obj:`▁`): -/// The replacement character. Must be exactly one character. By default we -/// use the `▁` (U+2581) meta symbol (Same as in SentencePiece). -/// -/// prepend_scheme (:obj:`str`, `optional`, defaults to :obj:`"always"`): -/// Whether to add a space to the first word if there isn't already one. This -/// lets us treat `hello` exactly like `say hello`. -/// Choices: "always", "never", "first". First means the space is only added on the first -/// token (relevant when special tokens are used or other pre_tokenizer are used). -/// -/// Example:: -/// -/// >>> from tokenizers.pre_tokenizers import Metaspace -/// >>> pre_tokenizer = Metaspace() -/// >>> pre_tokenizer.pre_tokenize_str("Hello my friend") -/// [('▁Hello', (0, 5)), ('▁my', (6, 8)), ('▁friend', (9, 15))] -/// -#[pyclass(extends=PyPreTokenizer, module = "tokenizers.pre_tokenizers", name = "Metaspace")] -pub struct PyMetaspace {} #[pymethods] -impl PyMetaspace { - #[getter] - fn get_replacement(self_: PyRef) -> String { - getter!(self_, Metaspace, get_replacement().to_string()) - } - - #[setter] - fn set_replacement(self_: PyRef, replacement: char) { - setter!(self_, Metaspace, @set_replacement, replacement); - } - - #[getter] - fn get_split(self_: PyRef) -> bool { - getter!(self_, Metaspace, get_split()) - } - - #[setter] - fn set_split(self_: PyRef, split: bool) { - setter!(self_, Metaspace, @set_split, split); - } - - #[getter] - fn get_prepend_scheme(self_: PyRef) -> String { - // Assuming Metaspace has a method to get the prepend_scheme as a string - getter!(self_, Metaspace, get_prepend_scheme()).to_string() - } - - #[setter] - fn set_prepend_scheme(self_: PyRef, prepend_scheme: String) -> PyResult<()> { - let scheme = from_string(prepend_scheme)?; - setter!(self_, Metaspace, @set_prepend_scheme, scheme); - Ok(()) - } - +impl PySplit { #[new] - #[pyo3(signature = (replacement = '▁', prepend_scheme=String::from("always"), split=true), text_signature = "(self, replacement=\"▁\", prepend_scheme=\"always\", split=True)")] + #[pyo3(signature = (pattern, behavior = String::from("isolated"), *, invert = false, regex = false))] fn new( - replacement: char, - prepend_scheme: String, - split: bool, + pattern: &str, + behavior: String, + invert: bool, + regex: bool, ) -> PyResult> { - // Create a new Metaspace instance - let prepend_scheme = from_string(prepend_scheme)?; - let new_instance: Metaspace = Metaspace::new(replacement, prepend_scheme, split); - Ok( - PyClassInitializer::::from(PyPreTokenizer::from(new_instance)) - .add_subclass(PyMetaspace {}), - ) - } -} - -/// This pre-tokenizer simply splits using the digits in separate tokens -/// -/// Args: -/// individual_digits (:obj:`bool`, `optional`, defaults to :obj:`False`): -/// If set to True, digits will each be separated as follows:: -/// -/// "Call 123 please" -> "Call ", "1", "2", "3", " please" -/// -/// If set to False, digits will grouped as follows:: -/// -/// "Call 123 please" -> "Call ", "123", " please" -#[pyclass(extends=PyPreTokenizer, module = "tokenizers.pre_tokenizers", name = "Digits")] -pub struct PyDigits {} -#[pymethods] -impl PyDigits { - #[getter] - fn get_individual_digits(self_: PyRef) -> bool { - getter!(self_, Digits, individual_digits) - } - - #[setter] - fn set_individual_digits(self_: PyRef, individual_digits: bool) { - setter!(self_, Digits, individual_digits, individual_digits); - } - - #[new] - #[pyo3(signature = (individual_digits = false), text_signature = "(self, individual_digits=False)")] - fn new(individual_digits: bool) -> PyClassInitializer { - PyClassInitializer::::from(PyPreTokenizer::from(Digits::new( - individual_digits, - ))) - .add_subclass(PyDigits {}) + let pattern = if regex { + SplitPattern::Regex(pattern.to_owned()) + } else { + SplitPattern::String(pattern.to_owned()) + }; + let split = Split::new(pattern, parse_behavior(&behavior)?, invert).map_err(to_pyerr)?; + Ok(PyClassInitializer::from(PyPreTokenizer { + inner: split.into(), + }) + .add_subclass(PySplit)) } } -/// This pre-tokenizer splits the text into fixed length chunks as used -/// [here](https://www.biorxiv.org/content/10.1101/2023.01.11.523679v1.full) -/// -/// Args: -/// length (:obj:`int`, `optional`, defaults to :obj:`5`): -/// The length of the chunks to split the text into. -/// -/// Strings are split on the character level rather than the byte level to avoid -/// splitting unicode characters consisting of multiple bytes. -/// -/// Example:: -/// -/// >>> from tokenizers.pre_tokenizers import FixedLength -/// >>> pre_tokenizer = FixedLength(length=3) -/// >>> pre_tokenizer.pre_tokenize_str("Hello") -/// [('Hel', (0, 3)), ('lo', (3, 5))] -/// -#[pyclass(extends=PyPreTokenizer, module = "tokenizers.pre_tokenizers", name = "FixedLength")] -pub struct PyFixedLength {} -#[pymethods] -impl PyFixedLength { - #[getter] - fn get_length(self_: PyRef) -> usize { - getter!(self_, FixedLength, length) - } +/// Runs several pre-tokenizers in order, each one further splitting the +/// pieces left by the previous. +#[pyclass(frozen, extends = PyPreTokenizer, name = "Sequence", module = "tokenizers.pre_tokenizers")] +pub struct PySequence; - #[setter] - fn set_length(self_: PyRef, length: usize) { - setter!(self_, FixedLength, length, length); - } - - #[new] - #[pyo3(signature = (length = 5), text_signature = "(self, length=5)")] - fn new(length: usize) -> PyClassInitializer { - PyClassInitializer::::from(PyPreTokenizer::from(FixedLength::new(length))) - .add_subclass(PyFixedLength {}) - } -} - -/// This pre-tokenizer splits on characters that belong to different language families. -/// It roughly follows the SentencePiece script boundaries, with Hiragana and Katakana -/// fused into the Han script category. This mimics the SentencePiece Unigram -/// implementation and is useful for multilingual models that need to handle CJK text. -/// -/// Example:: -/// -/// >>> from tokenizers.pre_tokenizers import UnicodeScripts -/// >>> pre_tokenizer = UnicodeScripts() -/// >>> pre_tokenizer.pre_tokenize_str("どこ Where") -/// [('どこ', (0, 2)), ('Where', (3, 8))] -/// -#[pyclass(extends=PyPreTokenizer, module = "tokenizers.pre_tokenizers", name = "UnicodeScripts")] -pub struct PyUnicodeScripts {} #[pymethods] -impl PyUnicodeScripts { +impl PySequence { #[new] - #[pyo3(text_signature = "(self)")] - fn new() -> PyClassInitializer { - PyClassInitializer::::from(PyPreTokenizer::from(UnicodeScripts::new())) - .add_subclass(PyUnicodeScripts {}) - } -} - -#[derive(Clone)] -pub(crate) struct CustomPreTokenizer { - inner: Py, -} - -impl CustomPreTokenizer { - pub fn new(inner: Py) -> Self { - Self { inner } - } -} - -impl tk::tokenizer::PreTokenizer for CustomPreTokenizer { - fn pre_tokenize(&self, sentence: &mut PreTokenizedString) -> tk::Result<()> { - Python::attach(|py| { - let pretok = PyPreTokenizedStringRefMut::new(sentence); - let py_pretok = self.inner.bind(py); - py_pretok.call_method("pre_tokenize", (pretok.get().clone(),), None)?; - Ok(()) + fn new(pre_tokenizers: Vec>) -> PyClassInitializer { + let inner: Vec = + pre_tokenizers.iter().map(|p| p.inner.clone()).collect(); + PyClassInitializer::from(PyPreTokenizer { + inner: Sequence::new(inner).into(), }) + .add_subclass(PySequence) } } -impl Serialize for CustomPreTokenizer { - fn serialize(&self, _serializer: S) -> Result - where - S: Serializer, - { - Err(serde::ser::Error::custom( - "Custom PreTokenizer cannot be serialized", - )) - } -} - -impl<'de> Deserialize<'de> for CustomPreTokenizer { - fn deserialize(_deserializer: D) -> Result - where - D: Deserializer<'de>, - { - Err(serde::de::Error::custom( - "Custom PreTokenizer cannot be deserialized", - )) - } -} - -#[derive(Clone, Deserialize)] -#[serde(untagged)] -pub(crate) enum PyPreTokenizerWrapper { - Custom(CustomPreTokenizer), - Wrapped(PreTokenizerWrapper), -} - -impl Serialize for PyPreTokenizerWrapper { - fn serialize(&self, serializer: S) -> Result<::Ok, ::Error> - where - S: Serializer, - { - match self { - PyPreTokenizerWrapper::Wrapped(inner) => inner.serialize(serializer), - PyPreTokenizerWrapper::Custom(inner) => inner.serialize(serializer), - } - } -} - -#[derive(Clone)] -pub(crate) enum PyPreTokenizerTypeWrapper { - Sequence(Vec>>), - Single(Arc>), -} - -impl<'de> Deserialize<'de> for PyPreTokenizerTypeWrapper { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let wrapper = PreTokenizerWrapper::deserialize(deserializer)?; - let py_wrapper: PyPreTokenizerWrapper = wrapper.into(); - Ok(py_wrapper.into()) - } -} - -impl Serialize for PyPreTokenizerTypeWrapper { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - match self { - PyPreTokenizerTypeWrapper::Sequence(seq) => { - let mut ser = serializer.serialize_struct("Sequence", 2)?; - ser.serialize_field("type", "Sequence")?; - ser.serialize_field("pretokenizers", seq)?; - ser.end() - } - PyPreTokenizerTypeWrapper::Single(inner) => inner.serialize(serializer), - } - } -} - -impl From for PyPreTokenizerWrapper -where - I: Into, -{ - fn from(pretok: I) -> Self { - PyPreTokenizerWrapper::Wrapped(pretok.into()) - } -} - -impl From for PyPreTokenizerTypeWrapper -where - I: Into, -{ - fn from(pretok: I) -> Self { - let pretok = pretok.into(); - match pretok { - PyPreTokenizerWrapper::Wrapped(PreTokenizerWrapper::Sequence(seq)) => { - PyPreTokenizerTypeWrapper::Sequence( - seq.into_iter() - .map(|e| Arc::new(RwLock::new(PyPreTokenizerWrapper::Wrapped(e.clone())))) - .collect(), - ) - } - _ => PyPreTokenizerTypeWrapper::Single(Arc::new(RwLock::new(pretok))), - } - } -} - -impl From for PyPreTokenizer -where - I: Into, -{ - fn from(pretok: I) -> Self { - PyPreTokenizer { - pretok: pretok.into().into(), - } - } -} - -impl PreTokenizer for PyPreTokenizerTypeWrapper { - fn pre_tokenize(&self, pretok: &mut PreTokenizedString) -> tk::Result<()> { - match self { - PyPreTokenizerTypeWrapper::Single(inner) => inner - .read() - .map_err(|_| PyException::new_err("RwLock synchronisation primitive is poisoned, cannot get subtype of PyPreTokenizer"))? - .pre_tokenize(pretok), - PyPreTokenizerTypeWrapper::Sequence(inner) => inner.iter().try_for_each(|n| { - n.read() - .map_err(|_| PyException::new_err("RwLock synchronisation primitive is poisoned, cannot get subtype of PyPreTokenizer"))? - .pre_tokenize(pretok) - }), - } - } -} - -impl PreTokenizer for PyPreTokenizerWrapper { - fn pre_tokenize(&self, pretok: &mut PreTokenizedString) -> tk::Result<()> { - match self { - PyPreTokenizerWrapper::Wrapped(inner) => inner.pre_tokenize(pretok), - PyPreTokenizerWrapper::Custom(inner) => inner.pre_tokenize(pretok), - } - } -} - -/// PreTokenizers Module +/// How text is cut into pieces before the model runs. #[pymodule(gil_used = false)] pub mod pre_tokenizers { #[pymodule_export] - pub use super::PyBertPreTokenizer; - #[pymodule_export] - pub use super::PyByteLevel; - #[pymodule_export] - pub use super::PyCharDelimiterSplit; - #[pymodule_export] - pub use super::PyDigits; - #[pymodule_export] - pub use super::PyFixedLength; - #[pymodule_export] - pub use super::PyMetaspace; - #[pymodule_export] - pub use super::PyPreTokenizer; - #[pymodule_export] - pub use super::PyPunctuation; - #[pymodule_export] - pub use super::PySequence; - #[pymodule_export] - pub use super::PySplit; - #[pymodule_export] - pub use super::PyUnicodeScripts; - #[pymodule_export] - pub use super::PyWhitespace; - #[pymodule_export] - pub use super::PyWhitespaceSplit; -} - -#[cfg(test)] -mod test { - use pyo3::prelude::*; - use tk::pre_tokenizers::PreTokenizerWrapper; - use tk::pre_tokenizers::sequence::Sequence; - use tk::pre_tokenizers::whitespace::{Whitespace, WhitespaceSplit}; - - use crate::pre_tokenizers::{ - CustomPreTokenizer, PyPreTokenizer, PyPreTokenizerTypeWrapper, PyPreTokenizerWrapper, + pub use super::{ + PyBertPreTokenizer, PyByteLevel, PyCharDelimiterSplit, PyDigits, PyFixedLength, + PyPreTokenizer, PyPunctuation, PySequence, PySplit, PyUnicodeScripts, PyWhitespace, + PyWhitespaceSplit, }; - - #[test] - fn get_subtype() { - Python::attach(|py| { - let py_norm = PyPreTokenizer::new(Whitespace {}.into()); - let py_wsp = py_norm.get_as_subtype(py).unwrap(); - assert_eq!("Whitespace", py_wsp.bind(py).get_type().qualname().unwrap()); - }) - } - - #[test] - fn serialize() { - let py_wrapped: PyPreTokenizerWrapper = Whitespace {}.into(); - let py_ser = serde_json::to_string(&py_wrapped).unwrap(); - let rs_wrapped = PreTokenizerWrapper::Whitespace(Whitespace {}); - let rs_ser = serde_json::to_string(&rs_wrapped).unwrap(); - assert_eq!(py_ser, rs_ser); - let py_pretok: PyPreTokenizer = serde_json::from_str(&rs_ser).unwrap(); - match py_pretok.pretok { - PyPreTokenizerTypeWrapper::Single(inner) => match *inner.as_ref().read().unwrap() { - PyPreTokenizerWrapper::Wrapped(PreTokenizerWrapper::Whitespace(_)) => {} - _ => panic!("Expected Whitespace"), - }, - _ => panic!("Expected wrapped, not custom."), - } - - let py_seq: PyPreTokenizerWrapper = - Sequence::new(vec![Whitespace {}.into(), WhitespaceSplit.into()]).into(); - let py_wrapper_ser = serde_json::to_string(&py_seq).unwrap(); - let rs_wrapped = PreTokenizerWrapper::Sequence(Sequence::new(vec![ - Whitespace {}.into(), - WhitespaceSplit.into(), - ])); - let rs_ser = serde_json::to_string(&rs_wrapped).unwrap(); - assert_eq!(py_wrapper_ser, rs_ser); - - let py_seq = PyPreTokenizer::new(py_seq.into()); - let py_ser = serde_json::to_string(&py_seq).unwrap(); - assert_eq!(py_wrapper_ser, py_ser); - - let obj = Python::attach(|py| { - let py_wsp = PyPreTokenizer::new(Whitespace {}.into()); - Py::new(py, py_wsp).unwrap().into_any() - }); - let py_seq: PyPreTokenizerWrapper = - PyPreTokenizerWrapper::Custom(CustomPreTokenizer::new(obj)); - assert!(serde_json::to_string(&py_seq).is_err()); - } } diff --git a/bindings/python/src/processors.rs b/bindings/python/src/processors.rs deleted file mode 100644 index 6f82a1038..000000000 --- a/bindings/python/src/processors.rs +++ /dev/null @@ -1,932 +0,0 @@ -use std::convert::TryInto; -use std::sync::Arc; -use std::sync::RwLock; - -use crate::encoding::PyEncoding; -use crate::error::ToPyResult; -use pyo3::IntoPyObjectExt; -use pyo3::exceptions; -use pyo3::exceptions::PyException; -use pyo3::prelude::*; -use pyo3::types::*; -use serde::Deserializer; -use serde::Serializer; -use serde::ser::SerializeStruct; -use serde::{Deserialize, Serialize}; -use tk::processors::PostProcessorWrapper; -use tk::processors::bert::BertProcessing; -use tk::processors::byte_level::ByteLevel; -use tk::processors::roberta::RobertaProcessing; -use tk::processors::template::{SpecialToken, Template}; -use tk::{Encoding, PostProcessor}; -use tokenizers as tk; - -/// Base class for all post-processors -/// -/// This class is not supposed to be instantiated directly. Instead, any implementation of -/// a PostProcessor will return an instance of this class when instantiated. -#[pyclass( - dict, - module = "tokenizers.processors", - name = "PostProcessor", - subclass, - from_py_object -)] -#[derive(Clone, Deserialize, Serialize)] -#[serde(transparent)] -pub struct PyPostProcessor { - processor: PyPostProcessorTypeWrapper, -} - -impl From for PyPostProcessor -where - I: Into, -{ - fn from(processor: I) -> Self { - PyPostProcessor { - processor: processor.into().into(), - } - } -} - -impl PyPostProcessor { - pub(crate) fn new(processor: PyPostProcessorTypeWrapper) -> Self { - PyPostProcessor { processor } - } - - pub(crate) fn get_as_subtype(&self, py: Python<'_>) -> PyResult> { - let base = self.clone(); - Ok( - match &self.processor { - PyPostProcessorTypeWrapper::Sequence(_) => Py::new(py, (PySequence {}, base))?.into_any(), - PyPostProcessorTypeWrapper::Single(inner) => { - - match &*inner.read().map_err(|_| { - PyException::new_err("RwLock synchronisation primitive is poisoned, cannot get subtype of PyPostProcessor") - })? { - PostProcessorWrapper::ByteLevel(_) => Py::new(py, (PyByteLevel {}, base))? - .into_any(), - PostProcessorWrapper::Bert(_) => Py::new(py, (PyBertProcessing {}, base))? - .into_any(), - PostProcessorWrapper::Roberta(_) => Py::new(py, (PyRobertaProcessing {}, base))? - .into_any(), - PostProcessorWrapper::Template(_) => Py::new(py, (PyTemplateProcessing {}, base))? - .into_any(), - PostProcessorWrapper::Sequence(_) => Py::new(py, (PySequence {}, base))? - .into_any(), - } - } - } - ) - } -} - -impl PostProcessor for PyPostProcessor { - // TODO: update signature to `tk::Result` - fn added_tokens(&self, is_pair: bool) -> usize { - self.processor.added_tokens(is_pair) - } - - fn process_encodings( - &self, - encodings: Vec, - add_special_tokens: bool, - ) -> tk::Result> { - self.processor - .process_encodings(encodings, add_special_tokens) - } -} - -#[pymethods] -impl PyPostProcessor { - fn __getstate__(&self, py: Python) -> PyResult> { - let data = serde_json::to_string(&self.processor).map_err(|e| { - exceptions::PyException::new_err(format!( - "Error while attempting to pickle PostProcessor: {e}" - )) - })?; - Ok(PyBytes::new(py, data.as_bytes()).into()) - } - - fn __setstate__(&mut self, py: Python, state: Py) -> PyResult<()> { - match state.extract::<&[u8]>(py) { - Ok(s) => { - self.processor = serde_json::from_slice(s).map_err(|e| { - exceptions::PyException::new_err(format!( - "Error while attempting to unpickle PostProcessor: {e}" - )) - })?; - Ok(()) - } - Err(e) => Err(e.into()), - } - } - - /// Return the number of special tokens that would be added for single/pair sentences. - /// - /// Args: - /// is_pair (:obj:`bool`): - /// Whether the input would be a pair of sequences - /// - /// Returns: - /// :obj:`int`: The number of tokens to add - #[pyo3(text_signature = "(self, is_pair)")] - fn num_special_tokens_to_add(&self, is_pair: bool) -> PyResult { - Ok(self.processor.added_tokens(is_pair)) - } - - /// Post-process the given encodings, generating the final one - /// - /// Args: - /// encoding (:class:`~tokenizers.Encoding`): - /// The encoding for the first sequence - /// - /// pair (:class:`~tokenizers.Encoding`, `optional`): - /// The encoding for the pair sequence - /// - /// add_special_tokens (:obj:`bool`): - /// Whether to add the special tokens - /// - /// Return: - /// :class:`~tokenizers.Encoding`: The final encoding - #[pyo3( - signature = (encoding, pair = None, add_special_tokens = true) -> "Encoding" - )] - #[pyo3(text_signature = "(self, encoding, pair=None, add_special_tokens=True)")] - fn process( - &self, - encoding: &PyEncoding, - pair: Option<&PyEncoding>, - add_special_tokens: bool, - ) -> PyResult { - let final_encoding = ToPyResult(self.processor.process( - encoding.encoding.clone(), - pair.map(|e| e.encoding.clone()), - add_special_tokens, - )) - .into_py()?; - Ok(final_encoding.into()) - } - - fn __repr__(&self) -> PyResult { - crate::utils::serde_pyo3::repr(self) - .map_err(|e| exceptions::PyException::new_err(e.to_string())) - } - - fn __str__(&self) -> PyResult { - crate::utils::serde_pyo3::to_string(self) - .map_err(|e| exceptions::PyException::new_err(e.to_string())) - } -} - -macro_rules! getter { - ($self: ident, $variant: ident, $($name: tt)+) => {{ - let super_ = $self.as_ref(); - if let PyPostProcessorTypeWrapper::Single(ref single) = super_.processor { - if let PostProcessorWrapper::$variant(ref post) = *single.read().expect( - "RwLock synchronisation primitive is poisoned, cannot get subtype of PyPostProcessor" - ) { - post.$($name)+ - } else { - unreachable!() - } - } else { - unreachable!() - } - }}; -} - -macro_rules! setter { - ($self: ident, $variant: ident, $name: ident, $value: expr) => {{ - let super_ = $self.as_ref(); - if let PyPostProcessorTypeWrapper::Single(ref single) = super_.processor { - if let PostProcessorWrapper::$variant(ref mut post) = *single.write().expect( - "RwLock synchronisation primitive is poisoned, cannot get subtype of PyPostProcessor", - ) { - post.$name = $value; - } - } - }}; - ($self: ident, $variant: ident, @$name: ident, $value: expr) => {{ - let super_ = $self.as_ref(); - if let PyPostProcessorTypeWrapper::Single(ref single) = super_.processor { - if let PostProcessorWrapper::$variant(ref mut post) = *single.write().expect( - "RwLock synchronisation primitive is poisoned, cannot get subtype of PyPostProcessor", - ) { - post.$name($value); - } - } - };}; -} - -#[derive(Clone)] -pub(crate) enum PyPostProcessorTypeWrapper { - Sequence(Vec>>), - Single(Arc>), -} - -impl PostProcessor for PyPostProcessorTypeWrapper { - fn added_tokens(&self, is_pair: bool) -> usize { - match self { - PyPostProcessorTypeWrapper::Single(inner) => inner - .read() - .expect("RwLock synchronisation primitive is poisoned, cannot get subtype of PyPostProcessor") - .added_tokens(is_pair), - PyPostProcessorTypeWrapper::Sequence(inner) => inner.iter().map(|p| { - p.read() - .expect("RwLock synchronisation primitive is poisoned, cannot get subtype of PyPostProcessor") - .added_tokens(is_pair) - }).sum::(), - } - } - - fn process_encodings( - &self, - mut encodings: Vec, - add_special_tokens: bool, - ) -> tk::Result> { - match self { - PyPostProcessorTypeWrapper::Single(inner) => inner - .read() - .map_err(|_| PyException::new_err("RwLock synchronisation primitive is poisoned, cannot get subtype of PyPreTokenizer"))? - .process_encodings(encodings, add_special_tokens), - PyPostProcessorTypeWrapper::Sequence(inner) => { - for processor in inner.iter() { - encodings = processor - .read() - .map_err(|_| PyException::new_err("RwLock synchronisation primitive is poisoned, cannot get subtype of PyPreTokenizer"))? - .process_encodings(encodings, add_special_tokens)?; - } - Ok(encodings) - }, - } - } -} - -impl<'de> Deserialize<'de> for PyPostProcessorTypeWrapper { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let wrapper = PostProcessorWrapper::deserialize(deserializer)?; - Ok(wrapper.into()) - } -} - -impl Serialize for PyPostProcessorTypeWrapper { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - match self { - PyPostProcessorTypeWrapper::Sequence(seq) => { - let mut ser = serializer.serialize_struct("Sequence", 2)?; - ser.serialize_field("type", "Sequence")?; - ser.serialize_field("processors", seq)?; - ser.end() - } - PyPostProcessorTypeWrapper::Single(inner) => inner.serialize(serializer), - } - } -} - -impl From for PyPostProcessorTypeWrapper -where - I: Into, -{ - fn from(processor: I) -> Self { - let processor = processor.into(); - match processor { - PostProcessorWrapper::Sequence(seq) => PyPostProcessorTypeWrapper::Sequence( - seq.into_iter().map(|p| Arc::new(RwLock::new(p))).collect(), - ), - _ => PyPostProcessorTypeWrapper::Single(Arc::new(RwLock::new(processor.clone()))), - } - } -} - -/// This post-processor takes care of adding the special tokens needed by -/// a Bert model: -/// -/// - a SEP token -/// - a CLS token -/// -/// Args: -/// sep (:obj:`Tuple[str, int]`): -/// A tuple with the string representation of the SEP token, and its id -/// -/// cls (:obj:`Tuple[str, int]`): -/// A tuple with the string representation of the CLS token, and its id -/// -/// Example:: -/// -/// >>> from tokenizers.processors import BertProcessing -/// >>> processor = BertProcessing(("[SEP]", 102), ("[CLS]", 101)) -/// >>> processor.process(encoding) -/// # Encoding with [CLS] at start and [SEP] at end -/// -#[pyclass(extends=PyPostProcessor, module = "tokenizers.processors", name = "BertProcessing")] -pub struct PyBertProcessing {} -#[pymethods] -impl PyBertProcessing { - #[new] - #[pyo3(text_signature = "(self, sep, cls_token: str| int)")] - fn new(sep: (String, u32), cls_token: (String, u32)) -> PyClassInitializer { - PyClassInitializer::::from(PyPostProcessor::from(BertProcessing::new( - sep, cls_token, - ))) - .add_subclass(PyBertProcessing {}) - } - - fn __getnewargs__<'p>(&self, py: Python<'p>) -> PyResult> { - PyTuple::new(py, [("", 0), ("", 0)]) - } - - #[getter] - fn get_sep(self_: PyRef<'_, Self>) -> Result, PyErr> { - let py = self_.py(); - let (tok, id) = getter!(self_, Bert, get_sep_copy()); - PyTuple::new( - py, - Vec::>::from([tok.into_py_any(py)?, id.into_py_any(py)?]), - ) - } - - #[setter] - fn set_sep(self_: PyRef, sep: Bound<'_, PyTuple>) -> PyResult<()> { - let sep = sep.extract()?; - setter!(self_, Bert, sep, sep); - Ok(()) - } - - #[getter] - fn get_cls(self_: PyRef<'_, Self>) -> Result, PyErr> { - let py = self_.py(); - let (tok, id) = getter!(self_, Bert, get_cls_copy()); - PyTuple::new( - py, - Vec::>::from([tok.into_py_any(py)?, id.into_py_any(py)?]), - ) - } - - #[setter] - fn set_cls(self_: PyRef, cls: Bound<'_, PyTuple>) -> PyResult<()> { - let cls = cls.extract()?; - setter!(self_, Bert, cls, cls); - Ok(()) - } -} - -/// This post-processor takes care of adding the special tokens needed by -/// a Roberta model: -/// -/// - a SEP token -/// - a CLS token -/// -/// It also takes care of trimming the offsets. -/// By default, the ByteLevel BPE might include whitespaces in the produced tokens. If you don't -/// want the offsets to include these whitespaces, then this PostProcessor should be initialized -/// with :obj:`trim_offsets=True` -/// -/// Args: -/// sep (:obj:`Tuple[str, int]`): -/// A tuple with the string representation of the SEP token, and its id -/// -/// cls (:obj:`Tuple[str, int]`): -/// A tuple with the string representation of the CLS token, and its id -/// -/// trim_offsets (:obj:`bool`, `optional`, defaults to :obj:`True`): -/// Whether to trim the whitespaces from the produced offsets. -/// -/// add_prefix_space (:obj:`bool`, `optional`, defaults to :obj:`True`): -/// Whether the add_prefix_space option was enabled during pre-tokenization. This -/// is relevant because it defines the way the offsets are trimmed out. -/// -/// Example:: -/// -/// >>> from tokenizers.processors import RobertaProcessing -/// >>> processor = RobertaProcessing(("", 2), ("", 0)) -/// >>> processor.process(encoding) -/// # Encoding with at start and at end -/// -#[pyclass(extends=PyPostProcessor, module = "tokenizers.processors", name = "RobertaProcessing")] -pub struct PyRobertaProcessing {} -#[pymethods] -impl PyRobertaProcessing { - #[new] - #[pyo3( - signature = (sep, cls_token, trim_offsets = true, add_prefix_space = true), - text_signature = "(self, sep, cls_token, trim_offsets=True, add_prefix_space=True)" - )] - fn new( - sep: (String, u32), - cls_token: (String, u32), - trim_offsets: bool, - add_prefix_space: bool, - ) -> PyClassInitializer { - let proc = RobertaProcessing::new(sep, cls_token) - .trim_offsets(trim_offsets) - .add_prefix_space(add_prefix_space); - PyClassInitializer::::from(PyPostProcessor::from(proc)) - .add_subclass(PyRobertaProcessing {}) - } - - fn __getnewargs__<'p>(&self, py: Python<'p>) -> PyResult> { - PyTuple::new(py, [("", 0), ("", 0)]) - } - - #[getter] - fn get_sep(self_: PyRef<'_, Self>) -> Result, PyErr> { - let py = self_.py(); - let (tok, id) = getter!(self_, Roberta, get_sep_copy()); - PyTuple::new( - py, - Vec::>::from([tok.into_py_any(py)?, id.into_py_any(py)?]), - ) - } - - #[setter] - fn set_sep(self_: PyRef, sep: Bound<'_, PyTuple>) -> PyResult<()> { - let sep = sep.extract()?; - setter!(self_, Roberta, sep, sep); - Ok(()) - } - - #[getter] - fn get_cls(self_: PyRef<'_, Self>) -> Result, PyErr> { - let py = self_.py(); - let (tok, id) = getter!(self_, Roberta, get_cls_copy()); - PyTuple::new( - py, - Vec::>::from([tok.into_py_any(py)?, id.into_py_any(py)?]), - ) - } - - #[setter] - fn set_cls(self_: PyRef, cls: Bound<'_, PyTuple>) -> PyResult<()> { - let cls = cls.extract()?; - setter!(self_, Roberta, cls, cls); - Ok(()) - } - - #[getter] - fn get_trim_offsets(self_: PyRef) -> bool { - getter!(self_, Roberta, trim_offsets) - } - - #[setter] - fn set_trim_offsets(self_: PyRef, trim_offsets: bool) { - setter!(self_, Roberta, trim_offsets, trim_offsets) - } - - #[getter] - fn get_add_prefix_space(self_: PyRef) -> bool { - getter!(self_, Roberta, add_prefix_space) - } - - #[setter] - fn set_add_prefix_space(self_: PyRef, add_prefix_space: bool) { - setter!(self_, Roberta, add_prefix_space, add_prefix_space) - } -} - -/// This post-processor takes care of trimming the offsets. -/// -/// By default, the ByteLevel BPE might include whitespaces in the produced tokens. If you don't -/// want the offsets to include these whitespaces, then this PostProcessor must be used. -/// -/// Args: -/// trim_offsets (:obj:`bool`): -/// Whether to trim the whitespaces from the produced offsets. -/// -/// add_prefix_space (:obj:`bool`, `optional`, defaults to :obj:`True`): -/// If :obj:`True`, keeps the first token's offset as is. If :obj:`False`, increments -/// the start of the first token's offset by 1. Only has an effect if :obj:`trim_offsets` -/// is set to :obj:`True`. -/// -/// Example:: -/// -/// >>> from tokenizers.processors import ByteLevel -/// >>> processor = ByteLevel(trim_offsets=True) -/// >>> # Offsets will be trimmed to exclude leading whitespace bytes -/// -#[pyclass(extends=PyPostProcessor, module = "tokenizers.processors", name = "ByteLevel")] -pub struct PyByteLevel {} -#[pymethods] -impl PyByteLevel { - #[new] - #[pyo3( - signature = (add_prefix_space = None, trim_offsets = None, use_regex = None, **_kwargs), - text_signature = "(self, add_prefix_space=None, trim_offsets=None, use_regex=None)" - )] - fn new( - add_prefix_space: Option, - trim_offsets: Option, - use_regex: Option, - _kwargs: Option<&Bound<'_, PyDict>>, - ) -> PyClassInitializer { - let mut byte_level = ByteLevel::default(); - - if let Some(aps) = add_prefix_space { - byte_level = byte_level.add_prefix_space(aps); - } - - if let Some(to) = trim_offsets { - byte_level = byte_level.trim_offsets(to); - } - - if let Some(ur) = use_regex { - byte_level = byte_level.use_regex(ur); - } - - PyClassInitializer::::from(PyPostProcessor::from(byte_level)) - .add_subclass(PyByteLevel {}) - } - - #[getter] - fn get_add_prefix_space(self_: PyRef) -> bool { - getter!(self_, ByteLevel, add_prefix_space) - } - - #[setter] - fn set_add_prefix_space(self_: PyRef, add_prefix_space: bool) { - setter!(self_, ByteLevel, add_prefix_space, add_prefix_space) - } - - #[getter] - fn get_trim_offsets(self_: PyRef) -> bool { - getter!(self_, ByteLevel, trim_offsets) - } - - #[setter] - fn set_trim_offsets(self_: PyRef, trim_offsets: bool) { - setter!(self_, ByteLevel, trim_offsets, trim_offsets) - } - - #[getter] - fn get_use_regex(self_: PyRef) -> bool { - getter!(self_, ByteLevel, use_regex) - } - - #[setter] - fn set_use_regex(self_: PyRef, use_regex: bool) { - setter!(self_, ByteLevel, use_regex, use_regex) - } -} - -#[derive(Clone, Debug)] -pub struct PySpecialToken(SpecialToken); - -impl From for SpecialToken { - fn from(v: PySpecialToken) -> Self { - v.0 - } -} - -impl<'a, 'py> FromPyObject<'a, 'py> for PySpecialToken { - type Error = PyErr; - - fn extract(ob: Borrowed<'a, 'py, PyAny>) -> Result { - if let Ok(v) = ob.extract::<(String, u32)>() { - Ok(Self(v.into())) - } else if let Ok(v) = ob.extract::<(u32, String)>() { - Ok(Self(v.into())) - } else if let Ok(d) = ob.cast::() { - let id = d - .get_item("id")? - .ok_or_else(|| exceptions::PyValueError::new_err("`id` must be specified"))? - .extract::()?; - let ids = d - .get_item("ids")? - .ok_or_else(|| exceptions::PyValueError::new_err("`ids` must be specified"))? - .extract::>()?; - let tokens = d - .get_item("tokens")? - .ok_or_else(|| exceptions::PyValueError::new_err("`tokens` must be specified"))? - .extract::>()?; - - Ok(Self( - ToPyResult(SpecialToken::new(id, ids, tokens)).into_py()?, - )) - } else { - Err(exceptions::PyTypeError::new_err( - "Expected Union[Tuple[str, int], Tuple[int, str], dict]", - )) - } - } -} - -#[derive(Clone, Debug)] -pub struct PyTemplate(Template); - -impl From for Template { - fn from(v: PyTemplate) -> Self { - v.0 - } -} - -impl<'a, 'py> FromPyObject<'a, 'py> for PyTemplate { - type Error = PyErr; - - fn extract(ob: Borrowed<'a, 'py, PyAny>) -> Result { - if let Ok(s) = ob.extract::() { - Ok(Self( - s.try_into().map_err(exceptions::PyValueError::new_err)?, - )) - } else if let Ok(s) = ob.extract::>() { - Ok(Self( - s.try_into().map_err(exceptions::PyValueError::new_err)?, - )) - } else { - Err(exceptions::PyTypeError::new_err( - "Expected Union[str, List[str]]", - )) - } - } -} - -/// Provides a way to specify templates in order to add the special tokens to each -/// input sequence as relevant. -/// -/// Let's take :obj:`BERT` tokenizer as an example. It uses two special tokens, used to -/// delimitate each sequence. :obj:`[CLS]` is always used at the beginning of the first -/// sequence, and :obj:`[SEP]` is added at the end of both the first, and the pair -/// sequences. The final result looks like this: -/// -/// - Single sequence: :obj:`[CLS] Hello there [SEP]` -/// - Pair sequences: :obj:`[CLS] My name is Anthony [SEP] What is my name? [SEP]` -/// -/// With the type ids as following:: -/// -/// [CLS] ... [SEP] ... [SEP] -/// 0 0 0 1 1 -/// -/// You can achieve such behavior using a TemplateProcessing:: -/// -/// TemplateProcessing( -/// single="[CLS] $0 [SEP]", -/// pair="[CLS] $A [SEP] $B:1 [SEP]:1", -/// special_tokens=[("[CLS]", 1), ("[SEP]", 0)], -/// ) -/// -/// In this example, each input sequence is identified using a ``$`` construct. This identifier -/// lets us specify each input sequence, and the type_id to use. When nothing is specified, -/// it uses the default values. Here are the different ways to specify it: -/// -/// - Specifying the sequence, with default ``type_id == 0``: ``$A`` or ``$B`` -/// - Specifying the `type_id` with default ``sequence == A``: ``$0``, ``$1``, ``$2``, ... -/// - Specifying both: ``$A:0``, ``$B:1``, ... -/// -/// The same construct is used for special tokens: ``(:)?``. -/// -/// **Warning**: You must ensure that you are giving the correct tokens/ids as these -/// will be added to the Encoding without any further check. If the given ids correspond -/// to something totally different in a `Tokenizer` using this `PostProcessor`, it -/// might lead to unexpected results. -/// -/// Args: -/// single (:obj:`Template`): -/// The template used for single sequences -/// -/// pair (:obj:`Template`): -/// The template used when both sequences are specified -/// -/// special_tokens (:obj:`Tokens`): -/// The list of special tokens used in each sequences -/// -/// Types: -/// -/// Template (:obj:`str` or :obj:`List`): -/// - If a :obj:`str` is provided, the whitespace is used as delimiter between tokens -/// - If a :obj:`List[str]` is provided, a list of tokens -/// -/// Tokens (:obj:`List[Union[Tuple[int, str], Tuple[str, int], dict]]`): -/// - A :obj:`Tuple` with both a token and its associated ID, in any order -/// - A :obj:`dict` with the following keys: -/// - "id": :obj:`str` => The special token id, as specified in the Template -/// - "ids": :obj:`List[int]` => The associated IDs -/// - "tokens": :obj:`List[str]` => The associated tokens -/// -/// The given dict expects the provided :obj:`ids` and :obj:`tokens` lists to have -/// the same length. -#[pyclass(extends=PyPostProcessor, module = "tokenizers.processors", name = "TemplateProcessing")] -pub struct PyTemplateProcessing {} -#[pymethods] -impl PyTemplateProcessing { - #[new] - #[pyo3( - signature = (single = None, pair = None, special_tokens = None), - text_signature = "(self, single=None, pair=None, special_tokens=None)" - )] - fn new( - single: Option, - pair: Option, - special_tokens: Option>, - ) -> PyResult> { - let mut builder = tk::processors::template::TemplateProcessing::builder(); - - if let Some(seq) = single { - builder.single(seq.into()); - } - if let Some(seq) = pair { - builder.pair(seq.into()); - } - if let Some(sp) = special_tokens { - builder.special_tokens(sp); - } - let processor = builder - .build() - .map_err(|e| exceptions::PyValueError::new_err(e.to_string()))?; - - Ok( - PyClassInitializer::::from(PyPostProcessor::from(processor)) - .add_subclass(PyTemplateProcessing {}), - ) - } - - #[getter] - fn get_single(self_: PyRef) -> String { - getter!(self_, Template, get_single()) - } - - #[setter] - fn set_single(self_: PyRef, single: PyTemplate) -> PyResult<()> { - let template: Template = Template::from(single); - let super_ = self_.as_ref(); - if let PyPostProcessorTypeWrapper::Single(ref inner) = super_.processor - && let PostProcessorWrapper::Template(ref mut post) = *inner - .write() - .map_err(|_| PyException::new_err("RwLock synchronisation primitive is poisoned, cannot get subtype of PyPostProcessor"))? { - post.set_single(template); - } - Ok(()) - } -} - -/// Sequence Processor -/// -/// Chains multiple post-processors together, applying them in order. Each processor -/// in the sequence processes the output of the previous one. -/// -/// Args: -/// processors (:obj:`List[PostProcessor]`): -/// The list of post-processors to chain together. -/// -/// Example:: -/// -/// >>> from tokenizers.processors import BertProcessing, ByteLevel, Sequence -/// >>> processor = Sequence([ByteLevel(trim_offsets=True), BertProcessing(("[SEP]", 102), ("[CLS]", 101))]) -/// -#[pyclass(extends=PyPostProcessor, module = "tokenizers.processors", name = "Sequence")] -pub struct PySequence {} - -#[pymethods] -impl PySequence { - #[new] - #[pyo3(signature = (processors_py), text_signature = "(self, processors)")] - fn new(processors_py: &Bound<'_, PyList>) -> PyResult> { - let mut processors = Vec::with_capacity(processors_py.len()); - for n in processors_py.iter() { - let processor: PyRef = n.extract()?; - match &processor.processor { - PyPostProcessorTypeWrapper::Sequence(inner) => { - processors.extend(inner.iter().cloned()) - } - PyPostProcessorTypeWrapper::Single(inner) => processors.push(inner.clone()), - } - } - Ok( - PyClassInitializer::::from(PyPostProcessor::new( - PyPostProcessorTypeWrapper::Sequence(processors), - )) - .add_subclass(PySequence {}), - ) - } - - fn __getnewargs__<'p>(&self, py: Python<'p>) -> PyResult> { - PyTuple::new(py, [PyList::empty(py)]) - } - - fn __getitem__(self_: PyRef<'_, Self>, py: Python<'_>, index: usize) -> PyResult> { - match &self_.as_ref().processor { - PyPostProcessorTypeWrapper::Sequence(inner) => match inner.get(index) { - Some(item) => { - PyPostProcessor::new(PyPostProcessorTypeWrapper::Single(item.clone())) - .get_as_subtype(py) - } - _ => Err(PyErr::new::( - "Index not found", - )), - }, - _ => Err(PyErr::new::( - "This processor is not a Sequence, it does not support __getitem__", - )), - } - } - - fn __setitem__(self_: PyRef<'_, Self>, index: usize, value: Bound<'_, PyAny>) -> PyResult<()> { - let processor: PyPostProcessor = value.extract()?; - let PyPostProcessorTypeWrapper::Single(processor) = processor.processor else { - return Err(PyException::new_err("processor should not be a sequence")); - }; - - match &self_.as_ref().processor { - PyPostProcessorTypeWrapper::Sequence(inner) => match inner.get(index) { - Some(item) => { - *item - .write() - .map_err(|_| PyException::new_err("RwLock synchronisation primitive is poisoned, cannot get subtype of PyPostProcessor"))? = processor - .read() - .map_err(|_| PyException::new_err("RwLock synchronisation primitive is poisoned, cannot get subtype of PyPostProcessor"))? - .clone(); - } - _ => { - return Err(PyErr::new::( - "Index not found", - )); - } - }, - _ => { - return Err(PyException::new_err( - "This processor is not a Sequence, it does not support __setitem__", - )); - } - }; - Ok(()) - } -} - -/// Processors Module -#[pymodule(gil_used = false)] -pub mod processors { - #[pymodule_export] - pub use super::PyBertProcessing; - #[pymodule_export] - pub use super::PyByteLevel; - #[pymodule_export] - pub use super::PyPostProcessor; - #[pymodule_export] - pub use super::PyRobertaProcessing; - #[pymodule_export] - pub use super::PySequence; - #[pymodule_export] - pub use super::PyTemplateProcessing; -} - -#[cfg(test)] -mod test { - use std::sync::{Arc, RwLock}; - - use pyo3::prelude::*; - use tk::processors::PostProcessorWrapper; - use tk::processors::bert::BertProcessing; - - use crate::processors::{PyPostProcessor, PyPostProcessorTypeWrapper}; - - #[test] - fn get_subtype() { - Python::attach(|py| { - let py_proc = PyPostProcessor::new(PyPostProcessorTypeWrapper::Single(Arc::new( - RwLock::new(BertProcessing::new(("SEP".into(), 0), ("CLS".into(), 1)).into()), - ))); - let py_bert = py_proc.get_as_subtype(py).unwrap(); - assert_eq!( - "BertProcessing", - py_bert.bind(py).get_type().qualname().unwrap() - ); - }) - } - - #[test] - fn serialize() { - let rs_processing = BertProcessing::new(("SEP".into(), 0), ("CLS".into(), 1)); - let rs_wrapper: PostProcessorWrapper = rs_processing.clone().into(); - let rs_processing_ser = serde_json::to_string(&rs_processing).unwrap(); - let rs_wrapper_ser = serde_json::to_string(&rs_wrapper).unwrap(); - - let py_processing = PyPostProcessor::new(PyPostProcessorTypeWrapper::Single(Arc::new( - RwLock::new(rs_wrapper), - ))); - let py_ser = serde_json::to_string(&py_processing).unwrap(); - assert_eq!(py_ser, rs_processing_ser); - assert_eq!(py_ser, rs_wrapper_ser); - - let py_processing: PyPostProcessor = serde_json::from_str(&rs_processing_ser).unwrap(); - match py_processing.processor { - PyPostProcessorTypeWrapper::Single(inner) => match *inner.as_ref().read().unwrap() { - PostProcessorWrapper::Bert(_) => (), - _ => panic!("Expected Bert postprocessor."), - }, - _ => panic!("Expected a single processor, got a sequence"), - } - - let py_processing: PyPostProcessor = serde_json::from_str(&rs_wrapper_ser).unwrap(); - match py_processing.processor { - PyPostProcessorTypeWrapper::Single(inner) => match *inner.as_ref().read().unwrap() { - PostProcessorWrapper::Bert(_) => (), - _ => panic!("Expected Bert postprocessor."), - }, - _ => panic!("Expected a single processor, got a sequence"), - }; - } -} diff --git a/bindings/python/src/token.rs b/bindings/python/src/token.rs deleted file mode 100644 index 4163f8310..000000000 --- a/bindings/python/src/token.rs +++ /dev/null @@ -1,50 +0,0 @@ -use pyo3::prelude::*; -use tk::Token; - -#[pyclass(module = "tokenizers", name = "Token", frozen, from_py_object)] -#[derive(Clone)] -pub struct PyToken { - token: Token, -} -impl From for PyToken { - fn from(token: Token) -> Self { - Self { token } - } -} -impl From for Token { - fn from(token: PyToken) -> Self { - token.token - } -} - -#[pymethods] -impl PyToken { - /// Create a token from id, string value and byte offsets - #[new] - #[pyo3( - signature = (id, value, offsets), - text_signature = "(self, id, value, offsets)" - )] - fn new(id: u32, value: String, offsets: (usize, usize)) -> PyToken { - Token::new(id, value, offsets).into() - } - - #[getter] - fn get_id(&self) -> u32 { - self.token.id - } - - #[getter] - fn get_value(&self) -> &str { - &self.token.value - } - - #[getter] - fn get_offsets(&self) -> (usize, usize) { - self.token.offsets - } - - fn as_tuple(&self) -> (u32, &str, (usize, usize)) { - (self.token.id, &self.token.value, self.token.offsets) - } -} diff --git a/bindings/python/src/tokenizer.rs b/bindings/python/src/tokenizer.rs index b7737c428..917e1a843 100644 --- a/bindings/python/src/tokenizer.rs +++ b/bindings/python/src/tokenizer.rs @@ -1,2020 +1,542 @@ -use serde::{Serialize, Serializer, ser::Error as SerError}; -use std::collections::{HashMap, hash_map::DefaultHasher}; -use std::hash::{Hash, Hasher}; -use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard}; - -use numpy::{PyArray1, PyArrayMethods, npyffi}; -use pyo3::IntoPyObject; -use pyo3::class::basic::CompareOp; -use pyo3::intern; +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +use numpy::{IntoPyArray, PyArray1}; +use pyo3::exceptions::{PyNotImplementedError, PyRuntimeError, PyStopIteration, PyTypeError}; +use pyo3::marker::Ungil; use pyo3::prelude::*; -use pyo3::types::*; -use pyo3::{IntoPyObjectExt, exceptions}; -use tk::models::bpe::BPE; -use tk::tokenizer::{ - PaddingDirection, PaddingParams, PaddingStrategy, PostProcessor, TokenizerImpl, - TruncationDirection, TruncationParams, TruncationStrategy, +use pyo3::pybacked::PyBackedStr; +use pyo3::types::{PyBytes, PyList}; +use rayon::prelude::*; +use tk_encode::Tokenizer as SpecTokenizer; +use tk_encode::pipeline::{ + Model as _, PipelineModelScratch, PipelineToken, PipelineTokenizer, Span, }; -use tk::utils::iter::ResultShunt; -use tk::{TokenizerTrainExt, Trainable}; -use tokenizers as tk; - -use super::decoders::PyDecoder; -use super::encoding::PyEncoding; -use super::error::{PyError, ToPyResult}; -use super::models::PyModel; -use super::normalizers::PyNormalizer; -use super::pre_tokenizers::PyPreTokenizer; -use super::trainers::PyTrainer; -use crate::processors::PyPostProcessor; -use crate::utils::{MaybeSizedIterator, PyBufferedIterator}; -use std::collections::BTreeMap; - -/// Represents a token that can be be added to a :class:`~tokenizers.Tokenizer`. -/// It can have special options that defines the way it should behave. -/// -/// Args: -/// content (:obj:`str`): The content of the token -/// -/// single_word (:obj:`bool`, defaults to :obj:`False`): -/// Defines whether this token should only match single words. If :obj:`True`, this -/// token will never match inside of a word. For example the token ``ing`` would match -/// on ``tokenizing`` if this option is :obj:`False`, but not if it is :obj:`True`. -/// The notion of "`inside of a word`" is defined by the word boundaries pattern in -/// regular expressions (ie. the token should start and end with word boundaries). -/// -/// lstrip (:obj:`bool`, defaults to :obj:`False`): -/// Defines whether this token should strip all potential whitespaces on its left side. -/// If :obj:`True`, this token will greedily match any whitespace on its left. For -/// example if we try to match the token ``[MASK]`` with ``lstrip=True``, in the text -/// ``"I saw a [MASK]"``, we would match on ``" [MASK]"``. (Note the space on the left). -/// -/// rstrip (:obj:`bool`, defaults to :obj:`False`): -/// Defines whether this token should strip all potential whitespaces on its right -/// side. If :obj:`True`, this token will greedily match any whitespace on its right. -/// It works just like :obj:`lstrip` but on the right. -/// -/// normalized (:obj:`bool`, defaults to :obj:`True` with :meth:`~tokenizers.Tokenizer.add_tokens` and :obj:`False` with :meth:`~tokenizers.Tokenizer.add_special_tokens`): -/// Defines whether this token should match against the normalized version of the input -/// text. For example, with the added token ``"yesterday"``, and a normalizer in charge of -/// lowercasing the text, the token could be extract from the input ``"I saw a lion -/// Yesterday"``. -/// special (:obj:`bool`, defaults to :obj:`False` with :meth:`~tokenizers.Tokenizer.add_tokens` and :obj:`False` with :meth:`~tokenizers.Tokenizer.add_special_tokens`): -/// Defines whether this token should be skipped when decoding. -/// -#[pyclass(dict, module = "tokenizers", name = "AddedToken")] -pub struct PyAddedToken { - pub content: String, - pub special: bool, - pub single_word: Option, - pub lstrip: Option, - pub rstrip: Option, - pub normalized: Option, +use tk_encode::tokenizer::PostProcessor as _; +use tk_encode::utils::parallelism::get_parallelism; +use tk_train::{TokenizerTrainExt, Trainable}; + +use crate::added_token::{TokenInput, parse_tokens}; +use crate::detached_lock::{Detached, DetachedRwLock}; +use crate::error::{TokenizersError, to_pyerr}; +use crate::models::{PyModel, wrap_model}; +use crate::normalizers::{PyNormalizer, wrap_normalizer}; +use crate::pre_tokenizers::{PyPreTokenizer, wrap_pre_tokenizer}; +use crate::trainers::PyTrainer; + +/// Set when the bindings actually run a rayon-parallel section, so the +/// pthread_atfork handler only disables parallelism in children of processes +/// that really used it (mirrors the v1 bindings' semantics). +pub static USED_PARALLELISM: AtomicBool = AtomicBool::new(false); + +/// The compiled encode path plus the facts about the spec the encode calls +/// need without re-locking it. +#[derive(Clone)] +struct Compiled { + pipe: Arc, + /// Whether the spec's post-processor would add special tokens. Post-processing + /// is not wired into the pipeline yet, so encode(add_special_tokens=True) + /// must fail loudly instead of silently dropping them. + post_adds_special_tokens: bool, } -impl PyAddedToken { - pub fn from>(content: S, special: Option) -> Self { - Self { - content: content.into(), - special: special.unwrap_or(false), - single_word: None, - lstrip: None, - rstrip: None, - normalized: None, - } - } - pub fn get_token(&self) -> tk::tokenizer::AddedToken { - let mut token = tk::AddedToken::from(&self.content, self.special); - - if let Some(sw) = self.single_word { - token = token.single_word(sw); - } - if let Some(ls) = self.lstrip { - token = token.lstrip(ls); - } - if let Some(rs) = self.rstrip { - token = token.rstrip(rs); - } - if let Some(n) = self.normalized { - token = token.normalized(n); - } - - token - } - - pub fn as_pydict<'py>(&self, py: Python<'py>) -> PyResult> { - let dict = PyDict::new(py); - let token = self.get_token(); - - dict.set_item("content", token.content)?; - dict.set_item("single_word", token.single_word)?; - dict.set_item("lstrip", token.lstrip)?; - dict.set_item("rstrip", token.rstrip)?; - dict.set_item("normalized", token.normalized)?; - dict.set_item("special", token.special)?; - - Ok(dict) - } +struct Inner { + /// Source of truth: the mutable, serializable tokenizer definition. + spec: SpecTokenizer, + /// Memoized compilation of `spec`; invalidated by every mutation. + compiled: Option, } -impl From for PyAddedToken { - fn from(token: tk::AddedToken) -> Self { - Self { - content: token.content, - single_word: Some(token.single_word), - lstrip: Some(token.lstrip), - rstrip: Some(token.rstrip), - normalized: Some(token.normalized), - special: token.special, - } - } -} - -#[pymethods] -impl PyAddedToken { - #[new] - #[pyo3( - signature = (content=None, **kwargs), - text_signature = "(self, content=None, single_word=False, lstrip=False, rstrip=False, normalized=True, special=False)" - )] - fn __new__(content: Option<&str>, kwargs: Option<&Bound<'_, PyDict>>) -> PyResult { - let mut token = PyAddedToken::from(content.unwrap_or(""), None); - - if let Some(kwargs) = kwargs { - for (key, value) in kwargs { - let key: String = key.extract()?; - match key.as_ref() { - "single_word" => token.single_word = Some(value.extract()?), - "lstrip" => token.lstrip = Some(value.extract()?), - "rstrip" => token.rstrip = Some(value.extract()?), - "normalized" => token.normalized = Some(value.extract()?), - "special" => token.special = value.extract()?, - _ => println!("Ignored unknown kwarg option {key}"), - } - } - } - - Ok(token) - } - - fn __getstate__<'py>(&self, py: Python<'py>) -> PyResult> { - self.as_pydict(py) - } - - fn __setstate__(&mut self, py: Python, state: Py) -> PyResult<()> { - match state.cast_bound::(py) { - Ok(state) => { - for (key, value) in state { - let key: String = key.extract()?; - match key.as_ref() { - "content" => self.content = value.extract()?, - "single_word" => self.single_word = Some(value.extract()?), - "lstrip" => self.lstrip = Some(value.extract()?), - "rstrip" => self.rstrip = Some(value.extract()?), - "normalized" => self.normalized = Some(value.extract()?), - "special" => self.special = value.extract()?, - _ => {} - } - } - Ok(()) - } - Err(e) => Err(e.into()), - } - } - - /// Get the content of this :obj:`AddedToken` - #[getter] - fn get_content(&self) -> &str { - &self.content - } - - /// Set the content of this :obj:`AddedToken` - #[setter] - fn set_content(&mut self, content: String) { - self.content = content; - } - - /// Get the value of the :obj:`rstrip` option - #[getter] - fn get_rstrip(&self) -> bool { - self.get_token().rstrip - } - - /// Get the value of the :obj:`lstrip` option - #[getter] - fn get_lstrip(&self) -> bool { - self.get_token().lstrip - } - - /// Get the value of the :obj:`single_word` option - #[getter] - fn get_single_word(&self) -> bool { - self.get_token().single_word - } - - /// Get the value of the :obj:`normalized` option - #[getter] - fn get_normalized(&self) -> bool { - self.get_token().normalized - } - /// Get the value of the :obj:`special` option - #[getter] - fn get_special(&self) -> bool { - self.get_token().special - } - - /// Set the value of the :obj:`special` option - #[setter] - fn set_special(&mut self, special: bool) { - self.special = special; - } - - fn __str__(&self) -> PyResult<&str> { - Ok(&self.content) - } - - fn __repr__(&self) -> PyResult { - let bool_to_python = |p| match p { - true => "True", - false => "False", - }; - - let token = self.get_token(); - Ok(format!( - "AddedToken(\"{}\", rstrip={}, lstrip={}, single_word={}, normalized={}, special={})", - self.content, - bool_to_python(token.rstrip), - bool_to_python(token.lstrip), - bool_to_python(token.single_word), - bool_to_python(token.normalized), - bool_to_python(token.special) - )) - } - - fn __richcmp__(&self, other: Py, op: CompareOp) -> bool { - use CompareOp::*; - Python::attach(|py| match op { - Lt | Le | Gt | Ge => false, - Eq => self.get_token() == other.borrow(py).get_token(), - Ne => self.get_token() != other.borrow(py).get_token(), - }) - } - - fn __hash__(&self) -> u64 { - let mut hasher = DefaultHasher::new(); - self.get_token().hash(&mut hasher); - hasher.finish() - } +fn poisoned(_: std::sync::PoisonError) -> PyErr { + PyRuntimeError::new_err("tokenizer lock poisoned") } -struct TextInputSequence<'s>(tk::InputSequence<'s>); -impl<'a, 'py> FromPyObject<'a, 'py> for TextInputSequence<'py> { - type Error = PyErr; - - fn extract(ob: Borrowed<'a, 'py, PyAny>) -> Result { - let err = exceptions::PyTypeError::new_err("TextInputSequence must be str"); - if let Ok(s) = ob.extract::() { - Ok(Self(s.into())) - } else { - Err(err) - } - } -} -impl<'s> From> for tk::InputSequence<'s> { - fn from(s: TextInputSequence<'s>) -> Self { - s.0 - } +/// A tokenizer: a model plus its optional normalizer and pre-tokenizer. +/// +/// Create one from a model (`Tokenizer(models.BPE())`), a file +/// (`Tokenizer.from_file`), or the Hub (`Tokenizer.from_pretrained`). +/// Changes — assigning components, training, adding tokens — apply to the +/// serializable definition; encoding runs a compiled pipeline that is rebuilt +/// automatically after any change. A definition the pipeline cannot run +/// raises `TokenizersError` at that point, with the reason. +// The lock/GIL ordering rule (never block on the lock while attached) is +// enforced by DetachedRwLock: guards are only reachable inside its +// detach-first `with` closure. See detached_lock.rs for the rationale and +// the residual hole. +#[pyclass(frozen, name = "Tokenizer", module = "tokenizers")] +pub struct PyTokenizer { + inner: DetachedRwLock, } -struct PyArrayUnicode(Vec); -impl<'a, 'py> FromPyObject<'a, 'py> for PyArrayUnicode { - type Error = PyErr; - - fn extract(ob: Borrowed<'a, 'py, PyAny>) -> Result { - // SAFETY Making sure the pointer is a valid numpy array requires calling numpy C code - if unsafe { npyffi::PyArray_Check(ob.py(), ob.as_ptr()) } == 0 { - return Err(exceptions::PyTypeError::new_err("Expected an np.array")); - } - let arr = ob.as_ptr() as *mut npyffi::PyArrayObject; - // SAFETY Getting all the metadata about the numpy array to check its sanity - let (type_num, elsize, _alignment, data, nd, flags) = unsafe { - let desc = (*arr).descr; - ( - (*desc).type_num, - npyffi::PyDataType_ELSIZE(ob.py(), desc) as usize, - npyffi::PyDataType_ALIGNMENT(ob.py(), desc) as usize, - (*arr).data, - (*arr).nd, - (*arr).flags, - ) - }; - - if nd != 1 { - return Err(exceptions::PyTypeError::new_err( - "Expected a 1 dimensional np.array", - )); - } - if flags & (npyffi::NPY_ARRAY_C_CONTIGUOUS | npyffi::NPY_ARRAY_F_CONTIGUOUS) == 0 { - return Err(exceptions::PyTypeError::new_err( - "Expected a contiguous np.array", - )); - } - if type_num != npyffi::types::NPY_TYPES::NPY_UNICODE as i32 { - return Err(exceptions::PyTypeError::new_err( - "Expected a np.array[dtype='U']", - )); - } - - // SAFETY Looking at the raw numpy data to create new owned Rust strings via copies (so it's safe afterwards). - unsafe { - let n_elem = *(*arr).dimensions as usize; - let all_bytes = std::slice::from_raw_parts(data as *const u8, elsize * n_elem); - - let seq = (0..n_elem) - .map(|i| { - let bytes = &all_bytes[i * elsize..(i + 1) * elsize]; - Ok(std::str::from_utf8(bytes) - .map_err(|e| exceptions::PyValueError::new_err(e.to_string()))? - .to_owned()) - // let unicode = pyo3::ffi::PyUnicode_FromKindAndData( - // pyo3::ffi::PyUnicode_4BYTE_KIND as _, - // bytes.as_ptr() as *const _, - // elsize as isize / alignment as isize, - // ); - // let py = ob.py(); - // let obj = Py::from_owned_ptr(py, unicode); - // let s = obj.downcast_bound::(py)?; - // Ok(s.to_string_lossy().trim_matches(char::from(0)).to_owned()) - }) - .collect::>>()?; - - Ok(Self(seq)) +impl PyTokenizer { + fn from_spec(spec: SpecTokenizer) -> Self { + Self { + inner: DetachedRwLock::new(Inner { + spec, + compiled: None, + }), } } -} -impl From for tk::InputSequence<'_> { - fn from(s: PyArrayUnicode) -> Self { - s.0.into() - } -} - -struct PyArrayStr(Vec); - -impl<'a, 'py> FromPyObject<'a, 'py> for PyArrayStr { - type Error = PyErr; - - fn extract(ob: Borrowed<'a, 'py, PyAny>) -> Result { - let array = ob.cast::>>()?; - let seq = array - .readonly() - .as_array() - .iter() - .map(|obj| { - let s = obj.cast_bound::(ob.py())?; - Ok(s.to_string_lossy().into_owned()) - }) - .collect::>>()?; - - Ok(Self(seq)) - } -} -impl From for tk::InputSequence<'_> { - fn from(s: PyArrayStr) -> Self { - s.0.into() - } -} - -struct PreTokenizedInputSequence<'s>(tk::InputSequence<'s>); -impl<'a, 'py> FromPyObject<'a, 'py> for PreTokenizedInputSequence<'py> { - type Error = PyErr; - fn extract(ob: Borrowed<'a, 'py, PyAny>) -> Result { - if let Ok(seq) = ob.extract::() { - return Ok(Self(seq.into())); - } - if let Ok(seq) = ob.extract::() { - return Ok(Self(seq.into())); - } - if let Ok(s) = ob.cast::() - && let Ok(seq) = s.extract::>() - { - return Ok(Self(seq.into())); - } - if let Ok(s) = ob.cast::() - && let Ok(seq) = s.extract::>() - { - return Ok(Self(seq.into())); - } - Err(exceptions::PyTypeError::new_err( - "PreTokenizedInputSequence must be Union[List[str], Tuple[str]]", - )) - } -} -impl<'s> From> for tk::InputSequence<'s> { - fn from(s: PreTokenizedInputSequence<'s>) -> Self { - s.0 + fn read_spec( + &self, + py: Python<'_>, + f: impl FnOnce(&SpecTokenizer) -> T + Ungil + Send, + ) -> PyResult { + self.inner.with(py, |lock| { + let guard = lock.read().map_err(poisoned)?; + Ok(f(&guard.spec)) + }) } -} - -struct TextEncodeInput<'s>(tk::EncodeInput<'s>); -impl<'a, 'py> FromPyObject<'a, 'py> for TextEncodeInput<'py> { - type Error = PyErr; - fn extract(ob: Borrowed<'a, 'py, PyAny>) -> Result { - if let Ok(i) = ob.extract::() { - return Ok(Self(i.into())); - } - if let Ok((i1, i2)) = ob.extract::<(TextInputSequence, TextInputSequence)>() { - return Ok(Self((i1, i2).into())); - } - if let Ok(arr) = ob.extract::>>() - && arr.len() == 2 - { - let py = ob.py(); - let first = arr[0].bind(py).extract::()?; - let second = arr[1].bind(py).extract::()?; - return Ok(Self((first, second).into())); - } - Err(exceptions::PyTypeError::new_err( - "TextEncodeInput must be Union[TextInputSequence, Tuple[InputSequence, InputSequence]]", - )) - } -} -impl<'s> From> for tk::tokenizer::EncodeInput<'s> { - fn from(i: TextEncodeInput<'s>) -> Self { - i.0 + /// Write access to the spec; invalidates the compiled pipeline. + fn mutate_spec( + &self, + py: Python<'_>, + f: impl FnOnce(&mut SpecTokenizer) -> PyResult + Ungil + Send, + ) -> PyResult { + self.inner.with(py, |lock| { + let mut guard = lock.write().map_err(poisoned)?; + let result = f(&mut guard.spec)?; + guard.compiled = None; + Ok(result) + }) } } -struct PreTokenizedEncodeInput<'s>(tk::EncodeInput<'s>); -impl<'a, 'py> FromPyObject<'a, 'py> for PreTokenizedEncodeInput<'py> { - type Error = PyErr; - fn extract(ob: Borrowed<'a, 'py, PyAny>) -> Result { - if let Ok(i) = ob.extract::() { - return Ok(Self(i.into())); - } - if let Ok((i1, i2)) = ob.extract::<(PreTokenizedInputSequence, PreTokenizedInputSequence)>() - { - return Ok(Self((i1, i2).into())); - } - if let Ok(arr) = ob.extract::>>() - && arr.len() == 2 - { - let py = ob.py(); - let first = arr[0].bind(py).extract::()?; - let second = arr[1].bind(py).extract::()?; - return Ok(Self((first, second).into())); +/// Get the compiled pipeline, building it from the spec on first use after a +/// mutation. The `Detached` parameter is the proof this runs off the GIL. +fn get_or_compile(lock: &Detached<'_, Inner>) -> PyResult { + { + let guard = lock.read().map_err(poisoned)?; + if let Some(compiled) = &guard.compiled { + return Ok(compiled.clone()); } - Err(exceptions::PyTypeError::new_err( - "PreTokenizedEncodeInput must be Union[PreTokenizedInputSequence, \ - Tuple[PreTokenizedInputSequence, PreTokenizedInputSequence]]", - )) - } -} -impl<'s> From> for tk::tokenizer::EncodeInput<'s> { - fn from(i: PreTokenizedEncodeInput<'s>) -> Self { - i.0 } -} - -type Tokenizer = TokenizerImpl; - -/// A :obj:`Tokenizer` works as a pipeline. It processes some raw text as input -/// and outputs an :class:`~tokenizers.Encoding`. -/// -/// The pipeline is structured as follows: -/// -/// 1. The :class:`~tokenizers.normalizers.Normalizer` normalizes the raw input text. -/// 2. The :class:`~tokenizers.pre_tokenizers.PreTokenizer` splits the normalized text -/// into word-level tokens. -/// 3. The :class:`~tokenizers.models.Model` tokenizes each word into subword tokens -/// and maps them to IDs. -/// 4. The :class:`~tokenizers.processors.PostProcessor` applies any final -/// transformations (e.g., adding special tokens like ``[CLS]`` and ``[SEP]``). -/// -/// Args: -/// model (:class:`~tokenizers.models.Model`): -/// The core algorithm that this :obj:`Tokenizer` should be using. -/// -/// Example:: -/// -/// >>> from tokenizers import Tokenizer -/// >>> from tokenizers.models import BPE -/// >>> from tokenizers.normalizers import Lowercase -/// >>> from tokenizers.pre_tokenizers import Whitespace -/// >>> tokenizer = Tokenizer(BPE(unk_token="")) -/// >>> tokenizer.normalizer = Lowercase() -/// >>> tokenizer.pre_tokenizer = Whitespace() -/// >>> # Load a pre-built tokenizer from HuggingFace Hub -/// >>> tokenizer = Tokenizer.from_pretrained("bert-base-uncased") -/// -#[pyclass( - dict, - weakref, - module = "tokenizers", - name = "Tokenizer", - from_py_object -)] -pub struct PyTokenizer { - /// `Arc` so cloning is a refcount bump (matches the pre-RwLock semantics - /// where the inner tokenizer was shared across `PyTokenizer` clones). - /// `RwLock` so concurrent setters and encoders don't race PyO3's - /// per-pyclass borrow check on free-threaded Python. - pub(crate) tokenizer: Arc>, -} - -impl Clone for PyTokenizer { - fn clone(&self) -> Self { - PyTokenizer { - tokenizer: Arc::clone(&self.tokenizer), - } + let mut guard = lock.write().map_err(poisoned)?; + if guard.compiled.is_none() { + let pipe = PipelineTokenizer::try_from(&guard.spec).map_err(|e| { + TokenizersError::new_err(format!( + "this tokenizer cannot be compiled to an encode pipeline: {e}" + )) + })?; + let post_adds_special_tokens = guard + .spec + .get_post_processor() + .is_some_and(|p| p.added_tokens(false) > 0); + guard.compiled = Some(Compiled { + pipe: Arc::new(pipe), + post_adds_special_tokens, + }); } + Ok(guard.compiled.clone().expect("just set")) } -impl Serialize for PyTokenizer { - fn serialize(&self, serializer: S) -> Result { - // Equivalent to the previous `#[serde(transparent)]` derive — forward - // through to the inner Tokenizer. - let guard = self - .tokenizer - .read() - .map_err(|_| S::Error::custom("Tokenizer RwLock is poisoned"))?; - guard.serialize(serializer) +fn check_special_tokens_flag(compiled: &Compiled, add_special_tokens: bool) -> PyResult<()> { + if add_special_tokens && compiled.post_adds_special_tokens { + return Err(PyNotImplementedError::new_err( + "this tokenizer's post-processor adds special tokens, but post-processing is not \ + implemented in the encode pipeline yet; pass add_special_tokens=False to encode \ + without them", + )); } + Ok(()) } -impl PyTokenizer { - fn new(tokenizer: Tokenizer) -> Self { - PyTokenizer { - tokenizer: Arc::new(RwLock::new(tokenizer)), - } - } - - /// Acquire the inner tokenizer for reading; surfaces lock poisoning as a - /// `PyException` instead of panicking. - pub(crate) fn read_inner(&self) -> PyResult> { - self.tokenizer - .read() - .map_err(|_| exceptions::PyException::new_err("Tokenizer RwLock is poisoned")) - } - - /// Acquire the inner tokenizer for writing; surfaces lock poisoning as a - /// `PyException` instead of panicking. - pub(crate) fn write_inner(&self) -> PyResult> { - self.tokenizer - .write() - .map_err(|_| exceptions::PyException::new_err("Tokenizer RwLock is poisoned")) - } - - fn from_model(model: PyModel) -> Self { - PyTokenizer::new(TokenizerImpl::new(model)) - } - - // Extract a pretokenized sequence into an owned Vec - fn extract_pretok_seq(ob: &Bound<'_, PyAny>) -> PyResult> { - if let Ok(seq) = ob.extract::() { - return Ok(seq.0); - } - if let Ok(seq) = ob.extract::() { - return Ok(seq.0); - } - if let Ok(list) = ob.cast::() { - return list.extract::>(); - } - if let Ok(tup) = ob.cast::() { - return tup.extract::>(); - } - Err(exceptions::PyTypeError::new_err( - "PreTokenizedInputSequence must be List[str] | Tuple[str] | np.ndarray[U] | np.ndarray[object[str]]", - )) - } - - // Convert Python inputs into fully-owned EncodeInput<'static> - fn build_owned_encode_inputs( - items: &[Bound<'_, PyAny>], - is_pretokenized: bool, - ) -> PyResult>> { - let mut out = Vec::with_capacity(items.len()); - - for it in items { - if is_pretokenized { - // Pair? - if let Ok(tup) = it.cast::() - && tup.len() == 2 - { - let a = Self::extract_pretok_seq(&tup.get_item(0)?)?; - let b = Self::extract_pretok_seq(&tup.get_item(1)?)?; - out.push(tk::EncodeInput::Dual(a.into(), b.into())); - continue; - } - if let Ok(lst) = it.cast::() - && lst.len() == 2 - { - let a = Self::extract_pretok_seq(&lst.get_item(0)?)?; - let b = Self::extract_pretok_seq(&lst.get_item(1)?)?; - out.push(tk::EncodeInput::Dual(a.into(), b.into())); - continue; - } - // Single pretokenized - let a = Self::extract_pretok_seq(it)?; - out.push(tk::EncodeInput::Single(a.into())); - } else { - // Raw text: pair? - if let Ok(tup) = it.cast::() - && tup.len() == 2 - { - let a: String = tup.get_item(0)?.extract()?; - let b: String = tup.get_item(1)?.extract()?; - out.push(tk::EncodeInput::Dual(a.into(), b.into())); - continue; - } - if let Ok(lst) = it.cast::() - && lst.len() == 2 - && lst.get_item(0)?.cast::().is_ok() - && lst.get_item(1)?.cast::().is_ok() - { - let a: String = lst.get_item(0)?.extract()?; - let b: String = lst.get_item(1)?.extract()?; - out.push(tk::EncodeInput::Dual(a.into(), b.into())); - continue; - } - // Single raw text - let s: String = it.extract()?; - out.push(tk::EncodeInput::Single(s.into())); - } - } - Ok(out) - } - - // Helper method to build a single owned encode input - fn build_single_owned_encode_input( - sequence: &Bound<'_, PyAny>, - pair: Option<&Bound<'_, PyAny>>, - is_pretokenized: bool, - ) -> PyResult> { - let owned_sequence: tk::InputSequence<'static> = if is_pretokenized { - let seq = Self::extract_pretok_seq(sequence)?; - seq.into() - } else { - let s: String = sequence.extract()?; - s.into() - }; - - if let Some(pair) = pair { - let owned_pair: tk::InputSequence<'static> = if is_pretokenized { - let seq = Self::extract_pretok_seq(pair)?; - seq.into() - } else { - let s: String = pair.extract()?; - s.into() - }; - Ok(tk::EncodeInput::Dual(owned_sequence, owned_pair)) - } else { - Ok(tk::EncodeInput::Single(owned_sequence)) - } - } +fn encode_one( + pipe: &PipelineTokenizer, + text: &str, + pre_tokens: &mut Vec, + scratch: &mut PipelineModelScratch, +) -> PyResult> { + let mut output: Vec = Vec::new(); + pipe.encode_generic::<{ PipelineTokenizer::STAGE_MODEL }>( + text, + pre_tokens, + scratch, + &mut output, + ) + .map_err(to_pyerr)?; + Ok(output.iter().map(|t| t.id).collect()) } #[pymethods] impl PyTokenizer { + /// Create an untrained tokenizer from a model. #[new] - #[pyo3(text_signature = "(self, model)")] - fn __new__(model: PyRef) -> Self { - PyTokenizer::from_model(model.clone()) + fn new(model: PyRef<'_, PyModel>) -> Self { + Self::from_spec(SpecTokenizer::new(model.inner.clone())) } - fn __getstate__(&self, py: Python) -> PyResult> { - let data = serde_json::to_string(&*self.read_inner()?).map_err(|e| { - exceptions::PyException::new_err(format!( - "Error while attempting to pickle Tokenizer: {e}" - )) - })?; - Ok(PyBytes::new(py, data.as_bytes()).into()) - } - - fn __setstate__(&self, py: Python, state: Py) -> PyResult<()> { - match state.extract::<&[u8]>(py) { - Ok(s) => { - *self.write_inner()? = serde_json::from_slice(s).map_err(|e| { - exceptions::PyException::new_err(format!( - "Error while attempting to unpickle Tokenizer: {e}" - )) - })?; - Ok(()) - } - Err(e) => Err(e.into()), - } - } - - fn __getnewargs__<'p>(&self, py: Python<'p>) -> PyResult> { - let model: Py = PyModel::from(BPE::default()) - .into_pyobject(py)? - .into_any() - .into(); - PyTuple::new(py, vec![model]) - } - - /// Instantiate a new :class:`~tokenizers.Tokenizer` from the given JSON string. - /// - /// Args: - /// json (:obj:`str`): - /// A valid JSON string representing a previously serialized - /// :class:`~tokenizers.Tokenizer` - /// - /// Returns: - /// :class:`~tokenizers.Tokenizer`: The new tokenizer - #[staticmethod] - #[pyo3(signature = (json) -> "Tokenizer")] - #[pyo3(text_signature = "(json)")] - fn from_str(json: &str) -> PyResult { - let tokenizer: PyResult<_> = ToPyResult(json.parse()).into(); - Ok(Self::new(tokenizer?)) - } - - /// Instantiate a new :class:`~tokenizers.Tokenizer` from the file at the given path. - /// - /// Args: - /// path (:obj:`str`): - /// A path to a local JSON file representing a previously serialized - /// :class:`~tokenizers.Tokenizer` - /// - /// Returns: - /// :class:`~tokenizers.Tokenizer`: The new tokenizer + /// Load a tokenizer from a `tokenizer.json` file. #[staticmethod] #[pyo3(signature = (path) -> "Tokenizer")] - #[pyo3(text_signature = "(path)")] - fn from_file(path: &str) -> PyResult { - let tokenizer: PyResult<_> = ToPyResult(Tokenizer::from_file(path)).into(); - Ok(Self::new(tokenizer?)) + fn from_file(py: Python<'_>, path: PathBuf) -> PyResult { + let spec = py + .detach(|| SpecTokenizer::from_file(path)) + .map_err(to_pyerr)?; + Ok(Self::from_spec(spec)) } - /// Instantiate a new :class:`~tokenizers.Tokenizer` from the given buffer. - /// - /// Args: - /// buffer (:obj:`bytes`): - /// A buffer containing a previously serialized :class:`~tokenizers.Tokenizer` - /// - /// Returns: - /// :class:`~tokenizers.Tokenizer`: The new tokenizer + /// Load a tokenizer from the bytes of a `tokenizer.json` file. #[staticmethod] #[pyo3(signature = (buffer) -> "Tokenizer")] - #[pyo3(text_signature = "(buffer)")] - fn from_buffer(buffer: &Bound<'_, PyBytes>) -> PyResult { - let tokenizer = serde_json::from_slice(buffer.as_bytes()).map_err(|e| { - exceptions::PyValueError::new_err(format!( - "Cannot instantiate Tokenizer from buffer: {e}" - )) - })?; - Ok(Self { tokenizer }) + fn from_buffer(py: Python<'_>, buffer: Vec) -> PyResult { + let spec = py + .detach(|| SpecTokenizer::from_bytes(&buffer)) + .map_err(to_pyerr)?; + Ok(Self::from_spec(spec)) } - /// Instantiate a new :class:`~tokenizers.Tokenizer` from an existing file on the - /// Hugging Face Hub. - /// - /// Args: - /// identifier (:obj:`str`): - /// The identifier of a Model on the Hugging Face Hub, that contains - /// a tokenizer.json file - /// revision (:obj:`str`, defaults to `main`): - /// A branch or commit id - /// token (:obj:`str`, `optional`, defaults to `None`): - /// An optional auth token used to access private repositories on the - /// Hugging Face Hub - /// - /// Returns: - /// :class:`~tokenizers.Tokenizer`: The new tokenizer + /// Download `tokenizer.json` from a model on the Hugging Face Hub (requires + /// the `huggingface_hub` package) and load it. #[staticmethod] - #[pyo3(signature = (identifier, revision = String::from("main"), token = None) -> "Tokenizer")] - #[pyo3(text_signature = "(identifier, revision=\"main\", token=None)")] + #[pyo3(signature = (identifier, *, revision = String::from("main"), token = None) -> "Tokenizer")] fn from_pretrained( + py: Python<'_>, identifier: &str, revision: String, token: Option, ) -> PyResult { - let path = Python::attach(|py| -> PyResult { - let huggingface_hub = PyModule::import(py, intern!(py, "huggingface_hub"))?; - let hf_hub_download = huggingface_hub.getattr(intern!(py, "hf_hub_download"))?; - let kwargs = [ - (intern!(py, "repo_id"), identifier), - (intern!(py, "filename"), "tokenizer.json"), - (intern!(py, "revision"), &revision), - ] - .into_py_dict(py)?; - if let Some(token) = token { - kwargs.set_item(intern!(py, "token"), token)?; - } - let path: String = hf_hub_download.call((), Some(&kwargs))?.extract()?; - Ok(path) - })?; - - let tokenizer: PyResult<_> = ToPyResult(Tokenizer::from_file(path)).into(); - Ok(Self::new(tokenizer?)) + let hub = py.import("huggingface_hub")?; + let kwargs = pyo3::types::PyDict::new(py); + kwargs.set_item("repo_id", identifier)?; + kwargs.set_item("filename", "tokenizer.json")?; + kwargs.set_item("revision", revision)?; + kwargs.set_item("token", token)?; + let path: PathBuf = hub + .getattr("hf_hub_download")? + .call((), Some(&kwargs))? + .extract()?; + Self::from_file(py, path) } - /// Gets a serialized string representing this :class:`~tokenizers.Tokenizer`. - /// - /// Args: - /// pretty (:obj:`bool`, defaults to :obj:`False`): - /// Whether the JSON string should be pretty formatted. - /// - /// Returns: - /// :obj:`str`: A string representing the serialized Tokenizer - #[pyo3(signature = (pretty = false) -> "str")] - #[pyo3(text_signature = "(self, pretty=False)")] - fn to_str(&self, pretty: bool) -> PyResult { - ToPyResult(self.read_inner()?.to_string(pretty)).into() + /// Serialize the tokenizer definition as a `tokenizer.json` string. + #[pyo3(signature = (*, pretty = false))] + fn to_str(&self, py: Python<'_>, pretty: bool) -> PyResult { + self.read_spec(py, move |spec| spec.to_string(pretty).map_err(to_pyerr))? } - /// Save the :class:`~tokenizers.Tokenizer` to the file at the given path. - /// - /// Args: - /// path (:obj:`str`): - /// A path to a file in which to save the serialized tokenizer. - /// - /// pretty (:obj:`bool`, defaults to :obj:`True`): - /// Whether the JSON file should be pretty formatted. - #[pyo3(signature = (path, pretty = true) -> "None")] - #[pyo3(text_signature = "(self, path, pretty=True)")] - fn save(&self, path: &str, pretty: bool) -> PyResult<()> { - ToPyResult(self.read_inner()?.save(path, pretty)).into() + /// Save the tokenizer definition to a `tokenizer.json` file. + #[pyo3(signature = (path, *, pretty = true))] + fn save(&self, py: Python<'_>, path: PathBuf, pretty: bool) -> PyResult<()> { + self.read_spec(py, move |spec| spec.save(path, pretty).map_err(to_pyerr))? } - fn __repr__(&self) -> PyResult { - crate::utils::serde_pyo3::repr(self) - .map_err(|e| exceptions::PyException::new_err(e.to_string())) - } - - fn __str__(&self) -> PyResult { - crate::utils::serde_pyo3::to_string(self) - .map_err(|e| exceptions::PyException::new_err(e.to_string())) - } - - /// Return the number of special tokens that would be added for single/pair sentences. - /// :param is_pair: Boolean indicating if the input would be a single sentence or a pair - /// :return: - #[pyo3(text_signature = "(self, is_pair)")] - fn num_special_tokens_to_add(&self, is_pair: bool) -> usize { - self.tokenizer - .read() - .unwrap() - .get_post_processor() - .map_or(0, |p| p.added_tokens(is_pair)) - } - - /// Get the underlying vocabulary - /// - /// Args: - /// with_added_tokens (:obj:`bool`, defaults to :obj:`True`): - /// Whether to include the added tokens - /// - /// Returns: - /// :obj:`Dict[str, int]`: The vocabulary - #[pyo3(signature = (with_added_tokens = true) -> "dict[str, int]")] - #[pyo3(text_signature = "(self, with_added_tokens=True)")] - fn get_vocab(&self, with_added_tokens: bool) -> PyResult> { - Ok(self.read_inner()?.get_vocab(with_added_tokens)) - } - - /// Get the underlying vocabulary - /// - /// Returns: - /// :obj:`Dict[int, AddedToken]`: The vocabulary - #[pyo3(signature = () -> "dict[int, AddedToken]")] - #[pyo3(text_signature = "(self)")] - fn get_added_tokens_decoder(&self) -> PyResult> { - let mut sorted_map = BTreeMap::new(); - - for (key, value) in self.read_inner()?.get_added_tokens_decoder() { - sorted_map.insert(key, value.into()); - } - - Ok(sorted_map) - } - - /// Get the size of the underlying vocabulary - /// - /// Args: - /// with_added_tokens (:obj:`bool`, defaults to :obj:`True`): - /// Whether to include the added tokens + /// Encode `text` into token ids. /// - /// Returns: - /// :obj:`int`: The size of the vocabulary - #[pyo3(signature = (with_added_tokens = true) -> "int")] - #[pyo3(text_signature = "(self, with_added_tokens=True)")] - fn get_vocab_size(&self, with_added_tokens: bool) -> usize { - self.tokenizer - .read() - .unwrap() - .get_vocab_size(with_added_tokens) - } - - /// Enable truncation - /// - /// Args: - /// max_length (:obj:`int`): - /// The max length at which to truncate - /// - /// stride (:obj:`int`, `optional`): - /// The length of the previous first sequence to be included in the overflowing - /// sequence - /// - /// strategy (:obj:`str`, `optional`, defaults to :obj:`longest_first`): - /// The strategy used to truncation. Can be one of ``longest_first``, ``only_first`` or - /// ``only_second``. - /// - /// direction (:obj:`str`, defaults to :obj:`right`): - /// Truncate direction - #[pyo3(signature = (max_length, **kwargs) -> "None")] - #[pyo3( - text_signature = "(self, max_length, stride=0, strategy='longest_first', direction='right')" - )] - fn enable_truncation( - &self, - max_length: usize, - kwargs: Option<&Bound<'_, PyDict>>, - ) -> PyResult<()> { - let mut params = TruncationParams { - max_length, - ..Default::default() - }; - - if let Some(kwargs) = kwargs { - for (key, value) in kwargs { - let key: String = key.extract()?; - match key.as_ref() { - "stride" => params.stride = value.extract()?, - "strategy" => { - let value: String = value.extract()?; - params.strategy = match value.as_ref() { - "longest_first" => Ok(TruncationStrategy::LongestFirst), - "only_first" => Ok(TruncationStrategy::OnlyFirst), - "only_second" => Ok(TruncationStrategy::OnlySecond), - _ => Err(PyError(format!( - "Unknown `strategy`: `{value}`. Use \ - one of `longest_first`, `only_first`, or `only_second`" - )) - .into_pyerr::()), - }? - } - "direction" => { - let value: String = value.extract()?; - params.direction = match value.as_ref() { - "left" => Ok(TruncationDirection::Left), - "right" => Ok(TruncationDirection::Right), - _ => Err(PyError(format!( - "Unknown `direction`: `{value}`. Use \ - one of `left` or `right`." - )) - .into_pyerr::()), - }? - } - _ => println!("Ignored unknown kwarg option {key}"), - } - } - } - - if let Err(error_message) = self - .tokenizer - .write() - .unwrap() - .with_truncation(Some(params)) - { - return Err(PyError(error_message.to_string()).into_pyerr::()); - } - Ok(()) - } - - /// Disable truncation - #[pyo3(text_signature = "(self)")] - fn no_truncation(&self) { - self.tokenizer - .write() - .unwrap() - .with_truncation(None) - .expect("Failed to set truncation to `None`! This should never happen"); - } - - /// Get the currently set truncation parameters - /// - /// `Cannot set, use` :meth:`~tokenizers.Tokenizer.enable_truncation` `instead` - /// - /// Returns: - /// (:obj:`dict`, `optional`): - /// A dict with the current truncation parameters if truncation is enabled - #[getter] - fn get_truncation<'py>(&self, py: Python<'py>) -> PyResult>> { - self.tokenizer - .read() - .unwrap() - .get_truncation() - .map_or(Ok(None), |params| { - let dict = PyDict::new(py); - - dict.set_item("max_length", params.max_length)?; - dict.set_item("stride", params.stride)?; - dict.set_item("strategy", params.strategy.as_ref())?; - dict.set_item("direction", params.direction.as_ref())?; - - Ok(Some(dict)) - }) - } - - /// Enable the padding - /// - /// Args: - /// direction (:obj:`str`, `optional`, defaults to :obj:`right`): - /// The direction in which to pad. Can be either ``right`` or ``left`` - /// - /// pad_to_multiple_of (:obj:`int`, `optional`): - /// If specified, the padding length should always snap to the next multiple of the - /// given value. For example if we were going to pad witha length of 250 but - /// ``pad_to_multiple_of=8`` then we will pad to 256. - /// - /// pad_id (:obj:`int`, defaults to 0): - /// The id to be used when padding - /// - /// pad_type_id (:obj:`int`, defaults to 0): - /// The type id to be used when padding - /// - /// pad_token (:obj:`str`, defaults to :obj:`[PAD]`): - /// The pad token to be used when padding - /// - /// length (:obj:`int`, `optional`): - /// If specified, the length at which to pad. If not specified we pad using the size of - /// the longest sequence in a batch. - #[pyo3(signature = (**kwargs) -> "None")] - #[pyo3( - text_signature = "(self, direction='right', pad_id=0, pad_type_id=0, pad_token='[PAD]', length=None, pad_to_multiple_of=None)" - )] - fn enable_padding(&self, kwargs: Option<&Bound<'_, PyDict>>) -> PyResult<()> { - let mut params = PaddingParams::default(); - - if let Some(kwargs) = kwargs { - for (key, value) in kwargs { - let key: String = key.extract()?; - match key.as_ref() { - "direction" => { - let value: String = value.extract()?; - params.direction = match value.as_ref() { - "left" => Ok(PaddingDirection::Left), - "right" => Ok(PaddingDirection::Right), - other => Err(PyError(format!( - "Unknown `direction`: `{other}`. Use \ - one of `left` or `right`" - )) - .into_pyerr::()), - }?; - } - "pad_to_multiple_of" => { - if let Some(multiple) = value.extract()? { - params.pad_to_multiple_of = multiple; - } - } - "pad_id" => params.pad_id = value.extract()?, - "pad_type_id" => params.pad_type_id = value.extract()?, - "pad_token" => params.pad_token = value.extract()?, - "max_length" => { - println!( - "enable_padding(max_length=X) is deprecated, \ - use enable_padding(length=X) instead" - ); - if let Some(l) = value.extract()? { - params.strategy = PaddingStrategy::Fixed(l); - } else { - params.strategy = PaddingStrategy::BatchLongest; - } - } - "length" => { - if let Some(l) = value.extract()? { - params.strategy = PaddingStrategy::Fixed(l); - } else { - params.strategy = PaddingStrategy::BatchLongest; - } - } - _ => println!("Ignored unknown kwarg option {key}"), - } - } - } - - self.write_inner()?.with_padding(Some(params)); - - Ok(()) - } - - /// Disable padding - #[pyo3(text_signature = "(self)")] - fn no_padding(&self) -> PyResult<()> { - self.write_inner()?.with_padding(None); - Ok(()) - } - - /// Get the current padding parameters - /// - /// `Cannot be set, use` :meth:`~tokenizers.Tokenizer.enable_padding` `instead` - /// - /// Returns: - /// (:obj:`dict`, `optional`): - /// A dict with the current padding parameters if padding is enabled - #[getter] - fn get_padding<'py>(&self, py: Python<'py>) -> PyResult>> { - self.tokenizer - .read() - .unwrap() - .get_padding() - .map_or(Ok(None), |params| { - let dict = PyDict::new(py); - - dict.set_item( - "length", - match params.strategy { - tk::PaddingStrategy::BatchLongest => None, - tk::PaddingStrategy::Fixed(size) => Some(size), - }, - )?; - dict.set_item("pad_to_multiple_of", params.pad_to_multiple_of)?; - dict.set_item("pad_id", params.pad_id)?; - dict.set_item("pad_token", ¶ms.pad_token)?; - dict.set_item("pad_type_id", params.pad_type_id)?; - dict.set_item("direction", params.direction.as_ref())?; - - Ok(Some(dict)) - }) - } - - /// Encode the given sequence and pair. This method can process raw text sequences - /// as well as already pre-tokenized sequences. - /// - /// Example: - /// Here are some examples of the inputs that are accepted:: - /// - /// encode("A single sequence")` - /// encode("A sequence", "And its pair")` - /// encode([ "A", "pre", "tokenized", "sequence" ], is_pretokenized=True)` - /// encode( - /// [ "A", "pre", "tokenized", "sequence" ], [ "And", "its", "pair" ], - /// is_pretokenized=True - /// ) - /// - /// Args: - /// sequence (:obj:`~tokenizers.InputSequence`): - /// The main input sequence we want to encode. This sequence can be either raw - /// text or pre-tokenized, according to the ``is_pretokenized`` argument: - /// - /// - If ``is_pretokenized=False``: :class:`~tokenizers.TextInputSequence` - /// - If ``is_pretokenized=True``: :class:`~tokenizers.PreTokenizedInputSequence` - /// - /// pair (:obj:`~tokenizers.InputSequence`, `optional`): - /// An optional input sequence. The expected format is the same that for ``sequence``. - /// - /// is_pretokenized (:obj:`bool`, defaults to :obj:`False`): - /// Whether the input is already pre-tokenized - /// - /// add_special_tokens (:obj:`bool`, defaults to :obj:`True`): - /// Whether to add the special tokens - /// - /// Returns: - /// :class:`~tokenizers.Encoding`: The encoded result - /// - #[pyo3(signature = (sequence, pair = None, is_pretokenized = false, add_special_tokens = true) -> "Encoding")] - #[pyo3( - text_signature = "(self, sequence, pair=None, is_pretokenized=False, add_special_tokens=True)" - )] - fn encode( - &self, - sequence: &Bound<'_, PyAny>, - pair: Option<&Bound<'_, PyAny>>, - is_pretokenized: bool, - add_special_tokens: bool, - ) -> PyResult { - let sequence: tk::InputSequence = if is_pretokenized { - sequence.extract::()?.into() - } else { - sequence.extract::()?.into() - }; - let input = match pair { - Some(pair) => { - let pair: tk::InputSequence = if is_pretokenized { - pair.extract::()?.into() - } else { - pair.extract::()?.into() - }; - tk::EncodeInput::Dual(sequence, pair) - } - None => tk::EncodeInput::Single(sequence), - }; - - ToPyResult( - self.tokenizer - .read() - .unwrap() - .encode_char_offsets(input, add_special_tokens) - .map(|e| e.into()), - ) - .into() - } - - /// Asynchronously encode the given input with character offsets. - /// - /// This is an async version of encode that can be awaited in async Python code. - /// - /// Example: - /// Here are some examples of the inputs that are accepted:: - /// - /// await async_encode("A single sequence") - /// - /// Args: - /// sequence (:obj:`~tokenizers.InputSequence`): - /// The main input sequence we want to encode. This sequence can be either raw - /// text or pre-tokenized, according to the ``is_pretokenized`` argument: - /// - /// - If ``is_pretokenized=False``: :class:`~tokenizers.TextInputSequence` - /// - If ``is_pretokenized=True``: :class:`~tokenizers.PreTokenizedInputSequence` - /// - /// pair (:obj:`~tokenizers.InputSequence`, `optional`): - /// An optional input sequence. The expected format is the same that for ``sequence``. - /// - /// is_pretokenized (:obj:`bool`, defaults to :obj:`False`): - /// Whether the input is already pre-tokenized - /// - /// add_special_tokens (:obj:`bool`, defaults to :obj:`True`): - /// Whether to add the special tokens - /// - /// Returns: - /// :class:`~tokenizers.Encoding`: The encoded result - /// - #[pyo3(signature = (sequence, pair = None, is_pretokenized = false, add_special_tokens = true))] - #[pyo3( - text_signature = "(self, sequence, pair=None, is_pretokenized=False, add_special_tokens=True)" - )] - fn async_encode<'py>( + /// Runs entirely outside the interpreter lock and returns a `numpy.uint32` + /// array backed by the Rust output buffer (no copy). + #[pyo3(signature = (text, *, add_special_tokens = true) -> "npt.NDArray[np.uint32]")] + fn encode<'py>( &self, py: Python<'py>, - sequence: &Bound<'_, PyAny>, - pair: Option<&Bound<'_, PyAny>>, - is_pretokenized: bool, + text: &str, add_special_tokens: bool, - ) -> PyResult> { - // Extract and fully own the inputs before leaving the GIL/thread - let input = Self::build_single_owned_encode_input(sequence, pair, is_pretokenized)?; - - let tokenizer = self.read_inner()?.clone(); - let rt = crate::TOKIO_RUNTIME.clone(); - - let fut = py.detach(|| async move { - rt.spawn_blocking(move || { - tokenizer - .encode(input, add_special_tokens) - .map(PyEncoding::from) - }) - .await - .unwrap() - .map_err(|e| exceptions::PyException::new_err(e.to_string())) - }); - - pyo3_async_runtimes::tokio::future_into_py(py, fut) + ) -> PyResult>> { + let ids = self.inner.with(py, |lock| -> PyResult> { + let compiled = get_or_compile(&lock)?; + check_special_tokens_flag(&compiled, add_special_tokens)?; + let mut pre_tokens = Vec::new(); + let mut scratch = compiled.pipe.get_model().init_scratch(); + encode_one(&compiled.pipe, text, &mut pre_tokens, &mut scratch) + })?; + Ok(ids.into_pyarray(py)) } - /// Encode the given batch of inputs. This method accept both raw text sequences - /// as well as already pre-tokenized sequences. The reason we use `PySequence` is - /// because it allows type checking with zero-cost (according to PyO3) as we don't - /// have to convert to check. - /// - /// Example: - /// Here are some examples of the inputs that are accepted:: - /// - /// encode_batch([ - /// "A single sequence", - /// ("A tuple with a sequence", "And its pair"), - /// [ "A", "pre", "tokenized", "sequence" ], - /// ([ "A", "pre", "tokenized", "sequence" ], "And its pair") - /// ]) - /// - /// Args: - /// input (A :obj:`List`/:obj:`Tuple` of :obj:`~tokenizers.EncodeInput`): - /// A list of single sequences or pair sequences to encode. Each sequence - /// can be either raw text or pre-tokenized, according to the ``is_pretokenized`` - /// argument: - /// - /// - If ``is_pretokenized=False``: :class:`~tokenizers.TextEncodeInput` - /// - If ``is_pretokenized=True``: :class:`~tokenizers.PreTokenizedEncodeInput` - /// - /// is_pretokenized (:obj:`bool`, defaults to :obj:`False`): - /// Whether the input is already pre-tokenized - /// - /// add_special_tokens (:obj:`bool`, defaults to :obj:`True`): - /// Whether to add the special tokens - /// - /// Returns: - /// A :obj:`List` of :class:`~tokenizers.Encoding`: The encoded batch - /// - #[pyo3(signature = (input, is_pretokenized = false, add_special_tokens = true) -> "list[Encoding]")] - #[pyo3(text_signature = "(self, input, is_pretokenized=False, add_special_tokens=True)")] - fn encode_batch( - &self, - py: Python<'_>, - input: Vec>, - is_pretokenized: bool, - add_special_tokens: bool, - ) -> PyResult> { - let mut items = Vec::::with_capacity(input.len()); - for item in &input { - let item: tk::EncodeInput = if is_pretokenized { - item.extract::()?.into() - } else { - item.extract::()?.into() - }; - items.push(item); - } - py.detach(|| { - ToPyResult( - self.tokenizer - .read() - .unwrap() - .encode_batch_char_offsets(items, add_special_tokens) - .map(|encodings| encodings.into_iter().map(|e| e.into()).collect()), - ) - .into() - }) - } - /// Asynchronously encode the given batch of inputs with character offsets. - /// - /// This is an async version of encode_batch that can be awaited in async Python code. - /// - /// Example: - /// Here are some examples of the inputs that are accepted:: - /// - /// await async_encode_batch([ - /// "A single sequence", - /// ("A tuple with a sequence", "And its pair"), - /// [ "A", "pre", "tokenized", "sequence" ], - /// ([ "A", "pre", "tokenized", "sequence" ], "And its pair") - /// ]) - /// - /// Args: - /// input (A :obj:`List`/:obj:`Tuple` of :obj:`~tokenizers.EncodeInput`): - /// A list of single sequences or pair sequences to encode. Each sequence - /// can be either raw text or pre-tokenized, according to the ``is_pretokenized`` - /// argument: - /// - /// - If ``is_pretokenized=False``: :class:`~tokenizers.TextEncodeInput` - /// - If ``is_pretokenized=True``: :class:`~tokenizers.PreTokenizedEncodeInput` - /// - /// is_pretokenized (:obj:`bool`, defaults to :obj:`False`): - /// Whether the input is already pre-tokenized - /// - /// add_special_tokens (:obj:`bool`, defaults to :obj:`True`): - /// Whether to add the special tokens - /// - /// Returns: - /// A :obj:`List` of :class:`~tokenizers.Encoding`: The encoded batch - /// - #[pyo3(name = "async_encode_batch", signature = (input, is_pretokenized = false, add_special_tokens = true))] - #[pyo3(text_signature = "(self, input, is_pretokenized=False, add_special_tokens=True)")] - fn async_encode_batch<'py>( + /// Encode a batch of texts, in parallel across Rust threads (respects + /// `TOKENIZERS_PARALLELISM`), without holding the interpreter lock. + /// Input strings are borrowed, not copied; each output is a `numpy.uint32` + /// array backed by its Rust buffer. + #[pyo3(signature = (texts, *, add_special_tokens = true) -> "list[npt.NDArray[np.uint32]]")] + fn encode_batch<'py>( &self, py: Python<'py>, - input: Vec>, - is_pretokenized: bool, - add_special_tokens: bool, - ) -> PyResult> { - // Fully own the inputs before leaving the GIL/thread - let owned_items = Self::build_owned_encode_inputs(&input, is_pretokenized)?; - - let tokenizer = self.read_inner()?.clone(); - let rt = crate::TOKIO_RUNTIME.clone(); - - let fut = py.detach(|| async move { - rt.spawn_blocking(move || { - tokenizer - .encode_batch_char_offsets(owned_items, add_special_tokens) - .map(|encs| encs.into_iter().map(PyEncoding::from).collect::>()) - }) - .await - .unwrap() - .map_err(|e| exceptions::PyException::new_err(e.to_string())) - }); - - pyo3_async_runtimes::tokio::future_into_py(py, fut) - } - - /// Encode the given batch of inputs. This method is faster than `encode_batch` - /// because it doesn't keep track of offsets, they will be all zeros. - /// - /// Example: - /// Here are some examples of the inputs that are accepted:: - /// - /// encode_batch_fast([ - /// "A single sequence", - /// ("A tuple with a sequence", "And its pair"), - /// [ "A", "pre", "tokenized", "sequence" ], - /// ([ "A", "pre", "tokenized", "sequence" ], "And its pair") - /// ]) - /// - /// Args: - /// input (A :obj:`List`/:obj:`Tuple` of :obj:`~tokenizers.EncodeInput`): - /// A list of single sequences or pair sequences to encode. Each sequence - /// can be either raw text or pre-tokenized, according to the ``is_pretokenized`` - /// argument: - /// - /// - If ``is_pretokenized=False``: :class:`~tokenizers.TextEncodeInput` - /// - If ``is_pretokenized=True``: :class:`~tokenizers.PreTokenizedEncodeInput` - /// - /// is_pretokenized (:obj:`bool`, defaults to :obj:`False`): - /// Whether the input is already pre-tokenized - /// - /// add_special_tokens (:obj:`bool`, defaults to :obj:`True`): - /// Whether to add the special tokens - /// - /// Returns: - /// A :obj:`List` of :class:`~tokenizers.Encoding`: The encoded batch - /// - #[pyo3(signature = (input, is_pretokenized = false, add_special_tokens = true) -> "list[Encoding]")] - #[pyo3(text_signature = "(self, input, is_pretokenized=False, add_special_tokens=True)")] - fn encode_batch_fast( - &self, - py: Python<'_>, - input: Vec>, - is_pretokenized: bool, + texts: Vec, add_special_tokens: bool, - ) -> PyResult> { - let mut items = Vec::::with_capacity(input.len()); - for item in &input { - let item: tk::EncodeInput = if is_pretokenized { - item.extract::()?.into() + ) -> PyResult> { + let batches = self.inner.with(py, |lock| -> PyResult>> { + let compiled = get_or_compile(&lock)?; + check_special_tokens_flag(&compiled, add_special_tokens)?; + if get_parallelism() && texts.len() > 1 { + USED_PARALLELISM.store(true, Ordering::SeqCst); + texts + .par_iter() + .map_init( + || (Vec::new(), compiled.pipe.get_model().init_scratch()), + |(pre_tokens, scratch), text| { + encode_one(&compiled.pipe, text, pre_tokens, scratch) + }, + ) + .collect() } else { - item.extract::()?.into() - }; - items.push(item); - } - py.detach(|| { - ToPyResult( - self.tokenizer - .read() - .unwrap() - .encode_batch_fast(items, add_special_tokens) - .map(|encodings| encodings.into_iter().map(|e| e.into()).collect()), - ) - .into() - }) - } - - /// Asynchronously encode the given batch of inputs without tracking character offsets. - /// - /// This is an async version of encode_batch_fast that can be awaited in async Python code. - /// - /// Example: - /// Here are some examples of the inputs that are accepted:: - /// - /// await async_encode_batch_fast([ - /// "A single sequence", - /// ("A tuple with a sequence", "And its pair"), - /// [ "A", "pre", "tokenized", "sequence" ], - /// ([ "A", "pre", "tokenized", "sequence" ], "And its pair") - /// ]) - /// - /// Args: - /// input (A :obj:`List`/:obj:`Tuple` of :obj:`~tokenizers.EncodeInput`): - /// A list of single sequences or pair sequences to encode. Each sequence - /// can be either raw text or pre-tokenized, according to the ``is_pretokenized`` - /// argument: - /// - /// - If ``is_pretokenized=False``: :class:`~tokenizers.TextEncodeInput` - /// - If ``is_pretokenized=True``: :class:`~tokenizers.PreTokenizedEncodeInput` - /// - /// is_pretokenized (:obj:`bool`, defaults to :obj:`False`): - /// Whether the input is already pre-tokenized - /// - /// add_special_tokens (:obj:`bool`, defaults to :obj:`True`): - /// Whether to add the special tokens - /// - /// Returns: - /// A :obj:`List` of :class:`~tokenizers.Encoding`: The encoded batch - /// - #[pyo3(name = "async_encode_batch_fast", signature = (input, is_pretokenized = false, add_special_tokens = true))] - #[pyo3(text_signature = "(self, input, is_pretokenized=False, add_special_tokens=True)")] - fn async_encode_batch_fast<'py>( - &self, - py: Python<'py>, - input: Vec>, - is_pretokenized: bool, - add_special_tokens: bool, - ) -> PyResult> { - let owned_items = Self::build_owned_encode_inputs(&input, is_pretokenized)?; - - let tokenizer = self.read_inner()?.clone(); - let rt = crate::TOKIO_RUNTIME.clone(); - let fut = py.detach(|| async move { - let result = rt - .spawn_blocking(move || { - tokenizer - .encode_batch_fast(owned_items, add_special_tokens) - .map(|encs| encs.into_iter().map(PyEncoding::from).collect::>()) - }) - .await - .unwrap(); - - // Convert to a Python object directly rather than going through ToPyResult - match result { - Ok(encodings) => Python::attach(|py| encodings.into_py_any(py)), - Err(e) => Err(exceptions::PyException::new_err(e.to_string())), + let mut pre_tokens = Vec::new(); + let mut scratch = compiled.pipe.get_model().init_scratch(); + texts + .iter() + .map(|text| encode_one(&compiled.pipe, text, &mut pre_tokens, &mut scratch)) + .collect() } - }); - - pyo3_async_runtimes::tokio::future_into_py(py, fut) + })?; + let list = PyList::empty(py); + for ids in batches { + list.append(ids.into_pyarray(py))?; + } + Ok(list) } - /// Decode the given list of ids back to a string - /// - /// This is used to decode anything coming back from a Language Model - /// - /// Args: - /// ids (A :obj:`List/Tuple` of :obj:`int`): - /// The list of ids that we want to decode - /// - /// skip_special_tokens (:obj:`bool`, defaults to :obj:`True`): - /// Whether the special tokens should be removed from the decoded string - /// - /// Returns: - /// :obj:`str`: The decoded string - #[pyo3(signature = (ids, skip_special_tokens = true) -> "str")] - #[pyo3(text_signature = "(self, ids, skip_special_tokens=True)")] + /// Not implemented yet: decoding is not part of the encode pipeline. + #[pyo3(signature = (ids, *, skip_special_tokens = true))] + #[allow(unused_variables)] fn decode(&self, ids: Vec, skip_special_tokens: bool) -> PyResult { - ToPyResult( - self.tokenizer - .read() - .unwrap() - .decode(&ids, skip_special_tokens), - ) - .into() + Err(PyNotImplementedError::new_err( + "decode is not implemented in the encode pipeline yet", + )) } - /// Decode a batch of ids back to their corresponding string - /// - /// Args: - /// sequences (:obj:`List` of :obj:`List[int]`): - /// The batch of sequences we want to decode - /// - /// skip_special_tokens (:obj:`bool`, defaults to :obj:`True`): - /// Whether the special tokens should be removed from the decoded strings - /// - /// Returns: - /// :obj:`List[str]`: A list of decoded strings - #[pyo3(signature = (sequences, skip_special_tokens = true) -> "list[str]")] - #[pyo3(text_signature = "(self, sequences, skip_special_tokens=True)")] - fn decode_batch( + /// Train the model's vocabulary on text files (one sequence per line). + /// Without a `trainer`, the model's default trainer is used. + #[pyo3(signature = (files, *, trainer = None))] + fn train( &self, py: Python<'_>, - sequences: Vec>, - skip_special_tokens: bool, - ) -> PyResult> { - py.detach(|| { - let slices = sequences.iter().map(|v| &v[..]).collect::>(); - ToPyResult( - self.tokenizer - .read() - .unwrap() - .decode_batch(&slices, skip_special_tokens), - ) - .into() + files: Vec, + trainer: Option>, + ) -> PyResult<()> { + let explicit = trainer.map(|t| t.inner.clone()); + self.inner.with(py, |lock| { + let mut guard = lock.write().map_err(poisoned)?; + let mut trainer = explicit.unwrap_or_else(|| guard.spec.get_model().get_trainer()); + guard + .spec + .train_from_files(&mut trainer, files) + .map_err(to_pyerr)?; + guard.compiled = None; + Ok(()) }) } - /// Decode a batch of ids back to their corresponding string + /// Train the model's vocabulary from any iterator of `str`. Without a + /// `trainer`, the model's default trainer is used. /// - /// Args: - /// sequences (:obj:`List` of :obj:`List[int]`): - /// The batch of sequences we want to decode - /// - /// skip_special_tokens (:obj:`bool`, defaults to :obj:`True`): - /// Whether the special tokens should be removed from the decoded strings - /// - /// Returns: - /// :obj:`List[str]`: A list of decoded strings - #[pyo3(name = "async_decode_batch", signature = (sequences, skip_special_tokens = true))] - #[pyo3(text_signature = "(self, sequences, skip_special_tokens=True)")] - fn async_decode_batch<'py>( + /// The interpreter lock is only re-acquired to refill an internal buffer + /// (256 sequences at a time); the training itself runs multi-threaded in + /// Rust with the lock released. + #[pyo3(signature = (iterator, *, trainer = None))] + fn train_from_iterator( &self, - py: Python<'py>, - sequences: Vec>, - skip_special_tokens: bool, - ) -> PyResult> { - let tokenizer = self.read_inner()?.clone(); - let rt = crate::TOKIO_RUNTIME.clone(); - - let fut = py.detach(|| async move { - rt.spawn_blocking(move || { - let slices = sequences.iter().map(|v| &v[..]).collect::>(); - tokenizer.decode_batch(&slices, skip_special_tokens) - }) - .await - .unwrap() - .map_err(|e| exceptions::PyException::new_err(e.to_string())) - }); - - pyo3_async_runtimes::tokio::future_into_py(py, fut) - } - - /// Convert the given token to its corresponding id if it exists - /// - /// Args: - /// token (:obj:`str`): - /// The token to convert - /// - /// Returns: - /// :obj:`Optional[int]`: An optional id, :obj:`None` if out of vocabulary - #[pyo3(signature = (token) -> "int | None", text_signature = "(self, token)")] - fn token_to_id(&self, token: &str) -> PyResult> { - Ok(self.read_inner()?.token_to_id(token)) - } - - /// Convert the given id to its corresponding token if it exists - /// - /// Args: - /// id (:obj:`int`): - /// The id to convert - /// - /// Returns: - /// :obj:`Optional[str]`: An optional token, :obj:`None` if out of vocabulary - #[pyo3(signature = (id) -> "str | None", text_signature = "(self, id)")] - fn id_to_token(&self, id: u32) -> PyResult> { - Ok(self.read_inner()?.id_to_token(id)) - } - - /// Modifies the tokenizer in order to use or not the special tokens - /// during encoding. - /// - /// Args: - /// value (:obj:`bool`): - /// Whether to use the special tokens or not - /// - #[setter] - fn set_encode_special_tokens(&self, value: bool) { - self.tokenizer - .write() - .unwrap() - .set_encode_special_tokens(value); - } - /// Get the value of the `encode_special_tokens` attribute - /// - /// Returns: - /// :obj:`bool`: the tokenizer's encode_special_tokens attribute - #[getter] - fn get_encode_special_tokens(&self) -> PyResult { - Ok(self.read_inner()?.get_encode_special_tokens()) + py: Python<'_>, + iterator: &Bound<'_, PyAny>, + trainer: Option>, + ) -> PyResult<()> { + let explicit = trainer.map(|t| t.inner.clone()); + let sequences = BufferedPyIterator::new(iterator)?; + let error = sequences.error.clone(); + self.inner.with(py, |lock| { + USED_PARALLELISM.store(true, Ordering::SeqCst); + let mut guard = lock.write().map_err(poisoned)?; + let mut trainer = explicit.unwrap_or_else(|| guard.spec.get_model().get_trainer()); + guard + .spec + .train(&mut trainer, sequences) + .map_err(to_pyerr)?; + guard.compiled = None; + Ok::<_, PyErr>(()) + })?; + if let Some(err) = error.lock().expect("error slot poisoned").take() { + return Err(err); + } + Ok(()) } - /// Add the given tokens to the vocabulary - /// - /// The given tokens are added only if they don't already exist in the vocabulary. - /// Each token then gets a new attributed id. - /// - /// Args: - /// tokens (A :obj:`List` of :class:`~tokenizers.AddedToken` or :obj:`str`): - /// The list of tokens we want to add to the vocabulary. Each token can be either a - /// string or an instance of :class:`~tokenizers.AddedToken` for more customization. - /// - /// Returns: - /// :obj:`int`: The number of tokens that were created in the vocabulary - #[pyo3(text_signature = "(self, tokens)")] - fn add_tokens(&self, tokens: &Bound<'_, PyList>) -> PyResult { - let tokens = tokens - .into_iter() - .map(|token| { - if let Ok(content) = token.extract::() { - Ok(PyAddedToken::from(content, Some(false)).get_token()) - } else if let Ok(token) = token.extract::>() { - Ok(token.get_token()) - } else { - Err(exceptions::PyTypeError::new_err( - "Input must be a List[Union[str, AddedToken]]", - )) - } - }) - .collect::>>()?; - ToPyResult(self.write_inner()?.add_tokens(tokens)).into() + /// Add tokens to the vocabulary and match them in the input text from now + /// on. Plain strings match with default options; pass `AddedToken` to + /// control matching. Returns how many were actually new. + fn add_tokens(&self, py: Python<'_>, tokens: Vec) -> PyResult { + let tokens = parse_tokens(tokens, false); + self.mutate_spec(py, move |spec| spec.add_tokens(tokens).map_err(to_pyerr)) } - /// Add the given special tokens to the Tokenizer. - /// - /// If these tokens are already part of the vocabulary, it just let the Tokenizer know about - /// them. If they don't exist, the Tokenizer creates them, giving them a new id. - /// - /// These special tokens will never be processed by the model (ie won't be split into - /// multiple tokens), and they can be removed from the output when decoding. - /// - /// Args: - /// tokens (A :obj:`List` of :class:`~tokenizers.AddedToken` or :obj:`str`): - /// The list of special tokens we want to add to the vocabulary. Each token can either - /// be a string or an instance of :class:`~tokenizers.AddedToken` for more - /// customization. - /// - /// Returns: - /// :obj:`int`: The number of tokens that were created in the vocabulary - #[pyo3(text_signature = "(self, tokens)")] - fn add_special_tokens(&self, tokens: &Bound<'_, PyList>) -> PyResult { - let tokens = tokens - .into_iter() - .map(|token| { - if let Ok(content) = token.extract::() { - Ok(tk::tokenizer::AddedToken::from(content, true)) - } else if let Ok(mut token) = token.extract::>() { - token.special = true; - Ok(token.get_token()) - } else { - Err(exceptions::PyTypeError::new_err( - "Input must be a List[Union[str, AddedToken]]", - )) - } - }) - .collect::>>()?; - - ToPyResult(self.write_inner()?.add_special_tokens(tokens)).into() - } - - /// Train the Tokenizer using the given files. - /// - /// Reads the files line by line, while keeping all the whitespace, even new lines. - /// If you want to train from data store in-memory, you can check - /// :meth:`~tokenizers.Tokenizer.train_from_iterator` - /// - /// Args: - /// files (:obj:`List[str]`): - /// A list of path to the files that we should use for training - /// - /// trainer (:obj:`~tokenizers.trainers.Trainer`, `optional`): - /// An optional trainer that should be used to train our Model - #[pyo3(signature = (files, trainer = None))] - #[pyo3(text_signature = "(self, files, trainer = None)")] - fn train(&self, files: Vec, trainer: Option<&mut PyTrainer>) -> PyResult<()> { - let mut trainer = match trainer { - Some(t) => t.clone(), - None => self.read_inner()?.get_model().get_trainer(), - }; - Python::attach(|py| { - py.detach(|| { - ToPyResult( - self.write_inner()? - .train_from_files(&mut trainer, files) - .map(|_| {}), - ) - .into() - }) + /// Add special tokens ("", "[CLS]", …) to the vocabulary. Same as + /// `add_tokens`, but every token is marked `special`. Returns how many + /// were actually new. + fn add_special_tokens(&self, py: Python<'_>, tokens: Vec) -> PyResult { + let tokens = parse_tokens(tokens, true); + self.mutate_spec(py, move |spec| { + spec.add_special_tokens(tokens).map_err(to_pyerr) }) } - /// Train the Tokenizer using the provided iterator. - /// - /// You can provide anything that is a Python Iterator - /// - /// * A list of sequences :obj:`List[str]` - /// * A generator that yields :obj:`str` or :obj:`List[str]` - /// * A Numpy array of strings - /// * ... - /// - /// Args: - /// iterator (:obj:`Iterator`): - /// Any iterator over strings or list of strings - /// - /// trainer (:obj:`~tokenizers.trainers.Trainer`, `optional`): - /// An optional trainer that should be used to train our Model - /// - /// length (:obj:`int`, `optional`): - /// The total number of sequences in the iterator. This is used to - /// provide meaningful progress tracking - #[pyo3(signature = (iterator, trainer = None, length = None))] - #[pyo3(text_signature = "(self, iterator, trainer=None, length=None)")] - fn train_from_iterator( - &self, - py: Python, - iterator: &Bound<'_, PyAny>, - trainer: Option<&mut PyTrainer>, - length: Option, - ) -> PyResult<()> { - let mut trainer = match trainer { - Some(t) => t.clone(), - None => self.read_inner()?.get_model().get_trainer(), - }; - - let buffered_iter = PyBufferedIterator::new( - iterator, - |element| { - // Each element of the iterator can either be: - // - An iterator, to allow batching - // - A string - if let Ok(s) = element.cast::() { - itertools::Either::Right(std::iter::once(s.to_cow().map(|s| s.into_owned()))) - } else { - match element.try_iter() { - Ok(iter) => itertools::Either::Left( - iter.map(|i| i?.extract::()) - .collect::>() - .into_iter(), - ), - Err(e) => itertools::Either::Right(std::iter::once(Err(e))), - } - } - }, - 256, - )?; - - py.detach(|| { - ResultShunt::process(buffered_iter, |iter| { - self.tokenizer - .write() - .unwrap() - .train(&mut trainer, MaybeSizedIterator::new(iter, length)) - .map(|_| {}) - .map_err(|e| exceptions::PyException::new_err(e.to_string())) - })? - }) + /// The id of `token`, or None if it is not in the vocabulary. + fn token_to_id(&self, py: Python<'_>, token: &str) -> PyResult> { + self.read_spec(py, |spec| spec.token_to_id(token)) } - /// Apply all the post-processing steps to the given encodings. - /// - /// The various steps are: - /// - /// 1. Truncate according to the set truncation params (provided with - /// :meth:`~tokenizers.Tokenizer.enable_truncation`) - /// 2. Apply the :class:`~tokenizers.processors.PostProcessor` - /// 3. Pad according to the set padding params (provided with - /// :meth:`~tokenizers.Tokenizer.enable_padding`) - /// - /// Args: - /// encoding (:class:`~tokenizers.Encoding`): - /// The :class:`~tokenizers.Encoding` corresponding to the main sequence. - /// - /// pair (:class:`~tokenizers.Encoding`, `optional`): - /// An optional :class:`~tokenizers.Encoding` corresponding to the pair sequence. - /// - /// add_special_tokens (:obj:`bool`): - /// Whether to add the special tokens - /// - /// Returns: - /// :class:`~tokenizers.Encoding`: The final post-processed encoding - #[pyo3(signature = (encoding, pair = None, add_special_tokens = true))] - #[pyo3(text_signature = "(self, encoding, pair=None, add_special_tokens=True)")] - fn post_process( - &self, - encoding: &PyEncoding, - pair: Option<&PyEncoding>, - add_special_tokens: bool, - ) -> PyResult { - ToPyResult( - self.tokenizer - .read() - .unwrap() - .post_process( - encoding.encoding.clone(), - pair.map(|p| p.encoding.clone()), - add_special_tokens, - ) - .map(|e| e.into()), - ) - .into() + /// The token behind `id`, or None if the id is out of range. + fn id_to_token(&self, py: Python<'_>, id: u32) -> PyResult> { + self.read_spec(py, move |spec| spec.id_to_token(id)) } - /// The :class:`~tokenizers.models.Model` in use by the Tokenizer - #[getter] - fn get_model(&self, py: Python<'_>) -> PyResult> { - self.tokenizer - .read() - .unwrap() - .get_model() - .get_as_subtype(py) + /// The whole vocabulary as a dict. This copies every entry; prefer + /// `token_to_id` for lookups. + #[pyo3(signature = (*, with_added_tokens = true))] + fn get_vocab(&self, py: Python<'_>, with_added_tokens: bool) -> PyResult> { + self.read_spec(py, move |spec| spec.get_vocab(with_added_tokens)) } - /// Set the :class:`~tokenizers.models.Model` - #[setter] - fn set_model(&self, model: PyRef) -> PyResult<()> { - self.write_inner()?.with_model(model.clone()); - Ok(()) + /// Number of entries in the vocabulary. `with_added_tokens=False` counts + /// only what the model was trained with. + #[pyo3(signature = (*, with_added_tokens = true))] + fn get_vocab_size(&self, py: Python<'_>, with_added_tokens: bool) -> PyResult { + self.read_spec(py, move |spec| spec.get_vocab_size(with_added_tokens)) } - /// The `optional` :class:`~tokenizers.normalizers.Normalizer` in use by the Tokenizer + /// The model in use by this tokenizer (a copy: reassign to change it). #[getter] - fn get_normalizer(&self, py: Python<'_>) -> PyResult> { - if let Some(n) = self.read_inner()?.get_normalizer() { - n.get_as_subtype(py) - } else { - Ok(py.None()) - } + fn model(&self, py: Python<'_>) -> PyResult> { + let model = self.read_spec(py, |spec| spec.get_model().clone())?; + wrap_model(py, model) } - /// Set the :class:`~tokenizers.normalizers.Normalizer` #[setter] - fn set_normalizer(&self, normalizer: Option>) -> PyResult<()> { - let normalizer_option = normalizer.map(|norm| norm.clone()); - ToPyResult( - self.tokenizer - .write() - .unwrap() - .with_normalizer(normalizer_option) - .map(|_| ()), - ) - .into() + fn set_model(&self, py: Python<'_>, model: PyRef<'_, PyModel>) -> PyResult<()> { + let model = model.inner.clone(); + self.mutate_spec(py, move |spec| { + spec.with_model(model); + Ok(()) + }) } - /// The `optional` :class:`~tokenizers.pre_tokenizers.PreTokenizer` in use by the Tokenizer + /// The optional normalizer in use by this tokenizer (a copy: reassign to + /// change it). #[getter] - fn get_pre_tokenizer(&self, py: Python<'_>) -> PyResult> { - if let Some(pt) = self.read_inner()?.get_pre_tokenizer() { - pt.get_as_subtype(py) - } else { - Ok(py.None()) - } + fn normalizer(&self, py: Python<'_>) -> PyResult>> { + let normalizer = self.read_spec(py, |spec| spec.get_normalizer().cloned())?; + normalizer.map(|n| wrap_normalizer(py, n)).transpose() } - /// Set the :class:`~tokenizers.normalizers.Normalizer` #[setter] - fn set_pre_tokenizer(&self, pretok: Option>) { - self.tokenizer - .write() - .unwrap() - .with_pre_tokenizer(pretok.map(|pre| pre.clone())); + fn set_normalizer( + &self, + py: Python<'_>, + normalizer: Option>, + ) -> PyResult<()> { + let normalizer = normalizer.map(|n| n.inner.clone()); + self.mutate_spec(py, move |spec| { + spec.with_normalizer(normalizer).map_err(to_pyerr)?; + Ok(()) + }) } - /// The `optional` :class:`~tokenizers.processors.PostProcessor` in use by the Tokenizer + /// The optional pre-tokenizer in use by this tokenizer (a copy: reassign + /// to change it). #[getter] - fn get_post_processor(&self, py: Python<'_>) -> PyResult> { - if let Some(n) = self.read_inner()?.get_post_processor() { - n.get_as_subtype(py) - } else { - Ok(py.None()) - } + fn pre_tokenizer(&self, py: Python<'_>) -> PyResult>> { + let pre_tokenizer = self.read_spec(py, |spec| spec.get_pre_tokenizer().cloned())?; + pre_tokenizer.map(|p| wrap_pre_tokenizer(py, p)).transpose() } - /// Set the :class:`~tokenizers.processors.PostProcessor` #[setter] - fn set_post_processor(&self, processor: Option>) { - self.tokenizer - .write() - .unwrap() - .with_post_processor(processor.map(|p| p.clone())); + fn set_pre_tokenizer( + &self, + py: Python<'_>, + pre_tokenizer: Option>, + ) -> PyResult<()> { + let pre_tokenizer = pre_tokenizer.map(|p| p.inner.clone()); + self.mutate_spec(py, move |spec| { + spec.with_pre_tokenizer(pre_tokenizer); + Ok(()) + }) } - /// The `optional` :class:`~tokenizers.decoders.Decoder` in use by the Tokenizer - #[getter] - fn get_decoder(&self, py: Python<'_>) -> PyResult> { - if let Some(dec) = self.read_inner()?.get_decoder() { - dec.get_as_subtype(py) - } else { - Ok(py.None()) - } + fn __repr__(&self, py: Python<'_>) -> PyResult { + self.read_spec(py, |spec| { + format!( + "Tokenizer(model={}, vocab_size={})", + match spec.get_model() { + tk_encode::ModelWrapper::BPE(_) => "BPE", + tk_encode::ModelWrapper::WordPiece(_) => "WordPiece", + tk_encode::ModelWrapper::WordLevel(_) => "WordLevel", + tk_encode::ModelWrapper::Unigram(_) => "Unigram", + }, + spec.get_vocab_size(true) + ) + }) } - /// Set the :class:`~tokenizers.decoders.Decoder` - #[setter] - fn set_decoder(&self, decoder: Option>) { - self.tokenizer - .write() - .unwrap() - .with_decoder(decoder.map(|d| d.clone())); + fn __reduce__<'py>( + &self, + py: Python<'py>, + ) -> PyResult<(Bound<'py, PyAny>, (Bound<'py, PyBytes>,))> { + let data = self.to_str(py, false)?; + let from_buffer = py.get_type::().getattr("from_buffer")?; + Ok((from_buffer, (PyBytes::new(py, data.as_bytes()),))) } } -#[cfg(test)] -mod test { - use super::*; - use crate::models::PyModel; - use crate::normalizers::{PyNormalizer, PyNormalizerTypeWrapper}; - use std::sync::{Arc, RwLock}; - use tempfile::NamedTempFile; - use tk::normalizers::{Lowercase, NFKC}; +/// Pulls a Python iterator of `str` from Rust threads: re-attaches to the +/// interpreter only to refill an internal buffer, `CHUNK` items at a time. +/// A conversion error stops the stream and is stashed in `error` for the +/// caller to surface once training finishes. +struct BufferedPyIterator { + iterator: Py, + buffer: std::collections::VecDeque, + finished: bool, + error: Arc>>, +} - #[test] - fn serialize() { - let mut tokenizer = Tokenizer::new(PyModel::from(BPE::default())); - tokenizer - .with_normalizer(Some(PyNormalizer::new(PyNormalizerTypeWrapper::Sequence( - vec![ - Arc::new(RwLock::new(NFKC.into())), - Arc::new(RwLock::new(Lowercase.into())), - ], - )))) - .unwrap(); +impl BufferedPyIterator { + const CHUNK: usize = 256; - let tmp = NamedTempFile::new().unwrap().into_temp_path(); - tokenizer.save(&tmp, false).unwrap(); + fn new(iterable: &Bound<'_, PyAny>) -> PyResult { + Ok(Self { + iterator: iterable.try_iter()?.unbind().into(), + buffer: std::collections::VecDeque::with_capacity(Self::CHUNK), + finished: false, + error: Arc::new(Mutex::new(None)), + }) + } - Tokenizer::from_file(&tmp).unwrap(); + // The vetted lock-then-GIL direction: the caller (train) holds the write + // lock and re-attaches here. Safe because no attached thread can be + // blocking on the lock — DetachedRwLock makes that unrepresentable. + #[allow(clippy::disallowed_methods)] + fn refill(&mut self) { + let result = Python::attach(|py| -> PyResult { + let iterator = self.iterator.bind(py); + for _ in 0..Self::CHUNK { + match iterator.call_method0("__next__") { + Ok(item) => { + let sequence = item.extract::().map_err(|_| { + PyTypeError::new_err("train_from_iterator expects an iterator of str") + })?; + self.buffer.push_back(sequence); + } + Err(e) if e.is_instance_of::(py) => return Ok(true), + Err(e) => return Err(e), + } + } + Ok(false) + }); + match result { + Ok(done) => self.finished = done, + Err(e) => { + *self.error.lock().expect("error slot poisoned") = Some(e); + self.finished = true; + } + } } +} - #[test] - fn serde_pyo3() { - let mut tokenizer = Tokenizer::new(PyModel::from(BPE::default())); - tokenizer - .with_normalizer(Some(PyNormalizer::new(PyNormalizerTypeWrapper::Sequence( - vec![ - Arc::new(RwLock::new(NFKC.into())), - Arc::new(RwLock::new(Lowercase.into())), - ], - )))) - .unwrap(); +impl Iterator for BufferedPyIterator { + type Item = String; - let output = crate::utils::serde_pyo3::to_string(&tokenizer).unwrap(); - assert_eq!( - output, - "Tokenizer(version=\"1.0\", truncation=None, padding=None, added_tokens=[], normalizer=Sequence(normalizers=[NFKC(), Lowercase()]), pre_tokenizer=None, post_processor=None, decoder=None, model=BPE(dropout=None, unk_token=None, continuing_subword_prefix=None, end_of_word_suffix=None, fuse_unk=False, byte_fallback=False, ignore_merges=False, vocab={}, merges=[]))" - ); + fn next(&mut self) -> Option { + if self.buffer.is_empty() && !self.finished { + self.refill(); + } + self.buffer.pop_front() } } diff --git a/bindings/python/src/trainers.rs b/bindings/python/src/trainers.rs index 79e75ae9e..90bb4e83b 100644 --- a/bindings/python/src/trainers.rs +++ b/bindings/python/src/trainers.rs @@ -1,1781 +1,190 @@ -use std::sync::{Arc, RwLock}; - -use crate::models::PyModel; -use crate::tokenizer::PyAddedToken; -#[cfg(feature = "parity-aware-bpe")] -use crate::tokenizer::PyTokenizer; -use pyo3::exceptions; use pyo3::prelude::*; -use pyo3::types::*; -use serde::{Deserialize, Serialize}; -use tk::Trainer; -use tk::models::TrainerWrapper; -use tk::utils::ProgressFormat; -use tokenizers as tk; +use tk_train::trainers::{ + BpeTrainer, TrainerWrapper, UnigramTrainer, WordLevelTrainer, WordPieceTrainer, +}; + +use crate::added_token::{TokenInput, parse_tokens}; +use crate::error::to_pyerr; -/// Base class for all trainers +/// Base class for all trainers. /// -/// This class is not supposed to be instantiated directly. Instead, any implementation of a -/// Trainer will return an instance of this class when instantiated. -#[pyclass( - module = "tokenizers.trainers", - name = "Trainer", - subclass, - from_py_object -)] -#[derive(Clone, Deserialize, Serialize)] -#[serde(transparent)] +/// A trainer is the recipe for learning a model's vocabulary from text; pass +/// one to `Tokenizer.train` or `train_from_iterator`. Trainers are plain +/// configuration values — training copies them and writes nothing back. +#[pyclass(frozen, subclass, name = "Trainer", module = "tokenizers.trainers")] pub struct PyTrainer { - pub trainer: Arc>, + pub inner: TrainerWrapper, } -impl PyTrainer { - #[cfg(test)] - pub(crate) fn new(trainer: Arc>) -> Self { - PyTrainer { trainer } - } - pub(crate) fn get_as_subtype(&self, py: Python<'_>) -> PyResult> { - let base = self.clone(); - Ok(match *self.trainer.as_ref().read().unwrap() { - TrainerWrapper::BpeTrainer(_) => Py::new(py, (PyBpeTrainer {}, base))?.into_any(), - TrainerWrapper::WordPieceTrainer(_) => { - Py::new(py, (PyWordPieceTrainer {}, base))?.into_any() - } - TrainerWrapper::WordLevelTrainer(_) => { - Py::new(py, (PyWordLevelTrainer {}, base))?.into_any() - } - TrainerWrapper::UnigramTrainer(_) => { - Py::new(py, (PyUnigramTrainer {}, base))?.into_any() - } - }) - } -} #[pymethods] impl PyTrainer { - fn __getstate__(&self, py: Python) -> PyResult> { - let data = serde_json::to_string(&self.trainer).map_err(|e| { - exceptions::PyException::new_err(format!( - "Error while attempting to pickle PyTrainer: {e}" - )) - })?; - Ok(PyBytes::new(py, data.as_bytes()).into()) - } - - fn __setstate__(&mut self, py: Python, state: Py) -> PyResult<()> { - match state.extract::<&[u8]>(py) { - Ok(s) => { - let unpickled = serde_json::from_slice(s).map_err(|e| { - exceptions::PyException::new_err(format!( - "Error while attempting to unpickle PyTrainer: {e}" - )) - })?; - self.trainer = unpickled; - Ok(()) - } - Err(e) => Err(e.into()), - } - } - - fn __repr__(&self) -> PyResult { - crate::utils::serde_pyo3::repr(self) - .map_err(|e| exceptions::PyException::new_err(e.to_string())) - } - - fn __str__(&self) -> PyResult { - crate::utils::serde_pyo3::to_string(self) - .map_err(|e| exceptions::PyException::new_err(e.to_string())) - } -} - -impl Trainer for PyTrainer { - type Model = PyModel; - - fn should_show_progress(&self) -> bool { - self.trainer.read().unwrap().should_show_progress() - } - - fn train(&self, model: &mut PyModel) -> tk::Result> { - self.trainer - .read() - .unwrap() - .train(&mut model.model.write().unwrap()) - } - - fn feed(&mut self, iterator: I, process: F) -> tk::Result<()> - where - I: Iterator + Send, - S: AsRef + Send, - F: Fn(&str) -> tk::Result> + Sync, - { - self.trainer.write().unwrap().feed(iterator, process) - } -} - -impl From for PyTrainer -where - I: Into, -{ - fn from(trainer: I) -> Self { - PyTrainer { - trainer: Arc::new(RwLock::new(trainer.into())), - } + fn __repr__(&self) -> String { + crate::component_repr(&self.inner) } } -macro_rules! getter { - ($self: ident, $variant: ident, $($name: tt)+) => {{ - let super_ = $self.as_ref(); - if let TrainerWrapper::$variant(ref trainer) = *super_.trainer.read().unwrap() { - trainer.$($name)+ - } else { - unreachable!() - } - }}; -} - -macro_rules! setter { - ($self: ident, $variant: ident, $name: ident, $value: expr) => {{ - let super_ = $self.as_ref(); - if let TrainerWrapper::$variant(ref mut trainer) = *super_.trainer.write().unwrap() { - trainer.$name = $value; - } - }}; - ($self: ident, $variant: ident, @$name: ident, $value: expr) => {{ - let super_ = $self.as_ref(); - if let TrainerWrapper::$variant(ref mut trainer) = *super_.trainer.write().unwrap() { - trainer.$name($value); - } - }}; -} +/// Learns a BPE vocabulary: keeps merging the most frequent pair until +/// `vocab_size` is reached, ignoring pairs seen fewer than `min_frequency` +/// times. `special_tokens` get the first ids. `limit_alphabet` caps how many +/// distinct characters are kept; `initial_alphabet` forces characters in even +/// if the data never shows them; `max_token_length` caps merged token length. +#[pyclass(frozen, extends = PyTrainer, name = "BpeTrainer", module = "tokenizers.trainers")] +pub struct PyBpeTrainer; -/// Trainer capable of training a BPE model -/// -/// Args: -/// vocab_size (:obj:`int`, `optional`): -/// The size of the final vocabulary, including all tokens and alphabet. -/// -/// min_frequency (:obj:`int`, `optional`): -/// The minimum frequency a pair should have in order to be merged. -/// -/// show_progress (:obj:`bool`, `optional`): -/// Whether to show progress bars while training. -/// -/// special_tokens (:obj:`List[Union[str, AddedToken]]`, `optional`): -/// A list of special tokens the model should know of. -/// -/// limit_alphabet (:obj:`int`, `optional`): -/// The maximum different characters to keep in the alphabet. -/// -/// initial_alphabet (:obj:`List[str]`, `optional`): -/// A list of characters to include in the initial alphabet, even -/// if not seen in the training dataset. -/// If the strings contain more than one character, only the first one -/// is kept. -/// -/// continuing_subword_prefix (:obj:`str`, `optional`): -/// A prefix to be used for every subword that is not a beginning-of-word. -/// -/// end_of_word_suffix (:obj:`str`, `optional`): -/// A suffix to be used for every subword that is a end-of-word. -/// -/// max_token_length (:obj:`int`, `optional`): -/// Prevents creating tokens longer than the specified size. -/// This can help with reducing polluting your vocabulary with -/// highly repetitive tokens like `======` for wikipedia -/// -/// Example:: -/// -/// >>> from tokenizers.models import BPE -/// >>> from tokenizers.trainers import BpeTrainer -/// >>> trainer = BpeTrainer( -/// ... vocab_size=30000, -/// ... special_tokens=["", "", ""], -/// ... min_frequency=2, -/// ... ) -/// >>> tokenizer = Tokenizer(BPE()) -/// >>> tokenizer.train(["path/to/corpus.txt"], trainer) -/// -#[pyclass(extends=PyTrainer, module = "tokenizers.trainers", name = "BpeTrainer")] -pub struct PyBpeTrainer {} #[pymethods] impl PyBpeTrainer { - #[getter] - fn get_vocab_size(self_: PyRef) -> usize { - getter!(self_, BpeTrainer, vocab_size) - } - - #[setter] - fn set_vocab_size(self_: PyRef, vocab_size: usize) { - setter!(self_, BpeTrainer, vocab_size, vocab_size); - } - - #[getter] - fn get_min_frequency(self_: PyRef) -> u64 { - getter!(self_, BpeTrainer, min_frequency) - } - - #[setter] - fn set_min_frequency(self_: PyRef, freq: u64) { - setter!(self_, BpeTrainer, min_frequency, freq); - } - - #[getter] - fn get_show_progress(self_: PyRef) -> bool { - getter!(self_, BpeTrainer, show_progress) - } - - #[setter] - fn set_show_progress(self_: PyRef, show_progress: bool) { - setter!(self_, BpeTrainer, show_progress, show_progress); - } - - /// Get the progress output format ("indicatif", "json", or "silent") - #[getter] - fn get_progress_format(self_: PyRef) -> String { - let format = getter!(self_, BpeTrainer, progress_format); - match format { - ProgressFormat::Indicatif => "indicatif".to_string(), - ProgressFormat::JsonLines => "json".to_string(), - ProgressFormat::Silent => "silent".to_string(), + #[new] + #[pyo3(signature = (*, vocab_size = 30000, min_frequency = 0, special_tokens = vec![], limit_alphabet = None, initial_alphabet = vec![], continuing_subword_prefix = None, end_of_word_suffix = None, max_token_length = None, show_progress = true))] + #[allow(clippy::too_many_arguments)] + fn new( + vocab_size: usize, + min_frequency: u64, + special_tokens: Vec, + limit_alphabet: Option, + initial_alphabet: Vec, + continuing_subword_prefix: Option, + end_of_word_suffix: Option, + max_token_length: Option, + show_progress: bool, + ) -> PyResult> { + let mut builder = BpeTrainer::builder() + .vocab_size(vocab_size) + .min_frequency(min_frequency) + .special_tokens(parse_tokens(special_tokens, true)) + .initial_alphabet(initial_alphabet.into_iter().collect()) + .show_progress(show_progress); + if let Some(limit) = limit_alphabet { + builder = builder.limit_alphabet(limit); } - } - - /// Set the progress output format ("indicatif", "json", or "silent") - #[setter] - fn set_progress_format(self_: PyRef, format: &str) { - let fmt = match format { - "json" => ProgressFormat::JsonLines, - "silent" => ProgressFormat::Silent, - _ => ProgressFormat::Indicatif, - }; - setter!(self_, BpeTrainer, progress_format, fmt); - } - - /// Get the number of unique words after feeding the corpus - #[pyo3(name = "get_word_count")] - fn get_word_count(self_: PyRef) -> usize { - let super_ = self_.as_ref(); - if let TrainerWrapper::BpeTrainer(ref trainer) = *super_.trainer.read().unwrap() { - trainer.get_word_count() - } else { - 0 + if let Some(prefix) = continuing_subword_prefix { + builder = builder.continuing_subword_prefix(prefix); } - } - - #[getter] - fn get_special_tokens(self_: PyRef) -> Vec { - getter!( - self_, - BpeTrainer, - special_tokens - .iter() - .map(|tok| tok.clone().into()) - .collect() - ) - } - - #[setter] - fn set_special_tokens(self_: PyRef, special_tokens: &Bound<'_, PyList>) -> PyResult<()> { - setter!( - self_, - BpeTrainer, - special_tokens, - special_tokens - .into_iter() - .map(|token| { - if let Ok(content) = token.extract::() { - Ok(tk::tokenizer::AddedToken::from(content, true)) - } else if let Ok(mut token) = token.extract::>() { - token.special = true; - Ok(token.get_token()) - } else { - Err(exceptions::PyTypeError::new_err( - "Special tokens must be a List[Union[str, AddedToken]]", - )) - } - }) - .collect::>>()? - ); - Ok(()) - } - - #[getter] - fn get_limit_alphabet(self_: PyRef) -> Option { - getter!(self_, BpeTrainer, limit_alphabet) - } - - #[setter] - fn set_limit_alphabet(self_: PyRef, limit: Option) { - setter!(self_, BpeTrainer, limit_alphabet, limit); - } - - #[getter] - fn get_max_token_length(self_: PyRef) -> Option { - getter!(self_, BpeTrainer, max_token_length) - } - - #[setter] - fn set_max_token_length(self_: PyRef, limit: Option) { - setter!(self_, BpeTrainer, max_token_length, limit); - } - - #[getter] - fn get_initial_alphabet(self_: PyRef) -> Vec { - getter!( - self_, - BpeTrainer, - initial_alphabet.iter().map(|c| c.to_string()).collect() - ) - } - - #[setter] - fn set_initial_alphabet(self_: PyRef, alphabet: Vec) { - setter!( - self_, - BpeTrainer, - initial_alphabet, - alphabet.into_iter().collect() - ); - } - - #[getter] - fn get_continuing_subword_prefix(self_: PyRef) -> Option { - getter!(self_, BpeTrainer, continuing_subword_prefix.clone()) - } - - #[setter] - fn set_continuing_subword_prefix(self_: PyRef, prefix: Option) { - setter!(self_, BpeTrainer, continuing_subword_prefix, prefix); - } - - #[getter] - fn get_end_of_word_suffix(self_: PyRef) -> Option { - getter!(self_, BpeTrainer, end_of_word_suffix.clone()) - } - - #[setter] - fn set_end_of_word_suffix(self_: PyRef, suffix: Option) { - setter!(self_, BpeTrainer, end_of_word_suffix, suffix); - } - - #[new] - #[pyo3( - signature = (**kwargs), - text_signature = "(self, vocab_size=30000, min_frequency=0, show_progress=True, progress_format=\"indicatif\", special_tokens=[], limit_alphabet=None, initial_alphabet=[], continuing_subword_prefix=None, end_of_word_suffix=None, max_token_length=None, words={})" - )] - pub fn new(kwargs: Option<&Bound<'_, PyDict>>) -> PyResult> { - let mut builder = tk::models::bpe::BpeTrainer::builder(); - if let Some(kwargs) = kwargs { - for (key, val) in kwargs { - let key: String = key.extract()?; - match key.as_ref() { - "vocab_size" => builder = builder.vocab_size(val.extract()?), - "min_frequency" => builder = builder.min_frequency(val.extract()?), - "show_progress" => builder = builder.show_progress(val.extract()?), - "progress_format" => { - let fmt: String = val.extract()?; - let format = match fmt.as_str() { - "json" => ProgressFormat::JsonLines, - "silent" => ProgressFormat::Silent, - _ => ProgressFormat::Indicatif, - }; - builder = builder.progress_format(format); - } - "special_tokens" => { - builder = builder.special_tokens( - val.cast::()? - .into_iter() - .map(|token| { - if let Ok(content) = token.extract::() { - Ok(PyAddedToken::from(content, Some(true)).get_token()) - } else if let Ok(mut token) = - token.extract::>() - { - token.special = true; - Ok(token.get_token()) - } else { - Err(exceptions::PyTypeError::new_err( - "special_tokens must be a List[Union[str, AddedToken]]", - )) - } - }) - .collect::>>()?, - ); - } - "limit_alphabet" => builder = builder.limit_alphabet(val.extract()?), - "max_token_length" => builder = builder.max_token_length(val.extract()?), - "initial_alphabet" => { - let alphabet: Vec = val.extract()?; - builder = builder.initial_alphabet( - alphabet - .into_iter() - .filter_map(|s| s.chars().next()) - .collect(), - ); - } - "continuing_subword_prefix" => { - builder = builder.continuing_subword_prefix(val.extract()?) - } - "end_of_word_suffix" => builder = builder.end_of_word_suffix(val.extract()?), - _ => println!("Ignored unknown kwargs option {key}"), - }; - } + if let Some(suffix) = end_of_word_suffix { + builder = builder.end_of_word_suffix(suffix); + } + if let Some(max) = max_token_length { + builder = builder.max_token_length(Some(max)); } - Ok( - PyClassInitializer::::from(PyTrainer::from(builder.build())) - .add_subclass(PyBpeTrainer {}), - ) + Ok(PyClassInitializer::from(PyTrainer { + inner: builder.build().into(), + }) + .add_subclass(PyBpeTrainer)) } } -/// Trainer capable of training a WordPiece model -/// -/// Args: -/// vocab_size (:obj:`int`, `optional`): -/// The size of the final vocabulary, including all tokens and alphabet. -/// -/// min_frequency (:obj:`int`, `optional`): -/// The minimum frequency a pair should have in order to be merged. -/// -/// show_progress (:obj:`bool`, `optional`): -/// Whether to show progress bars while training. -/// -/// special_tokens (:obj:`List[Union[str, AddedToken]]`, `optional`): -/// A list of special tokens the model should know of. -/// -/// limit_alphabet (:obj:`int`, `optional`): -/// The maximum different characters to keep in the alphabet. -/// -/// initial_alphabet (:obj:`List[str]`, `optional`): -/// A list of characters to include in the initial alphabet, even -/// if not seen in the training dataset. -/// If the strings contain more than one character, only the first one -/// is kept. -/// -/// continuing_subword_prefix (:obj:`str`, `optional`): -/// A prefix to be used for every subword that is not a beginning-of-word. -/// -/// end_of_word_suffix (:obj:`str`, `optional`): -/// A suffix to be used for every subword that is a end-of-word. -/// -/// Example:: -/// -/// >>> from tokenizers.models import WordPiece -/// >>> from tokenizers.trainers import WordPieceTrainer -/// >>> trainer = WordPieceTrainer( -/// ... vocab_size=30000, -/// ... special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"], -/// ... ) -/// >>> tokenizer = Tokenizer(WordPiece(unk_token="[UNK]")) -/// >>> tokenizer.train(["path/to/corpus.txt"], trainer) -/// -#[pyclass(extends=PyTrainer, module = "tokenizers.trainers", name = "WordPieceTrainer")] -pub struct PyWordPieceTrainer {} +/// Learns a WordPiece vocabulary. Same knobs as `BpeTrainer`, plus the +/// continuation prefix ("##" by default). +#[pyclass(frozen, extends = PyTrainer, name = "WordPieceTrainer", module = "tokenizers.trainers")] +pub struct PyWordPieceTrainer; + #[pymethods] impl PyWordPieceTrainer { - #[getter] - fn get_vocab_size(self_: PyRef) -> usize { - getter!(self_, WordPieceTrainer, vocab_size()) - } - - #[setter] - fn set_vocab_size(self_: PyRef, vocab_size: usize) { - setter!(self_, WordPieceTrainer, @set_vocab_size, vocab_size); - } - - #[getter] - fn get_min_frequency(self_: PyRef) -> u64 { - getter!(self_, WordPieceTrainer, min_frequency()) - } - - #[setter] - fn set_min_frequency(self_: PyRef, freq: u64) { - setter!(self_, WordPieceTrainer, @set_min_frequency, freq); - } - - #[getter] - fn get_show_progress(self_: PyRef) -> bool { - getter!(self_, WordPieceTrainer, show_progress()) - } - - #[setter] - fn set_show_progress(self_: PyRef, show_progress: bool) { - setter!(self_, WordPieceTrainer, @set_show_progress, show_progress); - } - - #[getter] - fn get_special_tokens(self_: PyRef) -> Vec { - getter!( - self_, - WordPieceTrainer, - special_tokens() - .iter() - .map(|tok| tok.clone().into()) - .collect() - ) - } - - #[setter] - fn set_special_tokens(self_: PyRef, special_tokens: &Bound<'_, PyList>) -> PyResult<()> { - setter!( - self_, - WordPieceTrainer, - @set_special_tokens, - special_tokens - .into_iter() - .map(|token| { - if let Ok(content) = token.extract::() { - Ok(tk::tokenizer::AddedToken::from(content, true)) - } else if let Ok(mut token) = token.extract::>() { - token.special = true; - Ok(token.get_token()) - } else { - Err(exceptions::PyTypeError::new_err( - "Special tokens must be a List[Union[str, AddedToken]]", - )) - } - }) - .collect::>>()? - ); - Ok(()) - } - - #[getter] - fn get_limit_alphabet(self_: PyRef) -> Option { - getter!(self_, WordPieceTrainer, limit_alphabet()) - } - - #[setter] - fn set_limit_alphabet(self_: PyRef, limit: Option) { - setter!(self_, WordPieceTrainer, @set_limit_alphabet, limit); - } - - #[getter] - fn get_initial_alphabet(self_: PyRef) -> Vec { - getter!( - self_, - WordPieceTrainer, - initial_alphabet().iter().map(|c| c.to_string()).collect() - ) - } - - #[setter] - fn set_initial_alphabet(self_: PyRef, alphabet: Vec) { - setter!( - self_, - WordPieceTrainer, - @set_initial_alphabet, - alphabet.into_iter().collect() - ); - } - - #[getter] - fn get_continuing_subword_prefix(self_: PyRef) -> Option { - getter!(self_, WordPieceTrainer, continuing_subword_prefix().clone()) - } - - #[setter] - fn set_continuing_subword_prefix(self_: PyRef, prefix: Option) { - setter!(self_, WordPieceTrainer, @set_continuing_subword_prefix, prefix); - } - - #[getter] - fn get_end_of_word_suffix(self_: PyRef) -> Option { - getter!(self_, WordPieceTrainer, end_of_word_suffix().clone()) - } - - #[setter] - fn set_end_of_word_suffix(self_: PyRef, suffix: Option) { - setter!(self_, WordPieceTrainer, @set_end_of_word_suffix, suffix); - } - #[new] - #[pyo3( - signature = (** kwargs), - text_signature = "(self, vocab_size=30000, min_frequency=0, show_progress=True, special_tokens=[], limit_alphabet=None, initial_alphabet=[], continuing_subword_prefix=\"##\", end_of_word_suffix=None)" - )] - pub fn new(kwargs: Option<&Bound<'_, PyDict>>) -> PyResult> { - let mut builder = tk::models::wordpiece::WordPieceTrainer::builder(); - if let Some(kwargs) = kwargs { - for (key, val) in kwargs { - let key: String = key.extract()?; - match key.as_ref() { - "vocab_size" => builder = builder.vocab_size(val.extract()?), - "min_frequency" => builder = builder.min_frequency(val.extract()?), - "show_progress" => builder = builder.show_progress(val.extract()?), - "special_tokens" => { - builder = builder.special_tokens( - val.cast::()? - .into_iter() - .map(|token| { - if let Ok(content) = token.extract::() { - Ok(PyAddedToken::from(content, Some(true)).get_token()) - } else if let Ok(mut token) = - token.extract::>() - { - token.special = true; - Ok(token.get_token()) - } else { - Err(exceptions::PyTypeError::new_err( - "special_tokens must be a List[Union[str, AddedToken]]", - )) - } - }) - .collect::>>()?, - ); - } - "limit_alphabet" => builder = builder.limit_alphabet(val.extract()?), - "initial_alphabet" => { - let alphabet: Vec = val.extract()?; - builder = builder.initial_alphabet( - alphabet - .into_iter() - .filter_map(|s| s.chars().next()) - .collect(), - ); - } - "continuing_subword_prefix" => { - builder = builder.continuing_subword_prefix(val.extract()?) - } - "end_of_word_suffix" => builder = builder.end_of_word_suffix(val.extract()?), - _ => println!("Ignored unknown kwargs option {key}"), - }; - } + #[pyo3(signature = (*, vocab_size = 30000, min_frequency = 0, special_tokens = vec![], limit_alphabet = None, initial_alphabet = vec![], continuing_subword_prefix = String::from("##"), end_of_word_suffix = None, show_progress = true))] + #[allow(clippy::too_many_arguments)] + fn new( + vocab_size: usize, + min_frequency: u64, + special_tokens: Vec, + limit_alphabet: Option, + initial_alphabet: Vec, + continuing_subword_prefix: String, + end_of_word_suffix: Option, + show_progress: bool, + ) -> PyResult> { + let mut builder = WordPieceTrainer::builder() + .vocab_size(vocab_size) + .min_frequency(min_frequency) + .special_tokens(parse_tokens(special_tokens, true)) + .initial_alphabet(initial_alphabet.into_iter().collect()) + .continuing_subword_prefix(continuing_subword_prefix) + .show_progress(show_progress); + if let Some(limit) = limit_alphabet { + builder = builder.limit_alphabet(limit); } - - Ok( - PyClassInitializer::::from(PyTrainer::from(builder.build())) - .add_subclass(PyWordPieceTrainer {}), - ) - } -} - -/// Trainer capable of training a WordLevel model -/// -/// Args: -/// vocab_size (:obj:`int`, `optional`): -/// The size of the final vocabulary, including all tokens and alphabet. -/// -/// min_frequency (:obj:`int`, `optional`): -/// The minimum frequency a pair should have in order to be merged. -/// -/// show_progress (:obj:`bool`, `optional`): -/// Whether to show progress bars while training. -/// -/// special_tokens (:obj:`List[Union[str, AddedToken]]`): -/// A list of special tokens the model should know of. -/// -/// Example:: -/// -/// >>> from tokenizers.models import WordLevel -/// >>> from tokenizers.trainers import WordLevelTrainer -/// >>> trainer = WordLevelTrainer( -/// ... vocab_size=10000, -/// ... special_tokens=[""], -/// ... min_frequency=1, -/// ... ) -/// >>> tokenizer = Tokenizer(WordLevel(unk_token="")) -/// >>> tokenizer.train(["path/to/corpus.txt"], trainer) -/// -#[pyclass(extends=PyTrainer, module = "tokenizers.trainers", name = "WordLevelTrainer")] -pub struct PyWordLevelTrainer {} -#[pymethods] -impl PyWordLevelTrainer { - #[getter] - fn get_vocab_size(self_: PyRef) -> usize { - getter!(self_, WordLevelTrainer, vocab_size) - } - - #[setter] - fn set_vocab_size(self_: PyRef, vocab_size: usize) { - setter!(self_, WordLevelTrainer, vocab_size, vocab_size); - } - - #[getter] - fn get_min_frequency(self_: PyRef) -> u64 { - getter!(self_, WordLevelTrainer, min_frequency) - } - - #[setter] - fn set_min_frequency(self_: PyRef, freq: u64) { - setter!(self_, WordLevelTrainer, min_frequency, freq); - } - - #[getter] - fn get_show_progress(self_: PyRef) -> bool { - getter!(self_, WordLevelTrainer, show_progress) - } - - #[setter] - fn set_show_progress(self_: PyRef, show_progress: bool) { - setter!(self_, WordLevelTrainer, show_progress, show_progress); - } - - #[getter] - fn get_special_tokens(self_: PyRef) -> Vec { - getter!( - self_, - WordLevelTrainer, - special_tokens - .iter() - .map(|tok| tok.clone().into()) - .collect() - ) - } - - #[setter] - fn set_special_tokens(self_: PyRef, special_tokens: &Bound<'_, PyList>) -> PyResult<()> { - setter!( - self_, - WordLevelTrainer, - special_tokens, - special_tokens - .into_iter() - .map(|token| { - if let Ok(content) = token.extract::() { - Ok(tk::tokenizer::AddedToken::from(content, true)) - } else if let Ok(mut token) = token.extract::>() { - token.special = true; - Ok(token.get_token()) - } else { - Err(exceptions::PyTypeError::new_err( - "Special tokens must be a List[Union[str, AddedToken]]", - )) - } - }) - .collect::>>()? - ); - Ok(()) - } - - #[new] - #[pyo3( - signature = (**kwargs), - text_signature = "(self, vocab_size=30000, min_frequency=0, show_progress=True, special_tokens=[])" - )] - pub fn new(kwargs: Option<&Bound<'_, PyDict>>) -> PyResult> { - let mut builder = tk::models::wordlevel::WordLevelTrainer::builder(); - - if let Some(kwargs) = kwargs { - for (key, val) in kwargs { - let key: String = key.extract()?; - match key.as_ref() { - "vocab_size" => { - builder.vocab_size(val.extract()?); - } - "min_frequency" => { - builder.min_frequency(val.extract()?); - } - "show_progress" => { - builder.show_progress(val.extract()?); - } - "special_tokens" => { - builder.special_tokens( - val.cast::()? - .into_iter() - .map(|token| { - if let Ok(content) = token.extract::() { - Ok(PyAddedToken::from(content, Some(true)).get_token()) - } else if let Ok(mut token) = - token.extract::>() - { - token.special = true; - Ok(token.get_token()) - } else { - Err(exceptions::PyTypeError::new_err( - "special_tokens must be a List[Union[str, AddedToken]]", - )) - } - }) - .collect::>>()?, - ); - } - _ => println!("Ignored unknown kwargs option {key}"), - } - } + if let Some(suffix) = end_of_word_suffix { + builder = builder.end_of_word_suffix(suffix); } - - Ok(PyClassInitializer::::from(PyTrainer::from( - builder - .build() - .expect("WordLevelTrainerBuilder cannot fail"), - )) - .add_subclass(PyWordLevelTrainer {})) + Ok(PyClassInitializer::from(PyTrainer { + inner: builder.build().into(), + }) + .add_subclass(PyWordPieceTrainer)) } } -/// Trainer capable of training a Unigram model -/// -/// Args: -/// vocab_size (:obj:`int`): -/// The size of the final vocabulary, including all tokens and alphabet. -/// -/// show_progress (:obj:`bool`): -/// Whether to show progress bars while training. -/// -/// special_tokens (:obj:`List[Union[str, AddedToken]]`): -/// A list of special tokens the model should know of. -/// -/// initial_alphabet (:obj:`List[str]`): -/// A list of characters to include in the initial alphabet, even -/// if not seen in the training dataset. -/// If the strings contain more than one character, only the first one -/// is kept. -/// -/// shrinking_factor (:obj:`float`): -/// The shrinking factor used at each step of the training to prune the -/// vocabulary. -/// -/// unk_token (:obj:`str`): -/// The token used for out-of-vocabulary tokens. -/// -/// max_piece_length (:obj:`int`): -/// The maximum length of a given token. -/// -/// n_sub_iterations (:obj:`int`): -/// The number of iterations of the EM algorithm to perform before -/// pruning the vocabulary. -/// -/// Example:: -/// -/// >>> from tokenizers.models import Unigram -/// >>> from tokenizers.trainers import UnigramTrainer -/// >>> trainer = UnigramTrainer( -/// ... vocab_size=8000, -/// ... special_tokens=["", "", ""], -/// ... unk_token="", -/// ... ) -/// >>> tokenizer = Tokenizer(Unigram()) -/// >>> tokenizer.train(["path/to/corpus.txt"], trainer) -/// -#[pyclass(extends=PyTrainer, module = "tokenizers.trainers", name = "UnigramTrainer")] -pub struct PyUnigramTrainer {} +/// Learns a Unigram vocabulary: starts from a large candidate set and prunes +/// it by `shrinking_factor` each round until `vocab_size` pieces remain. +/// `unk_token` names the fallback piece for unknown characters. +#[pyclass(frozen, extends = PyTrainer, name = "UnigramTrainer", module = "tokenizers.trainers")] +pub struct PyUnigramTrainer; + #[pymethods] impl PyUnigramTrainer { - #[getter] - fn get_vocab_size(self_: PyRef) -> u32 { - getter!(self_, UnigramTrainer, vocab_size) - } - - #[setter] - fn set_vocab_size(self_: PyRef, vocab_size: u32) { - setter!(self_, UnigramTrainer, vocab_size, vocab_size); - } - - #[getter] - fn get_show_progress(self_: PyRef) -> bool { - getter!(self_, UnigramTrainer, show_progress) - } - - #[setter] - fn set_show_progress(self_: PyRef, show_progress: bool) { - setter!(self_, UnigramTrainer, show_progress, show_progress); - } - - #[getter] - fn get_special_tokens(self_: PyRef) -> Vec { - getter!( - self_, - UnigramTrainer, - special_tokens - .iter() - .map(|tok| tok.clone().into()) - .collect() - ) - } - - #[setter] - fn set_special_tokens(self_: PyRef, special_tokens: &Bound<'_, PyList>) -> PyResult<()> { - setter!( - self_, - UnigramTrainer, - special_tokens, - special_tokens - .into_iter() - .map(|token| { - if let Ok(content) = token.extract::() { - Ok(tk::tokenizer::AddedToken::from(content, true)) - } else if let Ok(mut token) = token.extract::>() { - token.special = true; - Ok(token.get_token()) - } else { - Err(exceptions::PyTypeError::new_err( - "Special tokens must be a List[Union[str, AddedToken]]", - )) - } - }) - .collect::>>()? - ); - Ok(()) - } - - #[getter] - fn get_initial_alphabet(self_: PyRef) -> Vec { - getter!( - self_, - UnigramTrainer, - initial_alphabet.iter().map(|c| c.to_string()).collect() - ) - } - - #[setter] - fn set_initial_alphabet(self_: PyRef, alphabet: Vec) { - setter!( - self_, - UnigramTrainer, - initial_alphabet, - alphabet.into_iter().collect() - ); - } - #[new] - #[pyo3( - signature = (**kwargs), - text_signature = "(self, vocab_size=8000, show_progress=True, special_tokens=[], initial_alphabet=[], shrinking_factor=0.75, unk_token=None, max_piece_length=16, n_sub_iterations=2)" - )] - pub fn new(kwargs: Option>) -> PyResult> { - let mut builder = tk::models::unigram::UnigramTrainer::builder(); - if let Some(kwargs) = kwargs { - for (key, val) in kwargs { - let key: String = key.extract()?; - match key.as_ref() { - "vocab_size" => builder.vocab_size(val.extract()?), - "show_progress" => builder.show_progress(val.extract()?), - "n_sub_iterations" => builder.n_sub_iterations(val.extract()?), - "shrinking_factor" => builder.shrinking_factor(val.extract()?), - "unk_token" => builder.unk_token(val.extract()?), - "max_piece_length" => builder.max_piece_length(val.extract()?), - "seed_size" => builder.seed_size(val.extract()?), - "initial_alphabet" => { - let alphabet: Vec = val.extract()?; - builder.initial_alphabet( - alphabet - .into_iter() - .filter_map(|s| s.chars().next()) - .collect(), - ) - } - "special_tokens" => builder.special_tokens( - val.cast::()? - .into_iter() - .map(|token| { - if let Ok(content) = token.extract::() { - Ok(PyAddedToken::from(content, Some(true)).get_token()) - } else if let Ok(mut token) = - token.extract::>() - { - token.special = true; - Ok(token.get_token()) - } else { - Err(exceptions::PyTypeError::new_err( - "special_tokens must be a List[Union[str, AddedToken]]", - )) - } - }) - .collect::>>()?, - ), - _ => { - println!("Ignored unknown kwargs option {key}"); - &mut builder - } - }; - } - } - - let trainer: tokenizers::models::unigram::UnigramTrainer = - builder.build().map_err(|e| { - exceptions::PyException::new_err(format!("Cannot build UnigramTrainer: {e}")) - })?; - Ok( - PyClassInitializer::::from(PyTrainer::from(trainer)) - .add_subclass(PyUnigramTrainer {}), - ) - } -} - -#[cfg(feature = "parity-aware-bpe")] -fn map_tk_err(result: tk::tokenizer::Result) -> PyResult { - result.map_err(|e| exceptions::PyRuntimeError::new_err(format!("{}", e))) -} - -/// Apply the tokenizer's normalizer + pre-tokenizer to a single text sequence -/// and return the resulting word strings. Mirrors the `process` closure that -/// `Tokenizer::train_from_files` builds internally. -/// -/// Generic over the concrete normalizer/pre-tokenizer types so that callers -/// can pass `&PyNormalizer` / `&PyPreTokenizer` directly (which are `Sync`) -/// rather than trait objects (which are not, and would break -/// `feed_language_from_iter`'s `Sync` bound under `maybe_par_bridge`). -#[cfg(feature = "parity-aware-bpe")] -fn pretokenize_sequence( - text: &str, - normalizer: Option<&N>, - pre_tokenizer: Option<&PT>, -) -> tk::tokenizer::Result> -where - N: tk::Normalizer + ?Sized, - PT: tk::PreTokenizer + ?Sized, -{ - use tk::{NormalizedString, OffsetReferential, OffsetType, PreTokenizedString}; - - let normalized_text: String = if let Some(norm) = normalizer { - let mut normalized = NormalizedString::from(text); - norm.normalize(&mut normalized)?; - normalized.get().to_string() - } else { - text.to_string() - }; - - if let Some(pretok) = pre_tokenizer { - let mut pretokenized = PreTokenizedString::from(normalized_text.as_str()); - pretok.pre_tokenize(&mut pretokenized)?; - let splits = pretokenized.get_splits(OffsetReferential::Original, OffsetType::Byte); - Ok(splits - .into_iter() - .filter_map(|(word, _, _)| { - if word.is_empty() { - None - } else { - Some(word.to_string()) - } - }) - .collect()) - } else { - let trimmed = normalized_text.trim(); - if trimmed.is_empty() { - Ok(Vec::new()) - } else { - Ok(vec![trimmed.to_string()]) - } - } -} - -/// Trainer for parity-aware BPE that ensures cross-lingual fairness in tokenization. -/// -/// Unlike standard BPE, this trainer takes one Python iterator per language and -/// balances merge operations across languages using a development set or target -/// compression ratios. The single training entry point is -/// :meth:`train_from_iterator`, the multi-corpus analogue of -/// :meth:`tokenizers.Tokenizer.train_from_iterator`. -/// -/// Args: -/// num_merges (:obj:`int`, `optional`): -/// Number of BPE merge operations to perform. Defaults to ``32000``. -/// -/// variant (:obj:`str`, `optional`): -/// Algorithm variant: ``"base"`` (default) or ``"window"`` (moving-window balancing). -/// -/// min_frequency (:obj:`int`, `optional`): -/// Minimum pair frequency to merge. Defaults to ``0``. -/// -/// global_merges (:obj:`int`, `optional`): -/// Number of initial standard BPE merges before switching to parity mode. Defaults to ``0``. -/// -/// window_size (:obj:`int`, `optional`): -/// Window size for the ``"window"`` variant. Defaults to ``100``. -/// -/// alpha (:obj:`float`, `optional`): -/// Alpha parameter for the ``"window"`` variant. Defaults to ``2.0``. -/// -/// total_symbols (:obj:`bool`, `optional`): -/// If True, subtract unique character count from ``num_merges``. Defaults to ``False``. -/// -/// Example:: -/// -/// from tokenizers import Tokenizer -/// from tokenizers.models import BPE -/// from tokenizers import pre_tokenizers -/// from tokenizers.trainers import ParityBpeTrainer -/// -/// tokenizer = Tokenizer(BPE()) -/// tokenizer.pre_tokenizer = pre_tokenizers.Whitespace() -/// -/// def lines(path): -/// with open(path) as f: -/// yield from f -/// -/// trainer = ParityBpeTrainer(num_merges=32000, variant="base") -/// trainer.train_from_iterator( -/// tokenizer, -/// train_iterators=[lines("train_en.txt"), lines("train_de.txt")], -/// dev_iterators=[lines("dev_en.txt"), lines("dev_de.txt")], -/// ) -/// output = tokenizer.encode("Hello world") -/// -#[cfg(feature = "parity-aware-bpe")] -#[pyclass(module = "tokenizers.trainers", name = "ParityBpeTrainer")] -pub struct PyParityBpeTrainer { - num_merges: usize, - variant: tk::models::bpe::ParityVariant, - min_frequency: u64, - ratio: Option>, - global_merges: usize, - window_size: usize, - alpha: f64, - total_symbols: bool, - special_tokens: Vec, - show_progress: bool, - limit_alphabet: Option, - initial_alphabet: Vec, - continuing_subword_prefix: Option, - end_of_word_suffix: Option, - max_token_length: Option, -} - -#[cfg(feature = "parity-aware-bpe")] -impl Default for PyParityBpeTrainer { - fn default() -> Self { - Self { - num_merges: 32000, - variant: tk::models::bpe::ParityVariant::Base, - min_frequency: 0, - ratio: None, - global_merges: 0, - window_size: 100, - alpha: 2.0, - total_symbols: false, - special_tokens: Vec::new(), - show_progress: true, - limit_alphabet: None, - initial_alphabet: Vec::new(), - continuing_subword_prefix: None, - end_of_word_suffix: None, - max_token_length: None, - } + #[pyo3(signature = (*, vocab_size = 8000, special_tokens = vec![], initial_alphabet = vec![], unk_token = None, shrinking_factor = 0.75, max_piece_length = 16, n_sub_iterations = 2, show_progress = true))] + #[allow(clippy::too_many_arguments)] + fn new( + vocab_size: u32, + special_tokens: Vec, + initial_alphabet: Vec, + unk_token: Option, + shrinking_factor: f64, + max_piece_length: usize, + n_sub_iterations: u32, + show_progress: bool, + ) -> PyResult> { + let trainer = UnigramTrainer::builder() + .vocab_size(vocab_size) + .special_tokens(parse_tokens(special_tokens, true)) + .initial_alphabet(initial_alphabet.into_iter().collect()) + .unk_token(unk_token) + .shrinking_factor(shrinking_factor) + .max_piece_length(max_piece_length) + .n_sub_iterations(n_sub_iterations) + .show_progress(show_progress) + .build() + .map_err(|e| to_pyerr(e.to_string().into()))?; + Ok(PyClassInitializer::from(PyTrainer { + inner: trainer.into(), + }) + .add_subclass(PyUnigramTrainer)) } } -#[cfg(feature = "parity-aware-bpe")] -impl PyParityBpeTrainer { - /// Parse the Python-facing variant string into the Rust `ParityVariant` - /// enum, erroring on anything other than `"base"` / `"window"`. - fn parse_variant(variant: &str) -> PyResult { - use tk::models::bpe::ParityVariant; - match variant { - "base" => Ok(ParityVariant::Base), - "window" => Ok(ParityVariant::Window), - _ => Err(exceptions::PyValueError::new_err(format!( - "Unknown variant '{}'. Use 'base' or 'window'.", - variant - ))), - } - } - - /// Inverse of [`parse_variant`](Self::parse_variant): the Python-facing - /// string for a `ParityVariant`, used by the getter, `__repr__` and - /// pickling so the public API stays string-based. - fn variant_str(variant: tk::models::bpe::ParityVariant) -> &'static str { - use tk::models::bpe::ParityVariant; - match variant { - ParityVariant::Base => "base", - ParityVariant::Window => "window", - } - } - - /// Build a Rust `ParityBpeTrainerBuilder` from the current Python-side settings. - fn make_builder( - &self, - parity_variant: tk::models::bpe::ParityVariant, - ) -> tk::models::bpe::ParityBpeTrainerBuilder { - use tk::models::bpe::ParityBpeTrainer as RustTrainer; - - let mut builder = RustTrainer::builder() - .min_frequency(self.min_frequency) - .num_merges(self.num_merges) - .show_progress(self.show_progress) - .variant(parity_variant) - .global_merges(self.global_merges) - .window_size(self.window_size) - .alpha(self.alpha) - .total_symbols(self.total_symbols) - .special_tokens(self.special_tokens.clone()); - - if let Some(limit) = self.limit_alphabet { - builder = builder.limit_alphabet(limit); - } - if !self.initial_alphabet.is_empty() { - builder = builder.initial_alphabet(self.initial_alphabet.iter().copied().collect()); - } - if let Some(ref prefix) = self.continuing_subword_prefix { - builder = builder.continuing_subword_prefix(prefix.clone()); - } - if let Some(ref suffix) = self.end_of_word_suffix { - builder = builder.end_of_word_suffix(suffix.clone()); - } - builder = builder.max_token_length(self.max_token_length); - builder - } -} +/// Learns a WordLevel vocabulary: the `vocab_size` most frequent words, +/// keeping only those seen at least `min_frequency` times. +#[pyclass(frozen, extends = PyTrainer, name = "WordLevelTrainer", module = "tokenizers.trainers")] +pub struct PyWordLevelTrainer; -#[cfg(feature = "parity-aware-bpe")] #[pymethods] -impl PyParityBpeTrainer { +impl PyWordLevelTrainer { #[new] - #[pyo3(signature = ( - num_merges = 32000, - variant = "base", - min_frequency = 0, - ratio = None, - global_merges = 0, - window_size = 100, - alpha = 2.0, - total_symbols = false, - special_tokens = None, - show_progress = true, - limit_alphabet = None, - initial_alphabet = None, - continuing_subword_prefix = None, - end_of_word_suffix = None, - max_token_length = None - ))] - #[allow(clippy::too_many_arguments)] + #[pyo3(signature = (*, vocab_size = 30000, min_frequency = 0, special_tokens = vec![], show_progress = true))] fn new( - num_merges: usize, - variant: &str, + vocab_size: usize, min_frequency: u64, - ratio: Option>, - global_merges: usize, - window_size: usize, - alpha: f64, - total_symbols: bool, - special_tokens: Option<&Bound<'_, PyList>>, + special_tokens: Vec, show_progress: bool, - limit_alphabet: Option, - initial_alphabet: Option>, - continuing_subword_prefix: Option, - end_of_word_suffix: Option, - max_token_length: Option, - ) -> PyResult { - let variant = Self::parse_variant(variant)?; - - let parsed_special_tokens = if let Some(tokens) = special_tokens { - tokens - .into_iter() - .map(|token| { - if let Ok(content) = token.extract::() { - Ok(tk::tokenizer::AddedToken::from(content, true)) - } else if let Ok(mut token) = token.extract::>() { - token.special = true; - Ok(token.get_token()) - } else { - Err(exceptions::PyTypeError::new_err( - "special_tokens must be a List[Union[str, AddedToken]]", - )) - } - }) - .collect::>>()? - } else { - Vec::new() - }; - - Ok(PyParityBpeTrainer { - num_merges, - variant, - min_frequency, - ratio, - global_merges, - window_size, - alpha, - total_symbols, - special_tokens: parsed_special_tokens, - show_progress, - limit_alphabet, - initial_alphabet: initial_alphabet.unwrap_or_default(), - continuing_subword_prefix, - end_of_word_suffix, - max_token_length, + ) -> PyResult> { + let trainer = WordLevelTrainer::builder() + .vocab_size(vocab_size) + .min_frequency(min_frequency) + .special_tokens(parse_tokens(special_tokens, true)) + .show_progress(show_progress) + .build() + .map_err(|e| to_pyerr(e.to_string().into()))?; + Ok(PyClassInitializer::from(PyTrainer { + inner: trainer.into(), }) - } - - /// Train a user-configured tokenizer with parity-aware BPE from per-language - /// Python iterators. - /// - /// Each entry of ``train_iterators`` (and optionally ``dev_iterators``) is a - /// Python iterator yielding strings (or batches / lists of strings) for one - /// language. This is the multi-corpus analogue of - /// :meth:`~tokenizers.Tokenizer.train_from_iterator`: file I/O happens in - /// Python, so users can pull data from plain text, parquet (via ``pyarrow``), - /// ``datasets``, etc. - /// - /// Args: - /// tokenizer (:class:`~tokenizers.Tokenizer`): - /// A tokenizer instance to train. Its pre-tokenizer (and optionally - /// normalizer) should already be configured. - /// - /// train_iterators (:obj:`List[Iterator]`): - /// One Python iterator per language, each yielding ``str`` or - /// ``List[str]``. - /// - /// dev_iterators (:obj:`List[Iterator]`, `optional`): - /// One Python iterator per language, used to drive parity-aware - /// language selection. Must have the same length as - /// ``train_iterators``. - /// - /// ratio (:obj:`List[float]`, `optional`): - /// Target compression ratios per language (alternative to - /// ``dev_iterators``). - #[pyo3(signature = (tokenizer, train_iterators, dev_iterators = None, ratio = None))] - fn train_from_iterator( - &self, - py: Python, - tokenizer: &mut PyTokenizer, - train_iterators: Vec>, - dev_iterators: Option>>, - ratio: Option>, - ) -> PyResult<()> { - use crate::utils::PyBufferedIterator; - use tk::models::bpe::BPE; - - let parity_variant = self.variant; - - if train_iterators.is_empty() { - return Err(exceptions::PyValueError::new_err( - "train_iterators must not be empty", - )); - } - let num_langs = train_iterators.len(); - if let Some(ref dev) = dev_iterators - && dev.len() != num_langs - { - return Err(exceptions::PyValueError::new_err(format!( - "dev_iterators length ({}) must match train_iterators length ({})", - dev.len(), - num_langs - ))); - } - - let has_dev = dev_iterators.as_ref().is_some_and(|d| !d.is_empty()); - let effective_ratio = if has_dev { - None - } else { - ratio.or_else(|| self.ratio.clone()) - }; - - let mut builder = self.make_builder(parity_variant); - if let Some(r) = effective_ratio { - builder = builder.ratio(r); - } - let mut trainer = builder.build(); - - // Extract normalizer and pre-tokenizer once; references are reused by - // the `process` closure below for both train and dev feeding. We keep - // the concrete `PyNormalizer` / `PyPreTokenizer` types (not `dyn`) so - // that the closure below stays `Sync` — `feed_language_from_iter` - // parallelizes via `maybe_par_bridge` and requires a `Sync` closure. - let normalizer = tokenizer.read_inner()?.get_normalizer().cloned(); - let pre_tokenizer = tokenizer.read_inner()?.get_pre_tokenizer().cloned(); - let norm_ref = normalizer.as_ref(); - let pretok_ref = pre_tokenizer.as_ref(); - let process = |text: &str| -> tk::tokenizer::Result> { - pretokenize_sequence(text, norm_ref, pretok_ref) - }; - - // Materialize each Python iterator into a Vec while still - // holding the GIL (PyBufferedIterator needs the GIL to pull elements - // from Python). After this loop the buffered_iter machinery is gone - // and only owned Rust strings remain — we can release the GIL for the - // expensive feed/do_train work below. - let buffer = |bound: &Bound<'_, PyAny>| -> PyResult> { - let buffered = PyBufferedIterator::new( - bound, - |element| { - if let Ok(s) = element.cast::() { - itertools::Either::Right(std::iter::once( - s.to_cow().map(|s| s.into_owned()), - )) - } else { - match element.try_iter() { - Ok(iter) => itertools::Either::Left( - iter.map(|i| i?.extract::()) - .collect::>() - .into_iter(), - ), - Err(e) => itertools::Either::Right(std::iter::once(Err(e))), - } - } - }, - 256, - )?; - buffered.collect::>>() - }; - - let train_data: Vec> = train_iterators - .iter() - .map(|it| buffer(it.bind(py))) - .collect::>()?; - let dev_data: Option>> = dev_iterators - .as_ref() - .map(|dev| { - dev.iter() - .map(|it| buffer(it.bind(py))) - .collect::>>>() - }) - .transpose()?; - - // Release the GIL for the actual training work — feeding the per- - // language word counts, the merge loop, and the post-train tokenizer - // mutation. None of this touches Python. - py.detach(|| -> PyResult<()> { - for (lang_idx, strings) in train_data.into_iter().enumerate() { - map_tk_err(trainer.feed_language_from_iter( - lang_idx, - strings.into_iter(), - process, - ))?; - } - if let Some(dev_data) = dev_data { - for (lang_idx, strings) in dev_data.into_iter().enumerate() { - map_tk_err(trainer.feed_dev_language_from_iter( - lang_idx, - strings.into_iter(), - process, - ))?; - } - } - - let mut model = BPE::default(); - let (special_tokens, _) = trainer.do_train(&mut model).map_err(|e| { - exceptions::PyRuntimeError::new_err(format!("Training error: {}", e)) - })?; - - let py_model: PyModel = model.into(); - let mut tok_guard = tokenizer.write_inner()?; - tok_guard.with_model(py_model); - tok_guard.add_special_tokens(special_tokens).map_err(|e| { - exceptions::PyRuntimeError::new_err(format!("Failed to add special tokens: {}", e)) - })?; - Ok(()) - })?; - - Ok(()) - } - - #[getter] - fn get_num_merges(&self) -> usize { - self.num_merges - } - - #[setter] - fn set_num_merges(&mut self, v: usize) { - self.num_merges = v; - } - - #[getter] - fn get_variant(&self) -> &'static str { - Self::variant_str(self.variant) - } - - #[getter] - fn get_min_frequency(&self) -> u64 { - self.min_frequency - } - - #[setter] - fn set_min_frequency(&mut self, v: u64) { - self.min_frequency = v; - } - - #[getter] - fn get_global_merges(&self) -> usize { - self.global_merges - } - - #[setter] - fn set_global_merges(&mut self, v: usize) { - self.global_merges = v; - } - - #[getter] - fn get_window_size(&self) -> usize { - self.window_size - } - - #[setter] - fn set_window_size(&mut self, v: usize) { - self.window_size = v; - } - - #[getter] - fn get_alpha(&self) -> f64 { - self.alpha - } - - #[setter] - fn set_alpha(&mut self, v: f64) { - self.alpha = v; - } - - #[getter] - fn get_total_symbols(&self) -> bool { - self.total_symbols - } - - #[setter] - fn set_total_symbols(&mut self, v: bool) { - self.total_symbols = v; - } - - #[getter] - fn get_show_progress(&self) -> bool { - self.show_progress - } - - #[setter] - fn set_show_progress(&mut self, v: bool) { - self.show_progress = v; - } - - #[getter] - fn get_special_tokens(&self) -> Vec { - self.special_tokens - .iter() - .map(|tok| tok.clone().into()) - .collect() - } - - #[setter] - fn set_special_tokens(&mut self, special_tokens: &Bound<'_, PyList>) -> PyResult<()> { - self.special_tokens = special_tokens - .into_iter() - .map(|token| { - if let Ok(content) = token.extract::() { - Ok(tk::tokenizer::AddedToken::from(content, true)) - } else if let Ok(mut token) = token.extract::>() { - token.special = true; - Ok(token.get_token()) - } else { - Err(exceptions::PyTypeError::new_err( - "special_tokens must be a List[Union[str, AddedToken]]", - )) - } - }) - .collect::>>()?; - Ok(()) - } - - #[getter] - fn get_limit_alphabet(&self) -> Option { - self.limit_alphabet - } - - #[setter] - fn set_limit_alphabet(&mut self, v: Option) { - self.limit_alphabet = v; - } - - #[getter] - fn get_initial_alphabet(&self) -> Vec { - self.initial_alphabet - .iter() - .map(|c| c.to_string()) - .collect() - } - - #[setter] - fn set_initial_alphabet(&mut self, alphabet: Vec) { - self.initial_alphabet = alphabet; - } - - #[getter] - fn get_continuing_subword_prefix(&self) -> Option<&str> { - self.continuing_subword_prefix.as_deref() - } - - #[setter] - fn set_continuing_subword_prefix(&mut self, v: Option) { - self.continuing_subword_prefix = v; - } - - #[getter] - fn get_end_of_word_suffix(&self) -> Option<&str> { - self.end_of_word_suffix.as_deref() - } - - #[setter] - fn set_end_of_word_suffix(&mut self, v: Option) { - self.end_of_word_suffix = v; - } - - #[getter] - fn get_max_token_length(&self) -> Option { - self.max_token_length - } - - #[setter] - fn set_max_token_length(&mut self, v: Option) { - self.max_token_length = v; - } - - fn __repr__(&self) -> String { - format!( - "ParityBpeTrainer(num_merges={}, variant=\"{}\", min_frequency={}, \ - global_merges={}, window_size={}, alpha={}, total_symbols={})", - self.num_merges, - Self::variant_str(self.variant), - self.min_frequency, - self.global_merges, - self.window_size, - self.alpha, - self.total_symbols, - ) - } - - fn __str__(&self) -> String { - self.__repr__() - } - - fn __getstate__(&self, py: Python) -> PyResult> { - let dict = PyDict::new(py); - dict.set_item("num_merges", self.num_merges)?; - dict.set_item("variant", Self::variant_str(self.variant))?; - dict.set_item("min_frequency", self.min_frequency)?; - dict.set_item("global_merges", self.global_merges)?; - dict.set_item("window_size", self.window_size)?; - dict.set_item("alpha", self.alpha)?; - dict.set_item("total_symbols", self.total_symbols)?; - dict.set_item("show_progress", self.show_progress)?; - dict.set_item( - "ratio", - self.ratio.as_ref().map(|r| PyList::new(py, r).unwrap()), - )?; - let special: Vec = self - .special_tokens - .iter() - .map(|t| t.content.clone()) - .collect(); - dict.set_item("special_tokens", PyList::new(py, &special)?)?; - dict.set_item("limit_alphabet", self.limit_alphabet)?; - let alphabet_strs: Vec = self - .initial_alphabet - .iter() - .map(|c| c.to_string()) - .collect(); - dict.set_item("initial_alphabet", PyList::new(py, &alphabet_strs)?)?; - dict.set_item("continuing_subword_prefix", &self.continuing_subword_prefix)?; - dict.set_item("end_of_word_suffix", &self.end_of_word_suffix)?; - dict.set_item("max_token_length", self.max_token_length)?; - Ok(dict.into_any().unbind()) - } - - fn __setstate__(&mut self, py: Python, state: Py) -> PyResult<()> { - let dict = state.cast_bound::(py)?; - self.num_merges = dict - .get_item("num_merges")? - .ok_or_else(|| exceptions::PyKeyError::new_err("num_merges"))? - .extract()?; - let variant_str: String = dict - .get_item("variant")? - .ok_or_else(|| exceptions::PyKeyError::new_err("variant"))? - .extract()?; - self.variant = Self::parse_variant(&variant_str)?; - self.min_frequency = dict - .get_item("min_frequency")? - .ok_or_else(|| exceptions::PyKeyError::new_err("min_frequency"))? - .extract()?; - self.global_merges = dict - .get_item("global_merges")? - .ok_or_else(|| exceptions::PyKeyError::new_err("global_merges"))? - .extract()?; - self.window_size = dict - .get_item("window_size")? - .ok_or_else(|| exceptions::PyKeyError::new_err("window_size"))? - .extract()?; - self.alpha = dict - .get_item("alpha")? - .ok_or_else(|| exceptions::PyKeyError::new_err("alpha"))? - .extract()?; - self.total_symbols = dict - .get_item("total_symbols")? - .ok_or_else(|| exceptions::PyKeyError::new_err("total_symbols"))? - .extract()?; - self.show_progress = dict - .get_item("show_progress")? - .ok_or_else(|| exceptions::PyKeyError::new_err("show_progress"))? - .extract()?; - self.ratio = dict.get_item("ratio")?.and_then(|v| { - if v.is_none() { - None - } else { - Some(v.extract().ok()?) - } - }); - let special_strs: Vec = dict - .get_item("special_tokens")? - .ok_or_else(|| exceptions::PyKeyError::new_err("special_tokens"))? - .extract()?; - self.special_tokens = special_strs - .into_iter() - .map(|s| tk::tokenizer::AddedToken::from(s, true)) - .collect(); - self.limit_alphabet = dict.get_item("limit_alphabet")?.and_then(|v| { - if v.is_none() { - None - } else { - Some(v.extract().ok()?) - } - }); - self.initial_alphabet = dict - .get_item("initial_alphabet")? - .and_then(|v| v.extract::>().ok()) - .unwrap_or_default() - .into_iter() - .filter_map(|s| s.chars().next()) - .collect(); - self.continuing_subword_prefix = - dict.get_item("continuing_subword_prefix")?.and_then(|v| { - if v.is_none() { - None - } else { - Some(v.extract().ok()?) - } - }); - self.end_of_word_suffix = dict.get_item("end_of_word_suffix")?.and_then(|v| { - if v.is_none() { - None - } else { - Some(v.extract().ok()?) - } - }); - self.max_token_length = dict.get_item("max_token_length")?.and_then(|v| { - if v.is_none() { - None - } else { - Some(v.extract().ok()?) - } - }); - Ok(()) + .add_subclass(PyWordLevelTrainer)) } } -/// Trainers Module +/// Recipes for learning a vocabulary from text. #[pymodule(gil_used = false)] pub mod trainers { #[pymodule_export] - pub use super::PyBpeTrainer; - #[cfg(feature = "parity-aware-bpe")] - #[pymodule_export] - pub use super::PyParityBpeTrainer; - #[pymodule_export] - pub use super::PyTrainer; - #[pymodule_export] - pub use super::PyUnigramTrainer; - #[pymodule_export] - pub use super::PyWordLevelTrainer; - #[pymodule_export] - pub use super::PyWordPieceTrainer; -} - -#[cfg(test)] -mod tests { - use super::*; - use tk::models::bpe::trainer::BpeTrainer; - - #[test] - fn get_subtype() { - Python::attach(|py| { - let py_trainer = PyTrainer::new(Arc::new(RwLock::new(BpeTrainer::default().into()))); - let py_bpe = py_trainer.get_as_subtype(py).unwrap(); - assert_eq!("BpeTrainer", py_bpe.bind(py).get_type().qualname().unwrap()); - }) - } + pub use super::{ + PyBpeTrainer, PyTrainer, PyUnigramTrainer, PyWordLevelTrainer, PyWordPieceTrainer, + }; } diff --git a/bindings/python/src/utils/iterators.rs b/bindings/python/src/utils/iterators.rs deleted file mode 100644 index d84d25ef1..000000000 --- a/bindings/python/src/utils/iterators.rs +++ /dev/null @@ -1,134 +0,0 @@ -use pyo3::prelude::*; -use std::collections::VecDeque; - -/// An simple iterator that can be instantiated with a specified length. -/// We use this with iterators that don't have a size_hint but we might -/// know its size. This is useful with progress bars for example. -pub struct MaybeSizedIterator { - length: Option, - iter: I, -} - -impl MaybeSizedIterator -where - I: Iterator, -{ - pub fn new(iter: I, length: Option) -> Self { - Self { length, iter } - } -} - -impl Iterator for MaybeSizedIterator -where - I: Iterator, -{ - type Item = I::Item; - - fn next(&mut self) -> Option { - self.iter.next() - } - - fn size_hint(&self) -> (usize, Option) { - (self.length.unwrap_or(0), None) - } -} - -/// A buffered iterator that takes care of locking the GIL only when needed. -/// The `PyIterator` provided by PyO3 keeps a Python GIL token all along -/// and thus doesn't allow us to release the GIL to allow having other threads. -/// -/// This iterator serves two purposes: -/// - First, as opposed to the `pyo3::PyIterator`, it is Send and can easily be parallelized -/// - Second, this let us release the GIL between two refills of the buffer, allowing other -/// Python threads to work -pub struct PyBufferedIterator { - iter: Option>, - converter: F, - buffer: VecDeque>, - size: usize, -} - -impl PyBufferedIterator -where - F: Fn(Bound<'_, PyAny>) -> I, - I: IntoIterator>, -{ - /// Create a new PyBufferedIterator using the provided Python object. - /// This object must implement the Python Iterator Protocol, and an error will - /// be return if the contract is not respected. - /// - /// The `converter` provides a way to convert each item in the iterator into - /// something that doesn't embed a 'py token and thus allows the GIL to be released - /// - /// The `buffer_size` represents the number of items that we buffer before we - /// need to acquire the GIL again. - pub fn new(iter: &Bound<'_, PyAny>, converter: F, buffer_size: usize) -> PyResult { - let py = iter.py(); - let iter: Py = unsafe { - Bound::from_borrowed_ptr_or_err(py, pyo3::ffi::PyObject_GetIter(iter.as_ptr()))?.into() - }; - - Ok(Self { - iter: Some(iter), - converter, - buffer: VecDeque::with_capacity(buffer_size), - size: buffer_size, - }) - } - - /// Refill the buffer, and set `self.iter` as `None` if nothing more to get - fn refill(&mut self) -> PyResult<()> { - if self.iter.is_none() { - return Ok(()); - } - - Python::attach(|py| { - loop { - if self.buffer.len() >= self.size { - return Ok(()); - } - - match unsafe { - Bound::from_owned_ptr_or_opt( - py, - pyo3::ffi::PyIter_Next(self.iter.as_ref().unwrap().bind(py).as_ptr()), - ) - } { - Some(obj) => self.buffer.extend((self.converter)(obj)), - None => { - if PyErr::occurred(py) { - return Err(PyErr::fetch(py)); - } else { - self.iter = None; - } - } - }; - - if self.iter.is_none() { - return Ok(()); - } - } - }) - } -} - -impl Iterator for PyBufferedIterator -where - F: Fn(Bound<'_, PyAny>) -> I, - I: IntoIterator>, -{ - type Item = PyResult; - - fn next(&mut self) -> Option { - if !self.buffer.is_empty() { - self.buffer.pop_front() - } else if self.iter.is_some() { - if let Err(e) = self.refill() { - return Some(Err(e)); - } - self.next() - } else { - None - } - } -} diff --git a/bindings/python/src/utils/mod.rs b/bindings/python/src/utils/mod.rs deleted file mode 100644 index 21b3fc1e1..000000000 --- a/bindings/python/src/utils/mod.rs +++ /dev/null @@ -1,75 +0,0 @@ -use std::marker::PhantomData; -use std::sync::{Arc, Mutex}; - -mod iterators; -mod normalization; -mod pretokenization; -mod regex; -pub mod serde_pyo3; - -pub use iterators::*; -pub use normalization::*; -pub use pretokenization::*; -pub use regex::*; - -// RefMut utils - -pub trait DestroyPtr { - fn destroy(&mut self); -} - -pub struct RefMutGuard<'r, T: DestroyPtr> { - content: T, - r: PhantomData<&'r mut T>, -} -impl RefMutGuard<'_, T> { - pub fn new(content: T) -> Self { - Self { - content, - r: PhantomData, - } - } - - pub fn get(&self) -> &T { - &self.content - } -} - -impl Drop for RefMutGuard<'_, T> { - fn drop(&mut self) { - self.content.destroy() - } -} - -#[derive(Clone)] -pub struct RefMutContainer { - inner: Arc>>, -} -impl RefMutContainer { - pub fn new(content: &mut T) -> Self { - Self { - inner: Arc::new(Mutex::new(Some(content))), - } - } - - pub fn map U, U>(&self, f: F) -> Option { - let lock = self.inner.lock().unwrap(); - let ptr = lock.as_ref()?; - Some(f(unsafe { ptr.as_ref().unwrap() })) - } - - pub fn map_mut U, U>(&mut self, f: F) -> Option { - let lock = self.inner.lock().unwrap(); - let ptr = lock.as_ref()?; - Some(f(unsafe { ptr.as_mut().unwrap() })) - } -} - -impl DestroyPtr for RefMutContainer { - fn destroy(&mut self) { - self.inner.lock().unwrap().take(); - } -} - -unsafe impl Send for RefMutContainer {} -unsafe impl Sync for RefMutContainer {} diff --git a/bindings/python/src/utils/normalization.rs b/bindings/python/src/utils/normalization.rs deleted file mode 100644 index 8745fd629..000000000 --- a/bindings/python/src/utils/normalization.rs +++ /dev/null @@ -1,604 +0,0 @@ -use super::regex::PyRegex; -use super::{DestroyPtr, RefMutContainer, RefMutGuard}; -use crate::error::ToPyResult; -use pyo3::exceptions; -use pyo3::prelude::*; -use pyo3::types::*; -use tk::normalizer::{NormalizedString, Range, SplitDelimiterBehavior, char_to_bytes}; -use tk::pattern::Pattern; - -/// Represents a Pattern as used by `NormalizedString` -#[derive(FromPyObject)] -pub enum PyPattern { - #[pyo3(annotation = "str")] - Str(String), - #[pyo3(annotation = "tokenizers.Regex")] - Regex(Py), - // TODO: Add the compatibility for Fn(char) -> bool -} - -impl Pattern for PyPattern { - fn find_matches(&self, inside: &str) -> tk::Result> { - match self { - PyPattern::Str(s) => { - let mut chars = s.chars(); - if let (Some(c), None) = (chars.next(), chars.next()) { - c.find_matches(inside) - } else { - s.find_matches(inside) - } - } - PyPattern::Regex(r) => Python::attach(|py| (&r.borrow(py).inner).find_matches(inside)), - } - } -} - -impl From for tk::normalizers::replace::ReplacePattern { - fn from(pattern: PyPattern) -> Self { - match pattern { - PyPattern::Str(s) => Self::String(s.to_owned()), - PyPattern::Regex(r) => Python::attach(|py| Self::Regex(r.borrow(py).pattern.clone())), - } - } -} - -impl From for tk::pre_tokenizers::split::SplitPattern { - fn from(pattern: PyPattern) -> Self { - match pattern { - PyPattern::Str(s) => Self::String(s.to_owned()), - PyPattern::Regex(r) => Python::attach(|py| Self::Regex(r.borrow(py).pattern.clone())), - } - } -} - -#[derive(Debug, Clone, FromPyObject)] -pub enum PyRange<'s> { - #[pyo3(annotation = "int")] - Single(isize), - #[pyo3(annotation = "Tuple[uint, uint]")] - Range(usize, usize), - #[pyo3(annotation = "slice")] - Slice(Bound<'s, PySlice>), -} -impl PyRange<'_> { - pub fn to_range(&self, max_len: usize) -> PyResult> { - match self { - PyRange::Single(i) => { - if i.is_negative() { - let i = -i as usize; - if i > max_len { - Err(exceptions::PyValueError::new_err(format!( - "{i} is bigger than max len" - ))) - } else { - Ok(max_len - i..max_len - i + 1) - } - } else { - let i = *i as usize; - Ok(i..i + 1) - } - } - PyRange::Range(s, e) => Ok(*s..*e), - PyRange::Slice(s) => { - let r = s.indices(max_len.try_into()?)?; - Ok(r.start as usize..r.stop as usize) - } - } - } -} - -#[derive(Clone)] -pub struct PySplitDelimiterBehavior(pub SplitDelimiterBehavior); - -impl<'a, 'py> FromPyObject<'a, 'py> for PySplitDelimiterBehavior { - type Error = PyErr; - - fn extract(obj: Borrowed<'a, 'py, PyAny>) -> Result { - let s = obj.extract::()?; - - Ok(Self(match s.as_ref() { - "removed" => Ok(SplitDelimiterBehavior::Removed), - "isolated" => Ok(SplitDelimiterBehavior::Isolated), - "merged_with_previous" => Ok(SplitDelimiterBehavior::MergedWithPrevious), - "merged_with_next" => Ok(SplitDelimiterBehavior::MergedWithNext), - "contiguous" => Ok(SplitDelimiterBehavior::Contiguous), - _ => Err(exceptions::PyValueError::new_err( - "Wrong value for SplitDelimiterBehavior, expected one of: \ - `removed, isolated, merged_with_previous, merged_with_next, contiguous`", - )), - }?)) - } -} - -impl From for SplitDelimiterBehavior { - fn from(v: PySplitDelimiterBehavior) -> Self { - v.0 - } -} - -impl From for PySplitDelimiterBehavior { - fn from(v: SplitDelimiterBehavior) -> Self { - Self(v) - } -} - -fn filter(normalized: &mut NormalizedString, func: &Bound<'_, PyAny>) -> PyResult<()> { - let err = "`filter` expect a callable with the signature: `fn(char) -> bool`"; - - if !func.is_callable() { - Err(exceptions::PyTypeError::new_err(err)) - } else { - normalized.filter(|c| { - func.call1((c.to_string(),)) - .expect(err) - .extract() - .expect(err) - }); - - Ok(()) - } -} - -fn for_each(normalized: &NormalizedString, func: &Bound<'_, PyAny>) -> PyResult<()> { - let err = "`for_each` expect a callable with the signature: `fn(char)`"; - - if !func.is_callable() { - Err(exceptions::PyTypeError::new_err(err)) - } else { - normalized.for_each(|c| { - func.call1((c.to_string(),)).expect(err); - }); - - Ok(()) - } -} - -fn map(normalized: &mut NormalizedString, func: &Bound<'_, PyAny>) -> PyResult<()> { - let err = "`map` expect a callable with the signature: `fn(char) -> char`"; - - if !func.is_callable() { - Err(exceptions::PyTypeError::new_err(err)) - } else { - normalized.map(|c| { - let c: String = func - .call1((c.to_string(),)) - .expect(err) - .extract() - .expect(err); - c.chars().next().expect(err) - }); - - Ok(()) - } -} - -fn slice( - normalized: &NormalizedString, - range: &PyRange<'_>, -) -> PyResult> { - let n_char = normalized.len(); - let char_range = range.to_range(n_char)?; - Ok( - char_to_bytes(normalized.get(), char_range).and_then(|bytes_range| { - normalized - .slice(Range::Normalized(bytes_range)) - .map(|n| n.into()) - }), - ) -} - -/// NormalizedString -/// -/// A NormalizedString takes care of modifying an "original" string, to obtain a "normalized" one. -/// While making all the requested modifications, it keeps track of the alignment information -/// between the two versions of the string. -/// -/// Args: -/// sequence: str: -/// The string sequence used to initialize this NormalizedString -#[pyclass(module = "tokenizers", name = "NormalizedString", from_py_object)] -#[derive(Clone)] -pub struct PyNormalizedString { - pub(crate) normalized: NormalizedString, -} - -#[pymethods] -impl PyNormalizedString { - #[new] - #[pyo3(signature = (sequence), text_signature = "(self, sequence)")] - fn new(sequence: &str) -> Self { - NormalizedString::from(sequence).into() - } - - /// The normalized part of the string - #[getter] - fn get_normalized(&self) -> &str { - self.normalized.get() - } - - #[getter] - fn get_original(&self) -> &str { - self.normalized.get_original() - } - - /// Runs the NFD normalization - #[pyo3(text_signature = "(self)")] - fn nfd(&mut self) { - self.normalized.nfd(); - } - - /// Runs the NFKD normalization - #[pyo3(text_signature = "(self)")] - fn nfkd(&mut self) { - self.normalized.nfkd(); - } - - /// Runs the NFC normalization - #[pyo3(text_signature = "(self)")] - fn nfc(&mut self) { - self.normalized.nfc(); - } - - /// Runs the NFKC normalization - #[pyo3(text_signature = "(self)")] - fn nfkc(&mut self) { - self.normalized.nfkc(); - } - - /// Lowercase the string - #[pyo3(text_signature = "(self)")] - fn lowercase(&mut self) { - self.normalized.lowercase(); - } - - /// Uppercase the string - #[pyo3(text_signature = "(self)")] - fn uppercase(&mut self) { - self.normalized.uppercase(); - } - - /// Prepend the given sequence to the string - #[pyo3(text_signature = "(self, s)")] - fn prepend(&mut self, s: &str) { - self.normalized.prepend(s); - } - - /// Append the given sequence to the string - #[pyo3(text_signature = "(self, s)")] - fn append(&mut self, s: &str) { - self.normalized.append(s); - } - - /// Strip the left of the string - #[pyo3(text_signature = "(self)")] - fn lstrip(&mut self) { - self.normalized.lstrip(); - } - - /// Strip the right of the string - #[pyo3(text_signature = "(self)")] - fn rstrip(&mut self) { - self.normalized.rstrip(); - } - - /// Strip both ends of the string - #[pyo3(text_signature = "(self)")] - fn strip(&mut self) { - self.normalized.strip(); - } - - /// Clears the string - #[pyo3(text_signature = "(self)")] - fn clear(&mut self) { - self.normalized.clear(); - } - - /// Slice the string using the given range - #[pyo3(text_signature = "(self, range)")] - fn slice(&self, range: PyRange) -> PyResult> { - slice(&self.normalized, &range) - } - - /// Filter each character of the string using the given func - #[pyo3(text_signature = "(self, func)")] - fn filter(&mut self, func: &Bound<'_, PyAny>) -> PyResult<()> { - filter(&mut self.normalized, func) - } - - /// Calls the given function for each character of the string - #[pyo3(text_signature = "(self, func)")] - fn for_each(&self, func: &Bound<'_, PyAny>) -> PyResult<()> { - for_each(&self.normalized, func) - } - - /// Calls the given function for each character of the string - /// - /// Replaces each character of the string using the returned value. Each - /// returned value **must** be a str of length 1 (ie a character). - #[pyo3(text_signature = "(self, func)")] - fn map(&mut self, func: &Bound<'_, PyAny>) -> PyResult<()> { - map(&mut self.normalized, func) - } - - /// Split the NormalizedString using the given pattern and the specified behavior - /// - /// Args: - /// pattern: Pattern: - /// A pattern used to split the string. Usually a string or a regex built with `tokenizers.Regex` - /// - /// behavior: SplitDelimiterBehavior: - /// The behavior to use when splitting. - /// Choices: "removed", "isolated", "merged_with_previous", "merged_with_next", - /// "contiguous" - /// - /// Returns: - /// A list of NormalizedString, representing each split - #[pyo3(text_signature = "(self, pattern, behavior)")] - fn split( - &mut self, - pattern: PyPattern, - behavior: PySplitDelimiterBehavior, - ) -> PyResult> { - Ok(ToPyResult(self.normalized.split(pattern, behavior.into())) - .into_py()? - .into_iter() - .map(|n| n.into()) - .collect()) - } - - /// Replace the content of the given pattern with the provided content - /// - /// Args: - /// pattern: Pattern: - /// A pattern used to match the string. Usually a string or a Regex - /// - /// content: str: - /// The content to be used as replacement - #[pyo3(text_signature = "(self, pattern, content)")] - fn replace(&mut self, pattern: PyPattern, content: &str) -> PyResult<()> { - ToPyResult(self.normalized.replace(pattern, content)).into() - } - - fn __repr__(&self) -> String { - format!( - r#"NormalizedString(original="{}", normalized="{}")"#, - self.normalized.get_original(), - self.normalized.get() - ) - } - - fn __str__(&self) -> &str { - self.normalized.get() - } - - fn __getitem__(&self, range: PyRange<'_>) -> PyResult> { - slice(&self.normalized, &range) - } -} - -impl From for PyNormalizedString { - fn from(normalized: NormalizedString) -> Self { - Self { normalized } - } -} - -impl From for NormalizedString { - fn from(normalized: PyNormalizedString) -> Self { - normalized.normalized - } -} - -#[pyclass(module = "tokenizers", name = "NormalizedStringRefMut", from_py_object)] -#[derive(Clone)] -pub struct PyNormalizedStringRefMut { - inner: RefMutContainer, -} - -impl DestroyPtr for PyNormalizedStringRefMut { - fn destroy(&mut self) { - self.inner.destroy(); - } -} - -impl PyNormalizedStringRefMut { - pub fn new(normalized: &mut NormalizedString) -> RefMutGuard<'_, Self> { - RefMutGuard::new(Self { - inner: RefMutContainer::new(normalized), - }) - } - - pub fn destroyed_error() -> PyErr { - exceptions::PyException::new_err("Cannot use a NormalizedStringRefMut outside `normalize`") - } - - /// Provides a way to access a reference to the underlying NormalizedString - pub fn map_as_ref U, U>(&self, f: F) -> PyResult { - self.inner - .map(f) - .ok_or_else(PyNormalizedStringRefMut::destroyed_error) - } - - /// Provides a way to access a mutable reference to the underlying NormalizedString - pub fn map_as_mut U, U>(&mut self, f: F) -> PyResult { - self.inner - .map_mut(f) - .ok_or_else(PyNormalizedStringRefMut::destroyed_error) - } -} - -#[pymethods] -impl PyNormalizedStringRefMut { - #[getter] - fn get_normalized(&self) -> PyResult { - self.inner - .map(|n| n.get().to_owned()) - .ok_or_else(PyNormalizedStringRefMut::destroyed_error) - } - - #[getter] - fn get_original(&self) -> PyResult { - self.inner - .map(|n| n.get_original().to_owned()) - .ok_or_else(PyNormalizedStringRefMut::destroyed_error) - } - - fn nfd(&mut self) -> PyResult<()> { - self.inner - .map_mut(|n| { - n.nfd(); - }) - .ok_or_else(PyNormalizedStringRefMut::destroyed_error)?; - Ok(()) - } - - fn nfkd(&mut self) -> PyResult<()> { - self.inner - .map_mut(|n| { - n.nfkd(); - }) - .ok_or_else(PyNormalizedStringRefMut::destroyed_error)?; - Ok(()) - } - - fn nfc(&mut self) -> PyResult<()> { - self.inner - .map_mut(|n| { - n.nfc(); - }) - .ok_or_else(PyNormalizedStringRefMut::destroyed_error)?; - Ok(()) - } - - fn nfkc(&mut self) -> PyResult<()> { - self.inner - .map_mut(|n| { - n.nfkc(); - }) - .ok_or_else(PyNormalizedStringRefMut::destroyed_error)?; - Ok(()) - } - - fn lowercase(&mut self) -> PyResult<()> { - self.inner - .map_mut(|n| { - n.lowercase(); - }) - .ok_or_else(PyNormalizedStringRefMut::destroyed_error)?; - Ok(()) - } - - fn uppercase(&mut self) -> PyResult<()> { - self.inner - .map_mut(|n| { - n.uppercase(); - }) - .ok_or_else(PyNormalizedStringRefMut::destroyed_error)?; - Ok(()) - } - - fn prepend(&mut self, s: &str) -> PyResult<()> { - self.inner - .map_mut(|n| { - n.prepend(s); - }) - .ok_or_else(PyNormalizedStringRefMut::destroyed_error)?; - Ok(()) - } - - fn append(&mut self, s: &str) -> PyResult<()> { - self.inner - .map_mut(|n| { - n.append(s); - }) - .ok_or_else(PyNormalizedStringRefMut::destroyed_error)?; - Ok(()) - } - - fn lstrip(&mut self) -> PyResult<()> { - self.inner - .map_mut(|n| { - n.lstrip(); - }) - .ok_or_else(PyNormalizedStringRefMut::destroyed_error)?; - Ok(()) - } - - fn rstrip(&mut self) -> PyResult<()> { - self.inner - .map_mut(|n| { - n.rstrip(); - }) - .ok_or_else(PyNormalizedStringRefMut::destroyed_error)?; - Ok(()) - } - - fn strip(&mut self) -> PyResult<()> { - self.inner - .map_mut(|n| { - n.strip(); - }) - .ok_or_else(PyNormalizedStringRefMut::destroyed_error)?; - Ok(()) - } - - fn clear(&mut self) -> PyResult<()> { - self.inner - .map_mut(|n| { - n.clear(); - }) - .ok_or_else(PyNormalizedStringRefMut::destroyed_error)?; - Ok(()) - } - - fn slice(&self, range: PyRange) -> PyResult> { - self.inner - .map(|n| slice(n, &range)) - .ok_or_else(PyNormalizedStringRefMut::destroyed_error)? - } - - fn filter(&mut self, func: &Bound<'_, PyAny>) -> PyResult<()> { - self.inner - .map_mut(|n| filter(n, func)) - .ok_or_else(PyNormalizedStringRefMut::destroyed_error)??; - Ok(()) - } - - fn for_each(&self, func: &Bound<'_, PyAny>) -> PyResult<()> { - self.inner - .map(|n| for_each(n, func)) - .ok_or_else(PyNormalizedStringRefMut::destroyed_error)??; - Ok(()) - } - - fn map(&mut self, func: &Bound<'_, PyAny>) -> PyResult<()> { - self.inner - .map_mut(|n| map(n, func)) - .ok_or_else(PyNormalizedStringRefMut::destroyed_error)??; - Ok(()) - } - - fn split( - &mut self, - pattern: PyPattern, - behavior: PySplitDelimiterBehavior, - ) -> PyResult> { - Ok(ToPyResult( - self.inner - .map_mut(|n| n.split(pattern, behavior.into())) - .ok_or_else(PyNormalizedStringRefMut::destroyed_error)?, - ) - .into_py()? - .into_iter() - .map(|n| n.into()) - .collect()) - } - - fn replace(&mut self, pattern: PyPattern, content: &str) -> PyResult<()> { - ToPyResult( - self.inner - .map_mut(|n| n.replace(pattern, content)) - .ok_or_else(PyNormalizedStringRefMut::destroyed_error)?, - ) - .into() - } -} diff --git a/bindings/python/src/utils/pretokenization.rs b/bindings/python/src/utils/pretokenization.rs deleted file mode 100644 index 22b8029ff..000000000 --- a/bindings/python/src/utils/pretokenization.rs +++ /dev/null @@ -1,335 +0,0 @@ -use tokenizers as tk; - -use pyo3::exceptions; -use pyo3::prelude::*; -use pyo3::types::*; - -use super::{ - DestroyPtr, PyNormalizedString, PyNormalizedStringRefMut, RefMutContainer, RefMutGuard, -}; -use crate::encoding::PyEncoding; -use crate::error::ToPyResult; -use crate::token::PyToken; -use tk::{OffsetReferential, OffsetType, Offsets, PreTokenizedString, Token}; - -fn split(pretok: &mut PreTokenizedString, func: &Bound<'_, PyAny>) -> PyResult<()> { - if !func.is_callable() { - Err(exceptions::PyTypeError::new_err( - "`split` expect a callable with the signature: \ - `fn(index: int, normalized: NormalizedString) -> List[NormalizedString]`", - )) - } else { - ToPyResult(pretok.split(|i, normalized| { - let output = func.call((i, PyNormalizedString::from(normalized)), None)?; - Ok(output - .extract::>()? - .into_iter() - .map(tk::NormalizedString::from)) - })) - .into() - } -} - -fn normalize(pretok: &mut PreTokenizedString, func: &Bound<'_, PyAny>) -> PyResult<()> { - if !func.is_callable() { - Err(exceptions::PyTypeError::new_err( - "`normalize` expect a callable with the signature: \ - `fn(normalized: NormalizedString)`", - )) - } else { - ToPyResult(pretok.normalize(|normalized| { - let norm = PyNormalizedStringRefMut::new(normalized); - func.call((norm.get().clone(),), None)?; - Ok(()) - })) - .into() - } -} - -fn tokenize(pretok: &mut PreTokenizedString, func: &Bound<'_, PyAny>) -> PyResult<()> { - if !func.is_callable() { - Err(exceptions::PyTypeError::new_err( - "`tokenize` expect a callable with the signature: \ - `fn(str) -> List[Token]`", - )) - } else { - ToPyResult(pretok.tokenize(|normalized| { - let output = func.call((normalized.get(),), None)?; - Ok(output - .extract::>() - .map_err(PyErr::from)? - .into_iter() - .map(|obj| Ok(Token::from(obj.extract::()?))) - .collect::>>()?) - })) - .into() - } -} - -/// This is an enum -#[derive(Clone)] -pub struct PyOffsetReferential(OffsetReferential); -impl<'a, 'py> FromPyObject<'a, 'py> for PyOffsetReferential { - type Error = PyErr; - - fn extract(obj: Borrowed<'a, 'py, PyAny>) -> Result { - let s = obj.extract::()?; - - Ok(Self(match s.as_ref() { - "original" => Ok(OffsetReferential::Original), - "normalized" => Ok(OffsetReferential::Normalized), - _ => Err(exceptions::PyValueError::new_err( - "Wrong value for OffsetReferential, expected one of `original, normalized`", - )), - }?)) - } -} - -#[derive(Clone)] -pub struct PyOffsetType(OffsetType); -impl<'a, 'py> FromPyObject<'a, 'py> for PyOffsetType { - type Error = PyErr; - - fn extract(obj: Borrowed<'a, 'py, PyAny>) -> Result { - let s = obj.extract::()?; - - Ok(Self(match s.as_ref() { - "byte" => Ok(OffsetType::Byte), - "char" => Ok(OffsetType::Char), - _ => Err(exceptions::PyValueError::new_err( - "Wrong value for OffsetType, expected one of `byte, char`", - )), - }?)) - } -} - -type PySplit = (String, Offsets, Option>); -fn get_splits( - pretok: &PreTokenizedString, - offset_referential: PyOffsetReferential, - offset_type: PyOffsetType, -) -> Vec { - pretok - .get_splits(offset_referential.0, offset_type.0) - .into_iter() - .map(|(s, o, t)| { - ( - s.to_owned(), - o, - t.as_ref() - .map(|tokens| tokens.iter().map(|t| t.clone().into()).collect()), - ) - }) - .collect() -} - -fn to_encoding( - pretok: &PreTokenizedString, - type_id: u32, - word_idx: Option, -) -> PyResult { - Ok(ToPyResult( - pretok - .clone() - .into_encoding(word_idx, type_id, tk::OffsetType::Char), - ) - .into_py()? - .into()) -} - -/// PreTokenizedString -/// -/// Wrapper over a string, that provides a way to normalize, pre-tokenize, tokenize the -/// underlying string, while keeping track of the alignment information (offsets). -/// -/// The PreTokenizedString manages what we call `splits`. Each split represents a substring -/// which is a subpart of the original string, with the relevant offsets and tokens. -/// -/// When calling one of the methods used to modify the PreTokenizedString (namely one of -/// `split`, `normalize` or `tokenize), only the `splits` that don't have any associated -/// tokens will get modified. -/// -/// Args: -/// sequence: str: -/// The string sequence used to initialize this PreTokenizedString -#[pyclass(module = "tokenizers", name = "PreTokenizedString")] -pub struct PyPreTokenizedString { - pub(crate) pretok: tk::PreTokenizedString, -} - -impl From for PyPreTokenizedString { - fn from(pretok: PreTokenizedString) -> Self { - Self { pretok } - } -} - -impl From for PreTokenizedString { - fn from(pretok: PyPreTokenizedString) -> Self { - pretok.pretok - } -} - -#[pymethods] -impl PyPreTokenizedString { - #[new] - #[pyo3(text_signature = "(self, sequence)")] - fn new(s: &str) -> Self { - PreTokenizedString::from(s).into() - } - - /// Split the PreTokenizedString using the given `func` - /// - /// Args: - /// func: Callable[[index, NormalizedString], List[NormalizedString]]: - /// The function used to split each underlying split. - /// It is expected to return a list of `NormalizedString`, that represent the new - /// splits. If the given `NormalizedString` does not need any splitting, we can - /// just return it directly. - /// In order for the offsets to be tracked accurately, any returned `NormalizedString` - /// should come from calling either `.split` or `.slice` on the received one. - #[pyo3(text_signature = "(self, func)")] - fn split(&mut self, func: &Bound<'_, PyAny>) -> PyResult<()> { - split(&mut self.pretok, func) - } - - /// Normalize each split of the `PreTokenizedString` using the given `func` - /// - /// Args: - /// func: Callable[[NormalizedString], None]: - /// The function used to normalize each underlying split. This function - /// does not need to return anything, just calling the methods on the provided - /// NormalizedString allow its modification. - #[pyo3(text_signature = "(self, func)")] - fn normalize(&mut self, func: &Bound<'_, PyAny>) -> PyResult<()> { - normalize(&mut self.pretok, func) - } - - /// Tokenize each split of the `PreTokenizedString` using the given `func` - /// - /// Args: - /// func: Callable[[str], List[Token]]: - /// The function used to tokenize each underlying split. This function must return - /// a list of Token generated from the input str. - #[pyo3(text_signature = "(self, func)")] - fn tokenize(&mut self, func: &Bound<'_, PyAny>) -> PyResult<()> { - tokenize(&mut self.pretok, func) - } - - /// Return an Encoding generated from this PreTokenizedString - /// - /// Args: - /// type_id: int = 0: - /// The type_id to be used on the generated Encoding. - /// - /// word_idx: Optional[int] = None: - /// An optional word index to be used for each token of this Encoding. If provided, - /// all the word indices in the generated Encoding will use this value, instead - /// of the one automatically tracked during pre-tokenization. - /// - /// Returns: - /// An Encoding - #[pyo3(signature = (type_id = 0, word_idx = None) -> "Encoding")] - #[pyo3(text_signature = "(self, type_id=0, word_idx=None)")] - fn to_encoding(&self, type_id: u32, word_idx: Option) -> PyResult { - to_encoding(&self.pretok, type_id, word_idx) - } - - /// Get the splits currently managed by the PreTokenizedString - /// - /// Args: - /// offset_referential: :obj:`str` - /// Whether the returned splits should have offsets expressed relative - /// to the original string, or the normalized one. choices: "original", "normalized". - /// - /// offset_type: :obj:`str` - /// Whether the returned splits should have offsets expressed in bytes or chars. - /// When slicing an str, we usually want to use chars, which is the default value. - /// Now in some cases it might be interesting to get these offsets expressed in bytes, - /// so it is possible to change this here. - /// choices: "char", "bytes" - /// - /// Returns - /// A list of splits - #[pyo3(signature = ( - offset_referential = PyOffsetReferential(OffsetReferential::Original), - offset_type = PyOffsetType(OffsetType::Char) - ))] - #[pyo3(text_signature = "(self, offset_referential=\"original\", offset_type=\"char\")")] - fn get_splits( - &self, - offset_referential: PyOffsetReferential, - offset_type: PyOffsetType, - ) -> Vec { - get_splits(&self.pretok, offset_referential, offset_type) - } -} - -#[pyclass(module = "tokenizers", name = "PreTokenizedString", from_py_object)] -#[derive(Clone)] -pub struct PyPreTokenizedStringRefMut { - inner: RefMutContainer, -} - -impl DestroyPtr for PyPreTokenizedStringRefMut { - fn destroy(&mut self) { - self.inner.destroy(); - } -} - -impl PyPreTokenizedStringRefMut { - pub fn new(pretok: &mut tk::PreTokenizedString) -> RefMutGuard<'_, Self> { - // SAFETY: This is safe because we return a RefMutGuard here. - // The compiler will make sure the &mut stays valid as necessary. - RefMutGuard::new(Self { - inner: RefMutContainer::new(pretok), - }) - } - - pub fn destroyed_error() -> PyErr { - exceptions::PyException::new_err( - "Cannot use a PreTokenizedStringRefMut outside `pre_tokenize`", - ) - } -} - -#[pymethods] -impl PyPreTokenizedStringRefMut { - fn split(&mut self, func: &Bound<'_, PyAny>) -> PyResult<()> { - self.inner - .map_mut(|pretok| split(pretok, func)) - .ok_or_else(PyPreTokenizedStringRefMut::destroyed_error)? - } - - fn normalize(&mut self, func: &Bound<'_, PyAny>) -> PyResult<()> { - self.inner - .map_mut(|pretok| normalize(pretok, func)) - .ok_or_else(PyPreTokenizedStringRefMut::destroyed_error)? - } - - fn tokenize(&mut self, func: &Bound<'_, PyAny>) -> PyResult<()> { - self.inner - .map_mut(|pretok| tokenize(pretok, func)) - .ok_or_else(PyPreTokenizedStringRefMut::destroyed_error)? - } - - #[pyo3(signature = (type_id = 0, word_idx = None))] - fn to_encoding(&self, type_id: u32, word_idx: Option) -> PyResult { - self.inner - .map(|pretok| to_encoding(pretok, type_id, word_idx)) - .ok_or_else(PyPreTokenizedStringRefMut::destroyed_error)? - } - - #[pyo3(signature = ( - offset_referential = PyOffsetReferential(OffsetReferential::Original), - offset_type = PyOffsetType(OffsetType::Char) - ))] - fn get_splits( - &self, - offset_referential: PyOffsetReferential, - offset_type: PyOffsetType, - ) -> PyResult> { - self.inner - .map(|pretok| get_splits(pretok, offset_referential, offset_type)) - .ok_or_else(PyPreTokenizedStringRefMut::destroyed_error) - } -} diff --git a/bindings/python/src/utils/regex.rs b/bindings/python/src/utils/regex.rs deleted file mode 100644 index fafba844f..000000000 --- a/bindings/python/src/utils/regex.rs +++ /dev/null @@ -1,23 +0,0 @@ -use pyo3::exceptions; -use pyo3::prelude::*; -use tk::utils::SysRegex; - -/// Instantiate a new Regex with the given pattern -#[pyclass(module = "tokenizers", name = "Regex", frozen)] -pub struct PyRegex { - pub inner: SysRegex, - pub pattern: String, -} - -#[pymethods] -impl PyRegex { - #[new] - #[pyo3(text_signature = "(self, pattern)")] - fn new(s: &str) -> PyResult { - Ok(Self { - inner: SysRegex::new(s) - .map_err(|e| exceptions::PyException::new_err(e.to_string().to_owned()))?, - pattern: s.to_owned(), - }) - } -} diff --git a/bindings/python/src/utils/serde_pyo3.rs b/bindings/python/src/utils/serde_pyo3.rs deleted file mode 100644 index abb0b3496..000000000 --- a/bindings/python/src/utils/serde_pyo3.rs +++ /dev/null @@ -1,773 +0,0 @@ -use serde::de::value::Error; -use serde::{Serialize, ser}; -type Result = ::std::result::Result; - -pub struct Serializer { - // This string starts empty and JSON is appended as values are serialized. - output: String, - /// Each levels remembers its own number of elements - num_elements: Vec, - max_elements: usize, - level: usize, - max_depth: usize, - /// Maximum string representation - /// Useful to ellipsis precompiled_charmap - max_string: usize, -} - -// By convention, the public API of a Serde serializer is one or more `to_abc` -// functions such as `to_string`, `to_bytes`, or `to_writer` depending on what -// Rust types the serializer is able to produce as output. -// -// This basic serializer supports only `to_string`. -pub fn to_string(value: &T) -> Result -where - T: Serialize, -{ - let max_depth = 20; - let max_elements = 6; - let max_string = 100; - let mut serializer = Serializer { - output: String::new(), - level: 0, - max_depth, - max_elements, - num_elements: vec![0; max_depth], - max_string, - }; - value.serialize(&mut serializer)?; - Ok(serializer.output) -} - -pub fn repr(value: &T) -> Result -where - T: Serialize, -{ - let max_depth = 200; - let max_string = usize::MAX; - let mut serializer = Serializer { - output: String::new(), - level: 0, - max_depth, - max_elements: 100, - num_elements: vec![0; max_depth], - max_string, - }; - value.serialize(&mut serializer)?; - Ok(serializer.output) -} - -impl ser::Serializer for &mut Serializer { - // The output type produced by this `Serializer` during successful - // serialization. Most serializers that produce text or binary output should - // set `Ok = ()` and serialize into an `io::Write` or buffer contained - // within the `Serializer` instance, as happens here. Serializers that build - // in-memory data structures may be simplified by using `Ok` to propagate - // the data structure around. - type Ok = (); - - // The error type when some error occurs during serialization. - type Error = Error; - - // Associated types for keeping track of additional state while serializing - // compound data structures like sequences and maps. In this case no - // additional state is required beyond what is already stored in the - // Serializer struct. - type SerializeSeq = Self; - type SerializeTuple = Self; - type SerializeTupleStruct = Self; - type SerializeTupleVariant = Self; - type SerializeMap = Self; - type SerializeStruct = Self; - type SerializeStructVariant = Self; - - // Here we go with the simple methods. The following 12 methods receive one - // of the primitive types of the data model and map it to JSON by appending - // into the output string. - fn serialize_bool(self, v: bool) -> Result<()> { - self.output += if v { "True" } else { "False" }; - Ok(()) - } - - // JSON does not distinguish between different sizes of integers, so all - // signed integers will be serialized the same and all unsigned integers - // will be serialized the same. Other formats, especially compact binary - // formats, may need independent logic for the different sizes. - fn serialize_i8(self, v: i8) -> Result<()> { - self.serialize_i64(i64::from(v)) - } - - fn serialize_i16(self, v: i16) -> Result<()> { - self.serialize_i64(i64::from(v)) - } - - fn serialize_i32(self, v: i32) -> Result<()> { - self.serialize_i64(i64::from(v)) - } - - // Not particularly efficient but this is example code anyway. A more - // performant approach would be to use the `itoa` crate. - fn serialize_i64(self, v: i64) -> Result<()> { - self.output += &v.to_string(); - Ok(()) - } - - fn serialize_u8(self, v: u8) -> Result<()> { - self.serialize_u64(u64::from(v)) - } - - fn serialize_u16(self, v: u16) -> Result<()> { - self.serialize_u64(u64::from(v)) - } - - fn serialize_u32(self, v: u32) -> Result<()> { - self.serialize_u64(u64::from(v)) - } - - fn serialize_u64(self, v: u64) -> Result<()> { - self.output += &v.to_string(); - Ok(()) - } - - fn serialize_f32(self, v: f32) -> Result<()> { - self.serialize_f64(f64::from(v)) - } - - fn serialize_f64(self, v: f64) -> Result<()> { - self.output += &v.to_string(); - Ok(()) - } - - // Serialize a char as a single-character string. Other formats may - // represent this differently. - fn serialize_char(self, v: char) -> Result<()> { - self.serialize_str(&v.to_string()) - } - - // This only works for strings that don't require escape sequences but you - // get the idea. For example it would emit invalid JSON if the input string - // contains a '"' character. - fn serialize_str(self, v: &str) -> Result<()> { - self.output += "\""; - if v.len() > self.max_string { - self.output += &v[..self.max_string]; - self.output += "..."; - } else { - self.output += v; - } - self.output += "\""; - Ok(()) - } - - // Serialize a byte array as an array of bytes. Could also use a base64 - // string here. Binary formats will typically represent byte arrays more - // compactly. - fn serialize_bytes(self, v: &[u8]) -> Result<()> { - use serde::ser::SerializeSeq; - let mut seq = self.serialize_seq(Some(v.len()))?; - for byte in v { - seq.serialize_element(byte)?; - } - seq.end() - } - - // An absent optional is represented as the JSON `null`. - fn serialize_none(self) -> Result<()> { - self.serialize_unit() - } - - // A present optional is represented as just the contained value. Note that - // this is a lossy representation. For example the values `Some(())` and - // `None` both serialize as just `null`. Unfortunately this is typically - // what people expect when working with JSON. Other formats are encouraged - // to behave more intelligently if possible. - fn serialize_some(self, value: &T) -> Result<()> - where - T: ?Sized + Serialize, - { - value.serialize(self) - } - - // In Serde, unit means an anonymous value containing no data. Map this to - // JSON as `null`. - fn serialize_unit(self) -> Result<()> { - self.output += "None"; - Ok(()) - } - - // Unit struct means a named value containing no data. Again, since there is - // no data, map this to JSON as `null`. There is no need to serialize the - // name in most formats. - fn serialize_unit_struct(self, _name: &'static str) -> Result<()> { - self.serialize_unit() - } - - // When serializing a unit variant (or any other kind of variant), formats - // can choose whether to keep track of it by index or by name. Binary - // formats typically use the index of the variant and human-readable formats - // typically use the name. - fn serialize_unit_variant( - self, - _name: &'static str, - _variant_index: u32, - variant: &'static str, - ) -> Result<()> { - // self.serialize_str(variant) - self.output += variant; - Ok(()) - } - - // As is done here, serializers are encouraged to treat newtype structs as - // insignificant wrappers around the data they contain. - fn serialize_newtype_struct(self, _name: &'static str, value: &T) -> Result<()> - where - T: ?Sized + Serialize, - { - value.serialize(self) - } - - // Note that newtype variant (and all of the other variant serialization - // methods) refer exclusively to the "externally tagged" enum - // representation. - // - // Serialize this to JSON in externally tagged form as `{ NAME: VALUE }`. - fn serialize_newtype_variant( - self, - _name: &'static str, - _variant_index: u32, - variant: &'static str, - value: &T, - ) -> Result<()> - where - T: ?Sized + Serialize, - { - // variant.serialize(&mut *self)?; - self.output += variant; - self.output += "("; - value.serialize(&mut *self)?; - self.output += ")"; - Ok(()) - } - - // Now we get to the serialization of compound types. - // - // The start of the sequence, each value, and the end are three separate - // method calls. This one is responsible only for serializing the start, - // which in JSON is `[`. - // - // The length of the sequence may or may not be known ahead of time. This - // doesn't make a difference in JSON because the length is not represented - // explicitly in the serialized form. Some serializers may only be able to - // support sequences for which the length is known up front. - fn serialize_seq(self, _len: Option) -> Result { - self.output += "["; - self.level = std::cmp::min(self.max_depth - 1, self.level + 1); - self.num_elements[self.level] = 0; - Ok(self) - } - - // Tuples look just like sequences in JSON. Some formats may be able to - // represent tuples more efficiently by omitting the length, since tuple - // means that the corresponding `Deserialize implementation will know the - // length without needing to look at the serialized data. - fn serialize_tuple(self, _len: usize) -> Result { - self.output += "("; - self.level = std::cmp::min(self.max_depth - 1, self.level + 1); - self.num_elements[self.level] = 0; - Ok(self) - } - - // Tuple structs look just like sequences in JSON. - fn serialize_tuple_struct( - self, - _name: &'static str, - len: usize, - ) -> Result { - self.serialize_tuple(len) - } - - // Tuple variants are represented in JSON as `{ NAME: [DATA...] }`. Again - // this method is only responsible for the externally tagged representation. - fn serialize_tuple_variant( - self, - _name: &'static str, - _variant_index: u32, - variant: &'static str, - _len: usize, - ) -> Result { - // variant.serialize(&mut *self)?; - self.output += variant; - self.output += "("; - self.level = std::cmp::min(self.max_depth - 1, self.level + 1); - self.num_elements[self.level] = 0; - Ok(self) - } - - // Maps are represented in JSON as `{ K: V, K: V, ... }`. - fn serialize_map(self, _len: Option) -> Result { - self.output += "{"; - self.level = std::cmp::min(self.max_depth - 1, self.level + 1); - self.num_elements[self.level] = 0; - Ok(self) - } - - // Structs look just like maps in JSON. In particular, JSON requires that we - // serialize the field names of the struct. Other formats may be able to - // omit the field names when serializing structs because the corresponding - // Deserialize implementation is required to know what the keys are without - // looking at the serialized data. - fn serialize_struct(self, name: &'static str, _len: usize) -> Result { - // self.serialize_map(Some(len)) - // name.serialize(&mut *self)?; - if let Some(stripped) = name.strip_suffix("Helper") { - self.output += stripped; - } else { - self.output += name - } - self.output += "("; - self.level = std::cmp::min(self.max_depth - 1, self.level + 1); - self.num_elements[self.level] = 0; - Ok(self) - } - - // Struct variants are represented in JSON as `{ NAME: { K: V, ... } }`. - // This is the externally tagged representation. - fn serialize_struct_variant( - self, - _name: &'static str, - _variant_index: u32, - variant: &'static str, - _len: usize, - ) -> Result { - // variant.serialize(&mut *self)?; - self.output += variant; - self.output += "("; - self.level = std::cmp::min(self.max_depth - 1, self.level + 1); - self.num_elements[self.level] = 0; - Ok(self) - } -} - -// The following 7 impls deal with the serialization of compound types like -// sequences and maps. Serialization of such types is begun by a Serializer -// method and followed by zero or more calls to serialize individual elements of -// the compound type and one call to end the compound type. -// -// This impl is SerializeSeq so these methods are called after `serialize_seq` -// is called on the Serializer. -impl ser::SerializeSeq for &mut Serializer { - // Must match the `Ok` type of the serializer. - type Ok = (); - // Must match the `Error` type of the serializer. - type Error = Error; - - // Serialize a single element of the sequence. - fn serialize_element(&mut self, value: &T) -> Result<()> - where - T: ?Sized + Serialize, - { - self.num_elements[self.level] += 1; - let num_elements = self.num_elements[self.level]; - if num_elements < self.max_elements { - if !self.output.ends_with('[') { - self.output += ", "; - } - value.serialize(&mut **self) - } else { - if num_elements == self.max_elements { - self.output += ", ..."; - } - Ok(()) - } - } - - // Close the sequence. - fn end(self) -> Result<()> { - self.num_elements[self.level] = 0; - self.level = self.level.saturating_sub(1); - self.output += "]"; - Ok(()) - } -} - -// Same thing but for tuples. -impl ser::SerializeTuple for &mut Serializer { - type Ok = (); - type Error = Error; - - fn serialize_element(&mut self, value: &T) -> Result<()> - where - T: ?Sized + Serialize, - { - self.num_elements[self.level] += 1; - let num_elements = self.num_elements[self.level]; - if num_elements < self.max_elements { - if !self.output.ends_with('(') { - self.output += ", "; - } - value.serialize(&mut **self) - } else { - if num_elements == self.max_elements { - self.output += ", ..."; - } - Ok(()) - } - } - - fn end(self) -> Result<()> { - self.num_elements[self.level] = 0; - self.level = self.level.saturating_sub(1); - self.output += ")"; - Ok(()) - } -} - -// Same thing but for tuple structs. -impl ser::SerializeTupleStruct for &mut Serializer { - type Ok = (); - type Error = Error; - - fn serialize_field(&mut self, value: &T) -> Result<()> - where - T: ?Sized + Serialize, - { - self.num_elements[self.level] += 1; - let num_elements = self.num_elements[self.level]; - if num_elements < self.max_elements { - if !self.output.ends_with('(') { - self.output += ", "; - } - value.serialize(&mut **self) - } else { - if num_elements == self.max_elements { - self.output += ", ..."; - } - Ok(()) - } - } - - fn end(self) -> Result<()> { - self.num_elements[self.level] = 0; - self.level = self.level.saturating_sub(1); - self.output += ")"; - Ok(()) - } -} - -// Tuple variants are a little different. Refer back to the -// `serialize_tuple_variant` method above: -// -// self.output += "{"; -// variant.serialize(&mut *self)?; -// self.output += ":["; -// -// So the `end` method in this impl is responsible for closing both the `]` and -// the `}`. -impl ser::SerializeTupleVariant for &mut Serializer { - type Ok = (); - type Error = Error; - - fn serialize_field(&mut self, value: &T) -> Result<()> - where - T: ?Sized + Serialize, - { - self.num_elements[self.level] += 1; - let num_elements = self.num_elements[self.level]; - if num_elements < self.max_elements { - if !self.output.ends_with('(') { - self.output += ", "; - } - value.serialize(&mut **self) - } else { - if num_elements == self.max_elements { - self.output += ", ..."; - } - Ok(()) - } - } - - fn end(self) -> Result<()> { - self.num_elements[self.level] = 0; - self.level = self.level.saturating_sub(1); - self.output += ")"; - Ok(()) - } -} - -// Some `Serialize` types are not able to hold a key and value in memory at the -// same time so `SerializeMap` implementations are required to support -// `serialize_key` and `serialize_value` individually. -// -// There is a third optional method on the `SerializeMap` trait. The -// `serialize_entry` method allows serializers to optimize for the case where -// key and value are both available simultaneously. In JSON it doesn't make a -// difference so the default behavior for `serialize_entry` is fine. -impl ser::SerializeMap for &mut Serializer { - type Ok = (); - type Error = Error; - - // The Serde data model allows map keys to be any serializable type. JSON - // only allows string keys so the implementation below will produce invalid - // JSON if the key serializes as something other than a string. - // - // A real JSON serializer would need to validate that map keys are strings. - // This can be done by using a different Serializer to serialize the key - // (instead of `&mut **self`) and having that other serializer only - // implement `serialize_str` and return an error on any other data type. - fn serialize_key(&mut self, key: &T) -> Result<()> - where - T: ?Sized + Serialize, - { - self.num_elements[self.level] += 1; - let num_elements = self.num_elements[self.level]; - if num_elements < self.max_elements { - if !self.output.ends_with('{') { - self.output += ", "; - } - key.serialize(&mut **self) - } else { - if num_elements == self.max_elements { - self.output += ", ..."; - } - Ok(()) - } - } - - // It doesn't make a difference whether the colon is printed at the end of - // `serialize_key` or at the beginning of `serialize_value`. In this case - // the code is a bit simpler having it here. - fn serialize_value(&mut self, value: &T) -> Result<()> - where - T: ?Sized + Serialize, - { - let num_elements = self.num_elements[self.level]; - if num_elements < self.max_elements { - self.output += ":"; - value.serialize(&mut **self) - } else { - Ok(()) - } - } - - fn end(self) -> Result<()> { - self.num_elements[self.level] = 0; - self.level = self.level.saturating_sub(1); - self.output += "}"; - Ok(()) - } -} - -// Structs are like maps in which the keys are constrained to be compile-time -// constant strings. -impl ser::SerializeStruct for &mut Serializer { - type Ok = (); - type Error = Error; - - fn serialize_field(&mut self, key: &'static str, value: &T) -> Result<()> - where - T: ?Sized + Serialize, - { - if !self.output.ends_with('(') { - self.output += ", "; - } - // key.serialize(&mut **self)?; - if key != "type" { - self.output += key; - self.output += "="; - value.serialize(&mut **self) - } else { - Ok(()) - } - } - - fn end(self) -> Result<()> { - self.num_elements[self.level] = 0; - self.level = self.level.saturating_sub(1); - self.output += ")"; - Ok(()) - } -} - -// Similar to `SerializeTupleVariant`, here the `end` method is responsible for -// closing both of the curly braces opened by `serialize_struct_variant`. -impl ser::SerializeStructVariant for &mut Serializer { - type Ok = (); - type Error = Error; - - fn serialize_field(&mut self, key: &'static str, value: &T) -> Result<()> - where - T: ?Sized + Serialize, - { - if !self.output.ends_with('(') { - self.output += ", "; - } - // key.serialize(&mut **self)?; - self.output += key; - self.output += "="; - value.serialize(&mut **self) - } - - fn end(self) -> Result<()> { - self.num_elements[self.level] = 0; - self.level = self.level.saturating_sub(1); - self.output += ")"; - Ok(()) - } -} - -//////////////////////////////////////////////////////////////////////////////// - -#[test] -fn test_basic() { - assert_eq!(to_string(&true).unwrap(), "True"); - assert_eq!(to_string(&Some(1)).unwrap(), "1"); - assert_eq!(to_string(&None::).unwrap(), "None"); -} - -#[test] -fn test_struct() { - #[derive(Serialize)] - struct Test { - int: u32, - seq: Vec<&'static str>, - } - - let test = Test { - int: 1, - seq: vec!["a", "b"], - }; - let expected = r#"Test(int=1, seq=["a", "b"])"#; - assert_eq!(to_string(&test).unwrap(), expected); -} - -#[test] -fn test_enum() { - #[derive(Serialize)] - enum E { - Unit, - Newtype(u32), - Tuple(u32, u32), - Struct { a: u32 }, - } - - let u = E::Unit; - let expected = r#"Unit"#; - assert_eq!(to_string(&u).unwrap(), expected); - - let n = E::Newtype(1); - let expected = r#"Newtype(1)"#; - assert_eq!(to_string(&n).unwrap(), expected); - - let t = E::Tuple(1, 2); - let expected = r#"Tuple(1, 2)"#; - assert_eq!(to_string(&t).unwrap(), expected); - - let s = E::Struct { a: 1 }; - let expected = r#"Struct(a=1)"#; - assert_eq!(to_string(&s).unwrap(), expected); -} - -#[test] -fn test_enum_untagged() { - #[derive(Serialize)] - #[serde(untagged)] - enum E { - Unit, - Newtype(u32), - Tuple(u32, u32), - Struct { a: u32 }, - } - - let u = E::Unit; - let expected = r#"None"#; - assert_eq!(to_string(&u).unwrap(), expected); - - let n = E::Newtype(1); - let expected = r#"1"#; - assert_eq!(to_string(&n).unwrap(), expected); - - let t = E::Tuple(1, 2); - let expected = r#"(1, 2)"#; - assert_eq!(to_string(&t).unwrap(), expected); - - let s = E::Struct { a: 1 }; - let expected = r#"E(a=1)"#; - assert_eq!(to_string(&s).unwrap(), expected); -} - -#[test] -fn test_struct_tagged() { - #[derive(Serialize)] - #[serde(untagged)] - enum E { - A(A), - } - - #[derive(Serialize)] - #[serde(tag = "type")] - struct A { - a: bool, - b: usize, - } - - let u = A { a: true, b: 1 }; - // let expected = r#"A(type="A", a=True, b=1)"#; - // No we skip all `type` manually inserted variants. - let expected = r#"A(a=True, b=1)"#; - assert_eq!(to_string(&u).unwrap(), expected); - - let u = E::A(A { a: true, b: 1 }); - let expected = r#"A(a=True, b=1)"#; - assert_eq!(to_string(&u).unwrap(), expected); -} - -#[test] -fn test_flatten() { - #[derive(Serialize)] - struct A { - a: bool, - b: usize, - } - - #[derive(Serialize)] - struct B { - c: A, - d: usize, - } - - #[derive(Serialize)] - struct C { - #[serde(flatten)] - c: A, - d: usize, - } - - #[derive(Serialize)] - #[serde(transparent)] - struct D { - e: A, - } - - let u = B { - c: A { a: true, b: 1 }, - d: 2, - }; - let expected = r#"B(c=A(a=True, b=1), d=2)"#; - assert_eq!(to_string(&u).unwrap(), expected); - - let u = C { - c: A { a: true, b: 1 }, - d: 2, - }; - // XXX This is unfortunate but true, flatten forces the serialization - // to use the serialize_map without any means for the Serializer to know about this - // flattening attempt - let expected = r#"{"a":True, "b":1, "d":2}"#; - assert_eq!(to_string(&u).unwrap(), expected); - - let u = D { - e: A { a: true, b: 1 }, - }; - let expected = r#"A(a=True, b=1)"#; - assert_eq!(to_string(&u).unwrap(), expected); -} diff --git a/bindings/python/test.txt b/bindings/python/test.txt deleted file mode 100644 index 7c16cdea3..000000000 --- a/bindings/python/test.txt +++ /dev/null @@ -1,36 +0,0 @@ - \test{bla} thisisatest - \test{bla} thisisatest - \test{bla} thisisatest - \test{bla} thisisatest - \test{bla} thisisatest - \test{bla} thisisatest - \test{bla} thisisatest - \test{bla} thisisatest - \test{bla} thisisatest - \test{bla} thisisatest - \test{bla} thisisatest - \test{bla} thisisatest - \test{bla} thisisatest - \test{bla} thisisatest - \test{bla} thisisatest - \test{bla} thisisatest - \test{bla} thisisatest - \test{bla} thisisatest - \test{bla} thisisatest - \test{bla} thisisatest - \test{bla} thisisatest - \test{bla} thisisatest - \test{bla} thisisatest - \test{bla} thisisatest - \test{bla} thisisatest - \test{bla} thisisatest - \test{bla} thisisatest - \test{bla} thisisatest - \test{bla} thisisatest - \test{bla} thisisatest - \test{bla} thisisatest - \test{bla} thisisatest - \test{bla} thisisatest - \test{bla} thisisatest - \test{bla} thisisatest - \test{bla} thisisatest diff --git a/bindings/python/tests/__init__.py b/bindings/python/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/bindings/python/tests/bindings/__init__.py b/bindings/python/tests/bindings/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/bindings/python/tests/bindings/test_decoders.py b/bindings/python/tests/bindings/test_decoders.py deleted file mode 100644 index 3ee7663bf..000000000 --- a/bindings/python/tests/bindings/test_decoders.py +++ /dev/null @@ -1,228 +0,0 @@ -import json -import pickle - -import pytest - -from tokenizers.decoders import ( - CTC, - BPEDecoder, - ByteLevel, - Decoder, - Metaspace, - Sequence, - WordPiece, - ByteFallback, - Replace, - Strip, - Fuse, -) - - -class TestByteLevel: - def test_instantiate(self): - assert ByteLevel() is not None - assert isinstance(ByteLevel(), Decoder) - assert isinstance(ByteLevel(), ByteLevel) - assert isinstance(pickle.loads(pickle.dumps(ByteLevel())), ByteLevel) - - def test_decoding(self): - decoder = ByteLevel() - assert decoder.decode(["My", "Ġname", "Ġis", "ĠJohn"]) == "My name is John" - - def test_manual_reload(self): - byte_level = ByteLevel() - state = json.loads(byte_level.__getstate__()) - reloaded = ByteLevel(**state) - assert isinstance(reloaded, ByteLevel) - - -class TestReplace: - def test_instantiate(self): - assert Replace("_", " ") is not None - assert isinstance(Replace("_", " "), Decoder) - assert isinstance(Replace("_", " "), Replace) - # assert isinstance(pickle.loads(pickle.dumps(Replace("_", " "))), Replace) - - def test_decoding(self): - decoder = Replace("_", " ") - assert decoder.decode(["My", "_name", "_is", "_John"]) == "My name is John" - - -class TestWordPiece: - def test_instantiate(self): - assert WordPiece() is not None - assert WordPiece(prefix="__") is not None - assert WordPiece(cleanup=True) is not None - assert isinstance(WordPiece(), Decoder) - assert isinstance(WordPiece(), WordPiece) - assert isinstance(pickle.loads(pickle.dumps(WordPiece())), WordPiece) - - def test_decoding(self): - decoder = WordPiece() - assert decoder.decode(["My", "na", "##me", "is", "Jo", "##hn"]) == "My name is John" - assert decoder.decode(["I", "'m", "Jo", "##hn"]) == "I'm John" - decoder = WordPiece(prefix="__", cleanup=False) - assert decoder.decode(["My", "na", "__me", "is", "Jo", "__hn"]) == "My name is John" - assert decoder.decode(["I", "'m", "Jo", "__hn"]) == "I 'm John" - - def test_can_modify(self): - decoder = WordPiece(prefix="$$", cleanup=False) - - assert decoder.prefix == "$$" - assert decoder.cleanup == False - - # Modify these - decoder.prefix = "__" - assert decoder.prefix == "__" - decoder.cleanup = True - assert decoder.cleanup == True - - -class TestByteFallback: - def test_instantiate(self): - assert ByteFallback() is not None - assert isinstance(ByteFallback(), Decoder) - assert isinstance(ByteFallback(), ByteFallback) - assert isinstance(pickle.loads(pickle.dumps(ByteFallback())), ByteFallback) - - def test_decoding(self): - decoder = ByteFallback() - assert decoder.decode(["My", " na", "me"]) == "My name" - assert decoder.decode(["<0x61>"]) == "a" - assert decoder.decode(["<0xE5>"]) == "�" - assert decoder.decode(["<0xE5>", "<0x8f>"]) == "��" - assert decoder.decode(["<0xE5>", "<0x8f>", "<0xab>"]) == "叫" - assert decoder.decode(["<0xE5>", "<0x8f>", "a"]) == "��a" - assert decoder.decode(["<0xE5>", "<0x8f>", "<0xab>", "a"]) == "叫a" - - -class TestFuse: - def test_instantiate(self): - assert Fuse() is not None - assert isinstance(Fuse(), Decoder) - assert isinstance(Fuse(), Fuse) - assert isinstance(pickle.loads(pickle.dumps(Fuse())), Fuse) - - def test_decoding(self): - decoder = Fuse() - assert decoder.decode(["My", " na", "me"]) == "My name" - - -class TestStrip: - def test_instantiate(self): - assert Strip(left=0, right=0) is not None - assert isinstance(Strip(content="_", left=0, right=0), Decoder) - assert isinstance(Strip(content="_", left=0, right=0), Strip) - assert isinstance(pickle.loads(pickle.dumps(Strip(content="_", left=0, right=0))), Strip) - - def test_decoding(self): - decoder = Strip(content="_", left=1, right=0) - assert decoder.decode(["_My", " na", "me", " _-", "__-"]) == "My name _-_-" - - -class TestMetaspace: - def test_instantiate(self): - assert Metaspace() is not None - assert Metaspace(replacement="-") is not None - with pytest.raises(ValueError, match="expected a string of length 1"): - Metaspace(replacement="") - assert Metaspace(prepend_scheme="always") is not None - assert isinstance(Metaspace(), Decoder) - assert isinstance(Metaspace(), Metaspace) - assert isinstance(pickle.loads(pickle.dumps(Metaspace())), Metaspace) - - def test_decoding(self): - decoder = Metaspace() - assert decoder.decode(["▁My", "▁name", "▁is", "▁John"]) == "My name is John" - decoder = Metaspace(replacement="-", prepend_scheme="never") - assert decoder.decode(["-My", "-name", "-is", "-John"]) == " My name is John" - - def test_can_modify(self): - decoder = Metaspace(replacement="*", prepend_scheme="never") - - assert decoder.replacement == "*" - assert decoder.prepend_scheme == "never" - - # Modify these - decoder.replacement = "&" - assert decoder.replacement == "&" - decoder.prepend_scheme = "first" - assert decoder.prepend_scheme == "first" - - -class TestBPEDecoder: - def test_instantiate(self): - assert BPEDecoder() is not None - assert BPEDecoder(suffix="_") is not None - assert isinstance(BPEDecoder(), Decoder) - assert isinstance(BPEDecoder(), BPEDecoder) - assert isinstance(pickle.loads(pickle.dumps(BPEDecoder())), BPEDecoder) - - def test_decoding(self): - decoder = BPEDecoder() - assert decoder.decode(["My", "na", "me", "is", "Jo", "hn"]) == "My name is John" - decoder = BPEDecoder(suffix="_") - assert decoder.decode(["My_", "na", "me_", "is_", "Jo", "hn_"]) == "My name is John" - - def test_can_modify(self): - decoder = BPEDecoder(suffix="123") - - assert decoder.suffix == "123" - - # Modify these - decoder.suffix = "" - assert decoder.suffix == "" - - -class TestCTCDecoder: - def test_instantiate(self): - assert CTC() is not None - assert CTC(pad_token="[PAD]") is not None - assert isinstance(CTC(), Decoder) - assert isinstance(CTC(), CTC) - assert isinstance(pickle.loads(pickle.dumps(CTC())), CTC) - - def test_decoding(self): - decoder = CTC() - assert ( - decoder.decode(["", "", "h", "e", "e", "l", "l", "", "l", "o", "o", "o", ""]) - == "hello" - ) - decoder = CTC(pad_token="[PAD]") - assert ( - decoder.decode(["[PAD]", "[PAD]", "h", "e", "e", "l", "l", "[PAD]", "l", "o", "o", "o", "[PAD]"]) - == "hello" - ) - - def test_can_modify(self): - decoder = CTC(pad_token="[PAD]") - - assert decoder.pad_token == "[PAD]" - assert decoder.word_delimiter_token == "|" - assert decoder.cleanup == True - - # Modify these - decoder.pad_token = "{pad}" - assert decoder.pad_token == "{pad}" - - decoder.word_delimiter_token = "_" - assert decoder.word_delimiter_token == "_" - - decoder.cleanup = False - assert decoder.cleanup == False - - -class TestSequenceDecoder: - def test_instantiate(self): - assert Sequence([]) is not None - assert Sequence([CTC()]) is not None - assert isinstance(Sequence([]), Decoder) - assert isinstance(Sequence([]), Sequence) - serialized = pickle.dumps(Sequence([])) - assert isinstance(pickle.loads(serialized), Sequence) - - def test_decoding(self): - decoder = Sequence([CTC(), Metaspace()]) - initial = ["▁", "▁", "H", "H", "i", "i", "▁", "y", "o", "u"] - expected = "Hi you" - assert decoder.decode(initial) == expected diff --git a/bindings/python/tests/bindings/test_encoding.py b/bindings/python/tests/bindings/test_encoding.py deleted file mode 100644 index 90ac36343..000000000 --- a/bindings/python/tests/bindings/test_encoding.py +++ /dev/null @@ -1,122 +0,0 @@ -import pytest - -from tokenizers import BertWordPieceTokenizer - -from ..utils import bert_files, data_dir - - -@pytest.mark.network -class TestEncoding: - @pytest.fixture(scope="class") - def encodings(self, bert_files): - tokenizer = BertWordPieceTokenizer.from_file(bert_files["vocab"]) - single_encoding = tokenizer.encode("I love HuggingFace") - pair_encoding = tokenizer.encode("I love HuggingFace", "Do you?") - return single_encoding, pair_encoding - - def test_sequence_ids(self, encodings): - single, pair = encodings - - assert single.sequence_ids == [None, 0, 0, 0, 0, None] - assert pair.sequence_ids == [None, 0, 0, 0, 0, None, 1, 1, 1, None] - - def test_n_sequences(self, encodings): - single, pair = encodings - assert single.n_sequences == 1 - assert pair.n_sequences == 2 - - def test_word_to_tokens(self, encodings): - single, pair = encodings - - assert single.tokens == ["[CLS]", "i", "love", "hugging", "##face", "[SEP]"] - assert single.word_to_tokens(0) == (1, 2) - - assert pair.tokens == [ - "[CLS]", - "i", - "love", - "hugging", - "##face", - "[SEP]", - "do", - "you", - "?", - "[SEP]", - ] - assert pair.word_to_tokens(0) == (1, 2) - assert pair.word_to_tokens(0, 0) == (1, 2) - assert pair.word_to_tokens(6, 0) == None - assert pair.word_to_tokens(0, 1) == (6, 7) - - def test_word_to_chars(self, encodings): - single, pair = encodings - - assert single.word_to_chars(2) == (7, 18) - assert pair.word_to_chars(2) == (7, 18) - assert pair.word_to_chars(2, 0) == (7, 18) - assert pair.word_to_chars(2, 1) == (6, 7) - - def test_token_to_sequence(self, encodings): - single, pair = encodings - - assert single.token_to_sequence(2) == 0 - assert pair.token_to_sequence(2) == 0 - assert pair.token_to_sequence(0) == None - assert pair.token_to_sequence(5) == None - assert pair.token_to_sequence(6) == 1 - assert pair.token_to_sequence(8) == 1 - assert pair.token_to_sequence(9) == None - assert pair.token_to_sequence(1200) == None - - def test_token_to_chars(self, encodings): - single, pair = encodings - - assert single.token_to_chars(0) == None - assert single.token_to_chars(2) == (2, 6) - assert pair.token_to_chars(2) == (2, 6) - assert pair.token_to_chars(5) == None - assert pair.token_to_chars(6) == (0, 2) - - def test_token_to_word(self, encodings): - single, pair = encodings - - assert single.token_to_word(0) == None - assert single.token_to_word(1) == 0 - assert single.token_to_word(4) == 2 - assert pair.token_to_word(1) == 0 - assert pair.token_to_word(4) == 2 - assert pair.token_to_word(5) == None - assert pair.token_to_word(6) == 0 - assert pair.token_to_word(7) == 1 - - def test_char_to_token(self, encodings): - single, pair = encodings - - assert single.char_to_token(0) == 1 - assert pair.char_to_token(0) == 1 - assert pair.char_to_token(0, 0) == 1 - assert pair.char_to_token(1, 0) == None - assert pair.char_to_token(0, 1) == 6 - assert pair.char_to_token(2, 1) == None - - def test_char_to_word(self, encodings): - single, pair = encodings - - assert single.char_to_word(0) == 0 - assert single.char_to_word(1) == None - assert pair.char_to_word(2) == 1 - assert pair.char_to_word(2, 0) == 1 - assert pair.char_to_word(2, 1) == None - assert pair.char_to_word(3, 1) == 1 - - def test_truncation(self, encodings): - single, _ = encodings - single.truncate(2, 1, "right") - assert single.tokens == ["[CLS]", "i"] - assert single.overflowing[0].tokens == ["i", "love"] - - def test_invalid_truncate_direction(self, encodings): - single, _ = encodings - with pytest.raises(ValueError) as excinfo: - single.truncate(2, 1, "not_a_direction") - assert "Invalid truncation direction value : not_a_direction" == str(excinfo.value) diff --git a/bindings/python/tests/bindings/test_models.py b/bindings/python/tests/bindings/test_models.py deleted file mode 100644 index 4b5545fd1..000000000 --- a/bindings/python/tests/bindings/test_models.py +++ /dev/null @@ -1,121 +0,0 @@ -import pickle - -import pytest - -from tokenizers.models import BPE, Model, Unigram, WordLevel, WordPiece -from ..utils import bert_files, data_dir, roberta_files - - -class TestBPE: - def test_can_modify(self): - model = BPE( - dropout=0.5, - unk_token="[UNK]", - continuing_subword_prefix="__prefix__", - end_of_word_suffix="__suffix__", - fuse_unk=False, - ) - - assert model.dropout == 0.5 - assert model.unk_token == "[UNK]" - assert model.continuing_subword_prefix == "__prefix__" - assert model.end_of_word_suffix == "__suffix__" - assert model.fuse_unk == False - assert model.byte_fallback == False - - # Modify these - model.dropout = 0.1 - assert pytest.approx(model.dropout) == 0.1 - model.unk_token = "" - assert model.unk_token == "" - model.continuing_subword_prefix = None - assert model.continuing_subword_prefix == None - model.end_of_word_suffix = "suff" - assert model.end_of_word_suffix == "suff" - model.fuse_unk = True - assert model.fuse_unk == True - model.byte_fallback = True - assert model.byte_fallback == True - - def test_dropout_zero(self): - model = BPE(dropout=0.0) - assert model.dropout == 0.0 - - -class TestUnigram: - def test_can_modify(self): - model = Unigram(alpha=0.5) - - assert model.alpha == 0.5 - assert model.nbest_size is None - - # Modify these - model.alpha = 0.1 - assert pytest.approx(model.alpha) == 0.1 - model.nbest_size = 64 - assert model.nbest_size == 64 - - def test_alpha_zero(self): - model = Unigram(alpha=0.0) - assert model.alpha == 0.0 - - -class TestWordPiece: - @pytest.mark.network - def test_instantiate(self, bert_files): - assert isinstance(WordPiece(), Model) - assert isinstance(WordPiece(), WordPiece) - - vocab = {"a": 0, "b": 1, "ab": 2} - assert isinstance(WordPiece(vocab), Model) - assert isinstance(WordPiece(vocab), WordPiece) - assert isinstance(WordPiece.from_file(bert_files["vocab"]), WordPiece) - assert isinstance(pickle.loads(pickle.dumps(WordPiece(vocab))), WordPiece) - - assert isinstance(WordPiece(bert_files["vocab"]), Model) - assert isinstance(pickle.loads(pickle.dumps(WordPiece(bert_files["vocab"]))), WordPiece) - - def test_can_modify(self): - model = WordPiece( - unk_token="", - continuing_subword_prefix="__prefix__", - max_input_chars_per_word=200, - ) - - assert model.unk_token == "" - assert model.continuing_subword_prefix == "__prefix__" - assert model.max_input_chars_per_word == 200 - - # Modify these - model.unk_token = "" - assert model.unk_token == "" - model.continuing_subword_prefix = "$$$" - assert model.continuing_subword_prefix == "$$$" - model.max_input_chars_per_word = 10 - assert model.max_input_chars_per_word == 10 - - -class TestWordLevel: - @pytest.mark.network - def test_instantiate(self, roberta_files): - assert isinstance(WordLevel(), Model) - assert isinstance(WordLevel(), WordLevel) - - vocab = {"a": 0, "b": 1, "ab": 2} - assert isinstance(WordLevel(vocab), Model) - assert isinstance(WordLevel(vocab), WordLevel) - assert isinstance(WordLevel.from_file(roberta_files["vocab"]), WordLevel) - - # The WordLevel model expects a vocab.json using the same format as roberta - # so we can just try to load with this file - assert isinstance(WordLevel(roberta_files["vocab"]), Model) - assert isinstance(WordLevel(roberta_files["vocab"]), WordLevel) - - def test_can_modify(self): - model = WordLevel(unk_token="") - - assert model.unk_token == "" - - # Modify these - model.unk_token = "" - assert model.unk_token == "" diff --git a/bindings/python/tests/bindings/test_normalizers.py b/bindings/python/tests/bindings/test_normalizers.py deleted file mode 100644 index 99ab07d39..000000000 --- a/bindings/python/tests/bindings/test_normalizers.py +++ /dev/null @@ -1,237 +0,0 @@ -import pickle - -import pytest - -from tokenizers import NormalizedString -from tokenizers.normalizers import ( - BertNormalizer, - Lowercase, - Normalizer, - Precompiled, - Sequence, - Strip, - Prepend, - Replace, -) - - -class TestBertNormalizer: - def test_instantiate(self): - assert isinstance(BertNormalizer(), Normalizer) - assert isinstance(BertNormalizer(), BertNormalizer) - assert isinstance(pickle.loads(pickle.dumps(BertNormalizer())), BertNormalizer) - - def test_strip_accents(self): - normalizer = BertNormalizer(strip_accents=True, lowercase=False, handle_chinese_chars=False, clean_text=False) - - output = normalizer.normalize_str("Héllò") - assert output == "Hello" - - def test_handle_chinese_chars(self): - normalizer = BertNormalizer(strip_accents=False, lowercase=False, handle_chinese_chars=True, clean_text=False) - - output = normalizer.normalize_str("你好") - assert output == " 你 好 " - - def test_clean_text(self): - normalizer = BertNormalizer(strip_accents=False, lowercase=False, handle_chinese_chars=False, clean_text=True) - - output = normalizer.normalize_str("\ufeffHello") - assert output == "Hello" - - def test_lowercase(self): - normalizer = BertNormalizer(strip_accents=False, lowercase=True, handle_chinese_chars=False, clean_text=False) - - output = normalizer.normalize_str("Héllò") - assert output == "héllò" - - def test_can_modify(self): - normalizer = BertNormalizer(clean_text=True, handle_chinese_chars=True, strip_accents=True, lowercase=True) - - assert normalizer.clean_text == True - assert normalizer.handle_chinese_chars == True - assert normalizer.strip_accents == True - assert normalizer.lowercase == True - - # Modify these - normalizer.clean_text = False - assert normalizer.clean_text == False - normalizer.handle_chinese_chars = False - assert normalizer.handle_chinese_chars == False - normalizer.strip_accents = None - assert normalizer.strip_accents == None - normalizer.lowercase = False - assert normalizer.lowercase == False - - -class TestSequence: - def test_instantiate(self): - assert isinstance(Sequence([]), Normalizer) - assert isinstance(Sequence([]), Sequence) - assert isinstance(pickle.loads(pickle.dumps(Sequence([]))), Sequence) - - def test_can_make_sequences(self): - normalizer = Sequence([Lowercase(), Strip()]) - - output = normalizer.normalize_str(" HELLO ") - assert output == "hello" - - def test_set_item(self): - normalizers = Sequence( - [ - BertNormalizer(True, True), - Prepend(prepend="test"), - ] - ) - assert normalizers[0].__class__ == BertNormalizer - assert normalizers[1].__class__ == Prepend - normalizers[1] = Strip() - assert normalizers[1].__class__ == Strip - with pytest.raises(IndexError): - print(normalizers[2]) - - def test_item_getters_and_setters(self): - normalizers = Sequence( - [ - BertNormalizer(clean_text=True, handle_chinese_chars=True, strip_accents=True, lowercase=True), - Strip(left=True, right=True), - Prepend(prepend="_"), - Replace(pattern="something", content="else"), - ] - ) - - assert normalizers[0].__class__ == BertNormalizer - normalizers[0].clean_text = False - normalizers[0].handle_chinese_chars = False - normalizers[0].strip_accents = False - normalizers[0].lowercase = False - assert not normalizers[0].clean_text - assert not normalizers[0].handle_chinese_chars - assert not normalizers[0].strip_accents - assert not normalizers[0].lowercase - - assert normalizers[1].__class__ == Strip - normalizers[1].left = False - normalizers[1].right = False - assert not normalizers[1].left - assert not normalizers[1].right - - assert normalizers[2].__class__ == Prepend - normalizers[2].prepend = " " - assert normalizers[2].prepend == " " - - assert normalizers[3].__class__ == Replace - with pytest.raises(Exception): - normalizers[3].pattern = "test" - with pytest.raises(Exception): - print(normalizers[3].pattern) - normalizers[3].content = "test" - assert normalizers[3].content == "test" - - -class TestLowercase: - def test_instantiate(self): - assert isinstance(Lowercase(), Normalizer) - assert isinstance(Lowercase(), Lowercase) - assert isinstance(pickle.loads(pickle.dumps(Lowercase())), Lowercase) - - def test_lowercase(self): - normalizer = Lowercase() - - output = normalizer.normalize_str("HELLO") - assert output == "hello" - - -class TestStrip: - def test_instantiate(self): - assert isinstance(Strip(), Normalizer) - assert isinstance(Strip(), Strip) - assert isinstance(pickle.loads(pickle.dumps(Strip())), Strip) - - def test_left_strip(self): - normalizer = Strip(left=True, right=False) - - output = normalizer.normalize_str(" hello ") - assert output == "hello " - - def test_right_strip(self): - normalizer = Strip(left=False, right=True) - - output = normalizer.normalize_str(" hello ") - assert output == " hello" - - def test_full_strip(self): - normalizer = Strip(left=True, right=True) - - output = normalizer.normalize_str(" hello ") - assert output == "hello" - - def test_can_modify(self): - normalizer = Strip(left=True, right=True) - - assert normalizer.left == True - assert normalizer.right == True - - # Modify these - normalizer.left = False - assert normalizer.left == False - normalizer.right = False - assert normalizer.right == False - - -class TestPrepend: - def test_instantiate(self): - assert isinstance(Prepend("▁"), Normalizer) - assert isinstance(Prepend("▁"), Prepend) - assert isinstance(pickle.loads(pickle.dumps(Prepend("▁"))), Prepend) - - def test_prepend(self): - normalizer = Prepend(prepend="▁") - - output = normalizer.normalize_str("hello") - assert output == "▁hello" - - def test_can_modify(self): - normalizer = Prepend("▁") - - assert normalizer.prepend == "▁" - - # Modify these - normalizer.prepend = "-" - assert normalizer.prepend == "-" - - -class TestCustomNormalizer: - class BadCustomNormalizer: - def normalize(self, normalized, wrong): - pass - - class GoodCustomNormalizer: - def normalize(self, normalized): - self.kept_normalized = normalized - normalized.replace("there", "you") - - def use_after_normalize(self): - self.kept_normalized.replace("something", "else") - - def test_instantiate(self): - bad = Normalizer.custom(TestCustomNormalizer.BadCustomNormalizer()) - good_custom = TestCustomNormalizer.GoodCustomNormalizer() - good = Normalizer.custom(good_custom) - - assert isinstance(bad, Normalizer) - assert isinstance(good, Normalizer) - with pytest.raises(Exception, match="TypeError:.*normalize()"): - bad.normalize_str("Hey there!") - assert good.normalize_str("Hey there!") == "Hey you!" - with pytest.raises(Exception, match="Cannot use a NormalizedStringRefMut outside `normalize`"): - good_custom.use_after_normalize() - - def test_normalizer_interface(self): - normalizer = Normalizer.custom(TestCustomNormalizer.GoodCustomNormalizer()) - - normalized = NormalizedString("Hey there!") - normalizer.normalize(normalized) - - assert repr(normalized) == 'NormalizedString(original="Hey there!", normalized="Hey you!")' - assert str(normalized) == "Hey you!" diff --git a/bindings/python/tests/bindings/test_pre_tokenizers.py b/bindings/python/tests/bindings/test_pre_tokenizers.py deleted file mode 100644 index 8ed865384..000000000 --- a/bindings/python/tests/bindings/test_pre_tokenizers.py +++ /dev/null @@ -1,362 +0,0 @@ -import json -import pickle - -import pytest - -from tokenizers.pre_tokenizers import ( - BertPreTokenizer, - ByteLevel, - CharDelimiterSplit, - Digits, - FixedLength, - Metaspace, - PreTokenizer, - Punctuation, - Sequence, - Split, - UnicodeScripts, - Whitespace, - WhitespaceSplit, -) - - -class TestByteLevel: - def test_instantiate(self): - assert ByteLevel() is not None - assert ByteLevel(add_prefix_space=True) is not None - assert ByteLevel(add_prefix_space=False) is not None - assert isinstance(ByteLevel(), PreTokenizer) - assert isinstance(ByteLevel(), ByteLevel) - assert isinstance(pickle.loads(pickle.dumps(ByteLevel())), ByteLevel) - - def test_has_alphabet(self): - assert isinstance(ByteLevel.alphabet(), list) - assert len(ByteLevel.alphabet()) == 256 - - def test_can_modify(self): - pretok = ByteLevel(add_prefix_space=False) - - assert pretok.add_prefix_space == False - - # Modify these - pretok.add_prefix_space = True - assert pretok.add_prefix_space == True - - def test_manual_reload(self): - byte_level = ByteLevel() - state = json.loads(byte_level.__getstate__()) - reloaded = ByteLevel(**state) - assert isinstance(reloaded, ByteLevel) - - -class TestSplit: - def test_instantiate(self): - pre_tokenizer = Split(pattern=" ", behavior="removed") - assert pre_tokenizer is not None - assert isinstance(pre_tokenizer, PreTokenizer) - assert isinstance(pre_tokenizer, Split) - assert isinstance(pickle.loads(pickle.dumps(Split(" ", "removed"))), Split) - - # test with invert=True - pre_tokenizer_with_invert = Split(pattern=" ", behavior="isolated", invert=True) - assert pre_tokenizer_with_invert is not None - assert isinstance(pre_tokenizer_with_invert, PreTokenizer) - assert isinstance(pre_tokenizer_with_invert, Split) - assert isinstance(pickle.loads(pickle.dumps(Split(" ", "removed", True))), Split) - - -class TestWhitespace: - def test_instantiate(self): - assert Whitespace() is not None - assert isinstance(Whitespace(), PreTokenizer) - assert isinstance(Whitespace(), Whitespace) - assert isinstance(pickle.loads(pickle.dumps(Whitespace())), Whitespace) - - -class TestWhitespaceSplit: - def test_instantiate(self): - assert WhitespaceSplit() is not None - assert isinstance(WhitespaceSplit(), PreTokenizer) - assert isinstance(WhitespaceSplit(), WhitespaceSplit) - assert isinstance(pickle.loads(pickle.dumps(WhitespaceSplit())), WhitespaceSplit) - - -class TestBertPreTokenizer: - def test_instantiate(self): - assert BertPreTokenizer() is not None - assert isinstance(BertPreTokenizer(), PreTokenizer) - assert isinstance(BertPreTokenizer(), BertPreTokenizer) - assert isinstance(pickle.loads(pickle.dumps(BertPreTokenizer())), BertPreTokenizer) - - -class TestMetaspace: - def test_instantiate(self): - assert Metaspace() is not None - assert Metaspace(replacement="-") is not None - with pytest.raises(ValueError, match="expected a string of length 1"): - Metaspace(replacement="") - assert Metaspace(prepend_scheme="always") is not None - assert isinstance(Metaspace(), PreTokenizer) - assert isinstance(Metaspace(), Metaspace) - assert isinstance(pickle.loads(pickle.dumps(Metaspace())), Metaspace) - - def test_can_modify(self): - pretok = Metaspace(replacement="$", prepend_scheme="never") - - assert pretok.replacement == "$" - assert pretok.prepend_scheme == "never" - assert pretok.split == True - - # Modify these - pretok.replacement = "%" - assert pretok.replacement == "%" - pretok.prepend_scheme = "first" - assert pretok.prepend_scheme == "first" - pretok.split = True - assert pretok.split == True - - -class TestCharDelimiterSplit: - def test_instantiate(self): - assert CharDelimiterSplit("-") is not None - with pytest.raises(ValueError, match="expected a string of length 1"): - CharDelimiterSplit("") - assert isinstance(CharDelimiterSplit(" "), PreTokenizer) - assert isinstance(CharDelimiterSplit(" "), CharDelimiterSplit) - assert isinstance(pickle.loads(pickle.dumps(CharDelimiterSplit("-"))), CharDelimiterSplit) - - def test_can_modify(self): - pretok = CharDelimiterSplit("@") - assert pretok.delimiter == "@" - - # Modify these - pretok.delimiter = "!" - assert pretok.delimiter == "!" - - -class TestPunctuation: - def test_instantiate(self): - assert Punctuation() is not None - assert Punctuation("removed") is not None - assert isinstance(Punctuation(), PreTokenizer) - assert isinstance(Punctuation(), Punctuation) - assert isinstance(pickle.loads(pickle.dumps(Punctuation())), Punctuation) - - -class TestSequence: - def test_instantiate(self): - assert Sequence([]) is not None - assert isinstance(Sequence([]), PreTokenizer) - assert isinstance(Sequence([]), Sequence) - dumped = pickle.dumps(Sequence([])) - assert isinstance(pickle.loads(dumped), Sequence) - - def test_bert_like(self): - pre_tokenizer = Sequence([WhitespaceSplit(), Punctuation()]) - assert isinstance(Sequence([]), PreTokenizer) - assert isinstance(Sequence([]), Sequence) - assert isinstance(pickle.loads(pickle.dumps(pre_tokenizer)), Sequence) - - result = pre_tokenizer.pre_tokenize_str("Hey friend! How are you?!?") - assert result == [ - ("Hey", (0, 3)), - ("friend", (4, 10)), - ("!", (10, 11)), - ("How", (16, 19)), - ("are", (20, 23)), - ("you", (24, 27)), - ("?", (27, 28)), - ("!", (28, 29)), - ("?", (29, 30)), - ] - - def test_set_item(self): - pre_tokenizers = Sequence( - [ - ByteLevel(), - Split(pattern="/test/", behavior="removed"), - ] - ) - assert pre_tokenizers[0].__class__ == ByteLevel - assert pre_tokenizers[1].__class__ == Split - pre_tokenizers[1] = Metaspace() - assert pre_tokenizers[1].__class__ == Metaspace - with pytest.raises(IndexError): - print(pre_tokenizers[2]) - - def test_item_getters_and_setters(self): - pre_tokenizers = Sequence( - [ - ByteLevel(add_prefix_space=True, trim_offsets=True, use_regex=True), - Split(pattern="/test/", behavior="removed", invert=False), - Metaspace("a", "never", split=False), - CharDelimiterSplit(delimiter=" "), - Punctuation(behavior="removed"), - Digits(individual_digits=True), - ] - ) - - assert pre_tokenizers[0].__class__ == ByteLevel - pre_tokenizers[0].add_prefix_space = False - pre_tokenizers[0].trim_offsets = False - pre_tokenizers[0].use_regex = False - assert not pre_tokenizers[0].add_prefix_space - assert not pre_tokenizers[0].trim_offsets - assert not pre_tokenizers[0].use_regex - - assert pre_tokenizers[1].__class__ == Split - with pytest.raises(Exception): - pre_tokenizers[1].pattern = "/pattern/" - pre_tokenizers[1].behavior = "isolated" - pre_tokenizers[1].invert = True - with pytest.raises(Exception): - pre_tokenizers[1].pattern - assert pre_tokenizers[1].behavior == "isolated" - assert pre_tokenizers[1].invert - - assert pre_tokenizers[2].__class__ == Metaspace - pre_tokenizers[2].replacement = " " - pre_tokenizers[2].prepend_scheme = "always" - pre_tokenizers[2].split = True - assert pre_tokenizers[2].replacement == " " - assert pre_tokenizers[2].prepend_scheme == "always" - assert pre_tokenizers[2].split - - assert pre_tokenizers[3].__class__ == CharDelimiterSplit - pre_tokenizers[3].delimiter = "_" - assert pre_tokenizers[3].delimiter == "_" - - assert pre_tokenizers[4].__class__ == Punctuation - pre_tokenizers[4].behavior = "isolated" - assert pre_tokenizers[4].behavior == "isolated" - - assert pre_tokenizers[5].__class__ == Digits - pre_tokenizers[5].individual_digits = False - assert not pre_tokenizers[5].individual_digits - - -class TestDigits: - def test_instantiate(self): - assert Digits() is not None - assert isinstance(Digits(), PreTokenizer) - assert isinstance(Digits(), Digits) - assert isinstance(Digits(True), Digits) - assert isinstance(Digits(False), Digits) - assert isinstance(pickle.loads(pickle.dumps(Digits())), Digits) - - def test_can_modify(self): - pretok = Digits(individual_digits=False) - assert pretok.individual_digits == False - - # Modify these - pretok.individual_digits = True - assert pretok.individual_digits == True - - -class TestFixedLength: - def test_instantiate(self): - assert FixedLength() is not None - assert isinstance(FixedLength(), PreTokenizer) - assert isinstance(FixedLength(), FixedLength) - assert isinstance(pickle.loads(pickle.dumps(FixedLength())), FixedLength) - - def test_pre_tokenize_str(self): - pretok = FixedLength(length=5) - assert pretok.length == 5 - assert pretok.pre_tokenize_str("ATCCTGGTACTG") == [ - ("ATCCT", (0, 5)), - ("GGTAC", (5, 10)), - ("TG", (10, 12)), - ] - - pretok.length = 10 - assert pretok.length == 10 - assert pretok.pre_tokenize_str("ATCCTGGTACTG") == [ - ("ATCCTGGTAC", (0, 10)), - ("TG", (10, 12)), - ] - - -class TestUnicodeScripts: - def test_instantiate(self): - assert UnicodeScripts() is not None - assert isinstance(UnicodeScripts(), PreTokenizer) - assert isinstance(UnicodeScripts(), UnicodeScripts) - assert isinstance(pickle.loads(pickle.dumps(UnicodeScripts())), UnicodeScripts) - - -class TestCustomPreTokenizer: - class BadCustomPretok: - def pre_tokenize(self, pretok, wrong): - # This method does not have the right signature: it takes one too many arg - pass - - class GoodCustomPretok: - def split(self, n, normalized): - # Here we just test that we can return a List[NormalizedString], it - # does not really make sense to return twice the same otherwise - return [normalized, normalized] - - def pre_tokenize(self, pretok): - pretok.split(self.split) - - def test_instantiate(self): - bad = PreTokenizer.custom(TestCustomPreTokenizer.BadCustomPretok()) - good = PreTokenizer.custom(TestCustomPreTokenizer.GoodCustomPretok()) - - assert isinstance(bad, PreTokenizer) - assert isinstance(good, PreTokenizer) - with pytest.raises(Exception, match="TypeError:.*pre_tokenize()"): - bad.pre_tokenize_str("Hey there!") - assert good.pre_tokenize_str("Hey there!") == [ - ("Hey there!", (0, 10)), - ("Hey there!", (0, 10)), - ] - - def test_camel_case(self): - class CamelCasePretok: - def get_state(self, c): - if c.islower(): - return "lower" - elif c.isupper(): - return "upper" - elif c.isdigit(): - return "digit" - else: - return "rest" - - def split(self, n, normalized): - i = 0 - # states = {"any", "lower", "upper", "digit", "rest"} - state = "any" - pieces = [] - for j, c in enumerate(normalized.normalized): - c_state = self.get_state(c) - if state == "any": - state = c_state - if state != "rest" and state == c_state: - pass - elif state == "upper" and c_state == "lower": - pass - else: - pieces.append(normalized[i:j]) - i = j - state = c_state - pieces.append(normalized[i:]) - return pieces - - def pre_tokenize(self, pretok): - pretok.split(self.split) - - camel = PreTokenizer.custom(CamelCasePretok()) - - assert camel.pre_tokenize_str("HeyThere!?-ThisIsLife") == [ - ("Hey", (0, 3)), - ("There", (3, 8)), - ("!", (8, 9)), - ("?", (9, 10)), - ("-", (10, 11)), - ("This", (11, 15)), - ("Is", (15, 17)), - ("Life", (17, 21)), - ] diff --git a/bindings/python/tests/bindings/test_processors.py b/bindings/python/tests/bindings/test_processors.py deleted file mode 100644 index e75c3bd88..000000000 --- a/bindings/python/tests/bindings/test_processors.py +++ /dev/null @@ -1,256 +0,0 @@ -import json -import pickle - -import pytest - -from tokenizers import Tokenizer -from tokenizers.models import BPE -from tokenizers.pre_tokenizers import ByteLevel as ByteLevelPreTokenizer -from tokenizers.processors import ( - BertProcessing, - ByteLevel, - PostProcessor, - RobertaProcessing, - Sequence, - TemplateProcessing, -) - -from ..utils import data_dir, roberta_files - - -class TestBertProcessing: - def test_instantiate(self): - processor = BertProcessing(("[SEP]", 0), ("[CLS]", 1)) - assert processor is not None - assert isinstance(processor, PostProcessor) - assert isinstance(processor, BertProcessing) - assert isinstance( - pickle.loads(pickle.dumps(BertProcessing(("[SEP]", 0), ("[CLS]", 1)))), - BertProcessing, - ) - - def test_processing(self): - tokenizer = Tokenizer(BPE()) - tokenizer.add_special_tokens(["[SEP]", "[CLS]"]) - tokenizer.add_tokens(["my", "name", "is", "john", "pair"]) - tokenizer.post_processor = BertProcessing(("[SEP]", 0), ("[CLS]", 1)) - - output = tokenizer.encode("my name", "pair") - assert output.tokens == ["[CLS]", "my", "name", "[SEP]", "pair", "[SEP]"] - assert output.ids == [1, 2, 3, 0, 6, 0] - - -class TestRobertaProcessing: - def test_instantiate(self): - processor = RobertaProcessing(("", 1), ("", 0)) - assert processor is not None - assert isinstance(processor, PostProcessor) - assert isinstance(processor, RobertaProcessing) - assert isinstance( - pickle.loads(pickle.dumps(RobertaProcessing(("", 1), ("", 0)))), - RobertaProcessing, - ) - - def test_processing(self): - tokenizer = Tokenizer(BPE()) - tokenizer.add_special_tokens(["", ""]) - tokenizer.add_tokens(["my", "name", "is", "john", "pair"]) - tokenizer.post_processor = RobertaProcessing(("", 1), ("", 0)) - - output = tokenizer.encode("my name", "pair") - assert output.tokens == ["", "my", "name", "", "", "pair", ""] - assert output.ids == [0, 2, 3, 1, 1, 6, 1] - - -class TestByteLevelProcessing: - def test_instantiate(self): - assert ByteLevel() is not None - assert ByteLevel(trim_offsets=True) is not None - assert ByteLevel(add_prefix_space=True) is not None - assert isinstance(ByteLevel(), PostProcessor) - assert isinstance(ByteLevel(), ByteLevel) - assert isinstance(pickle.loads(pickle.dumps(ByteLevel())), ByteLevel) - - @pytest.mark.network - def test_processing(self, roberta_files): - tokenizer = Tokenizer(BPE(roberta_files["vocab"], roberta_files["merges"])) - tokenizer.pre_tokenizer = ByteLevelPreTokenizer(add_prefix_space=True) - - # Keeps original offsets - output = tokenizer.encode("My name is John") - assert output.tokens == ["ĠMy", "Ġname", "Ġis", "ĠJohn"] - assert output.offsets == [(0, 2), (2, 7), (7, 10), (10, 15)] - - # Trims offsets when activated - tokenizer.post_processor = ByteLevel(trim_offsets=True, add_prefix_space=True) - output = tokenizer.encode("My name is John") - assert output.tokens == ["ĠMy", "Ġname", "Ġis", "ĠJohn"] - assert output.offsets == [(0, 2), (3, 7), (8, 10), (11, 15)] - - # Trims offsets without adding prefix space at first token - tokenizer.post_processor = ByteLevel(trim_offsets=True, add_prefix_space=False) - output = tokenizer.encode("My name is John") - assert output.tokens == ["ĠMy", "Ġname", "Ġis", "ĠJohn"] - assert output.offsets == [(1, 2), (3, 7), (8, 10), (11, 15)] - - # add_prefix_space without trimming offsets has no effect - tokenizer.post_processor = ByteLevel(trim_offsets=False, add_prefix_space=True) - output = tokenizer.encode("My name is John") - assert output.tokens == ["ĠMy", "Ġname", "Ġis", "ĠJohn"] - assert output.offsets == [(0, 2), (2, 7), (7, 10), (10, 15)] - - def test_manual_reload(self): - byte_level = ByteLevel() - state = json.loads(byte_level.__getstate__()) - reloaded = ByteLevel(**state) - assert isinstance(reloaded, ByteLevel) - - -class TestTemplateProcessing: - def get_bert(self): - return TemplateProcessing( - single=["[CLS]", "$0", "[SEP]"], - pair=["[CLS]", "$A", "[SEP]", "$B:1", "[SEP]:1"], - special_tokens=[("[CLS]", 1), ("[SEP]", 0)], - ) - - def get_roberta(self): - return TemplateProcessing( - single=" $0 ", - pair=" $A $B ", - special_tokens=[("", 0), ("", 1)], - ) - - def get_t5_squad(self): - # >>> from transformers import AutoTokenizer - # >>> tok = AutoTokenizer.from_pretrained("t5-small") - # >>> tok.tokenize("question: ") - # ['▁question', ':'] - # >>> tok.tokenize("context: ") - # ['▁context', ':'] - # >>> tok.encode("context: ") - # [2625, 10] - # >>> tok.encode("question: ") - # [822, 10] - - return TemplateProcessing( - single=["$0"], - pair=["Q", "$A", "C", "$B"], - special_tokens=[ - { - "id": "Q", - "ids": [2625, 10], - "tokens": ["_question", ":"], - }, - { - "id": "C", - "ids": [822, 10], - "tokens": ["_context", ":"], - }, - ], - ) - - def test_instantiate(self): - bert = self.get_bert() - assert bert is not None - assert isinstance(bert, PostProcessor) - assert isinstance(bert, TemplateProcessing) - assert isinstance(pickle.loads(pickle.dumps(bert)), TemplateProcessing) - - # It is absolutely legal to have tokens with spaces in the name: - TemplateProcessing( - single=["[ C L S ]", "Token with space"], - special_tokens=[("[ C L S ]", 0), ("Token with space", 1)], - ) - # Sequence identifiers must be well formed: - with pytest.raises(Exception, match="Cannot build Piece"): - TemplateProcessing(single="[CLS] $$ [SEP]") - with pytest.raises(Exception, match="Cannot build Piece"): - TemplateProcessing(single="[CLS] $A: [SEP]") - # Special tokens must be provided when used in template: - with pytest.raises(Exception, match="Missing SpecialToken\\(s\\) with id\\(s\\)"): - TemplateProcessing(single=["[CLS]"]) - - def test_bert_parity(self): - tokenizer = Tokenizer(BPE()) - tokenizer.add_special_tokens(["[SEP]", "[CLS]"]) - tokenizer.add_tokens(["my", "name", "is", "john", "pair"]) - tokenizer.post_processor = BertProcessing(("[SEP]", 0), ("[CLS]", 1)) - - original = tokenizer.encode("my name", "pair") - - tokenizer.post_processor = self.get_bert() - template = tokenizer.encode("my name", "pair") - assert original.ids == template.ids - - def test_roberta_parity(self): - tokenizer = Tokenizer(BPE()) - tokenizer.add_special_tokens(["", ""]) - tokenizer.add_tokens(["my", "name", "is", "john", "pair"]) - tokenizer.post_processor = RobertaProcessing(("", 1), ("", 0)) - - original = tokenizer.encode("my name is john", "pair") - tokenizer.post_processor = self.get_roberta() - template = tokenizer.encode("my name is john", "pair") - assert original.ids == template.ids - - -class TestSequenceProcessing: - def test_sequence_processing(self): - assert Sequence([]) is not None - assert Sequence([ByteLevel()]) is not None - assert isinstance(Sequence([]), PostProcessor) - assert isinstance(Sequence([]), Sequence) - serialized = pickle.dumps(Sequence([])) - assert isinstance(pickle.loads(serialized), Sequence) - - def test_post_process(self): - byte_level = ByteLevel(trim_offsets=True) - template = TemplateProcessing( - single=["[CLS]", "$0", "[SEP]"], - pair=["[CLS]:0", "$A", "[SEP]:0", "$B:1", "[SEP]:1"], - special_tokens=[("[CLS]", 1), ("[SEP]", 0)], - ) - - tokenizer = Tokenizer(BPE()) - tokenizer.add_special_tokens(["[SEP]", "[CLS]"]) - tokenizer.add_tokens(["my", "name", "is", "Ġjohn", "pair"]) - tokenizer.post_processor = template - - # Before the sequence - original = tokenizer.encode("my name is Ġjohn") - assert original.ids == [1, 2, 3, 4, 5, 0] - assert original.type_ids == [0, 0, 0, 0, 0, 0] - assert original.offsets == [(0, 0), (0, 2), (3, 7), (8, 10), (11, 16), (0, 0)] - pair = tokenizer.encode("my name is Ġjohn", "pair") - # assert pair.ids == [1, 2, 3, 4, 5, 0, 6, 0] - assert pair.type_ids == [0, 0, 0, 0, 0, 0, 1, 1] - assert pair.offsets == [(0, 0), (0, 2), (3, 7), (8, 10), (11, 16), (0, 0), (0, 4), (0, 0)] - - processor = Sequence([byte_level, template]) - tokenizer.post_processor = processor - - original = tokenizer.encode("my name is Ġjohn") - assert original.ids == [1, 2, 3, 4, 5, 0] - assert original.type_ids == [0, 0, 0, 0, 0, 0] - # Offsets ARE trimmed - assert original.offsets == [(0, 0), (0, 2), (3, 7), (8, 10), (12, 16), (0, 0)] - pair = tokenizer.encode("my name is Ġjohn", "pair") - # assert pair.ids == [1, 2, 3, 4, 5, 0, 6, 0] - assert pair.type_ids == [0, 0, 0, 0, 0, 0, 1, 1] - assert pair.offsets == [(0, 0), (0, 2), (3, 7), (8, 10), (12, 16), (0, 0), (0, 4), (0, 0)] - - def test_items(self): - processors = Sequence([RobertaProcessing(("", 1), ("", 0)), ByteLevel()]) - assert processors[0].__class__ == RobertaProcessing - assert processors[1].__class__ == ByteLevel - processors[0] = ByteLevel(add_prefix_space=False, trim_offsets=False, use_regex=False) - print(processors[0]) - processors[0].add_prefix_space = True - processors[0].trim_offsets = True - processors[0].use_regex = True - print(processors[0]) - assert processors[0].__class__ == ByteLevel - assert processors[0].add_prefix_space - assert processors[0].trim_offsets - assert processors[0].use_regex diff --git a/bindings/python/tests/bindings/test_tokenizer.py b/bindings/python/tests/bindings/test_tokenizer.py deleted file mode 100644 index 55cb0cb4b..000000000 --- a/bindings/python/tests/bindings/test_tokenizer.py +++ /dev/null @@ -1,1107 +0,0 @@ -import pickle -import copy -import concurrent.futures -import pytest -import numpy as np -import asyncio -from tokenizers import AddedToken, Encoding, Tokenizer, decoders -from tokenizers.implementations import BertWordPieceTokenizer -from tokenizers.models import BPE, Model, Unigram -from tokenizers.pre_tokenizers import ByteLevel, Metaspace -from tokenizers.processors import RobertaProcessing, TemplateProcessing -from tokenizers.normalizers import Strip, Lowercase, Sequence -from tokenizers.normalizers import ByteLevel as NormalizerByteLevel -from tokenizers.decoders import ByteFallback, DecodeStream, Metaspace as DecoderMetaspace -import time - -from ..utils import bert_files, data_dir, multiprocessing_with_parallelism, roberta_files - - -class TestAddedToken: - def test_instantiate_with_content_only(self): - added_token = AddedToken("") - added_token.content = "" - assert added_token.content == "" - assert type(added_token) == AddedToken - added_token.content = added_token.content.lower() - - assert added_token.special == False - added_token.special = True - assert added_token.special == True - added_token.special = False - assert str(added_token) == "" - assert ( - repr(added_token) - == 'AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=True, special=False)' - ) - assert added_token.rstrip == False - assert added_token.lstrip == False - assert added_token.single_word == False - assert added_token.normalized == True - assert isinstance(pickle.loads(pickle.dumps(added_token)), AddedToken) - - def test_can_set_rstrip(self): - added_token = AddedToken("", rstrip=True) - assert added_token.rstrip == True - assert added_token.lstrip == False - assert added_token.single_word == False - assert added_token.normalized == True - - def test_can_set_lstrip(self): - added_token = AddedToken("", lstrip=True) - assert added_token.rstrip == False - assert added_token.lstrip == True - assert added_token.single_word == False - assert added_token.normalized == True - - def test_can_set_single_world(self): - added_token = AddedToken("", single_word=True) - assert added_token.rstrip == False - assert added_token.lstrip == False - assert added_token.single_word == True - assert added_token.normalized == True - - def test_can_set_normalized(self): - added_token = AddedToken("", normalized=False) - assert added_token.rstrip == False - assert added_token.lstrip == False - assert added_token.single_word == False - assert added_token.normalized == False - - -class TestTokenizer: - def test_has_expected_type_and_methods(self): - tokenizer = Tokenizer(BPE()) - assert type(tokenizer) == Tokenizer - assert callable(tokenizer.num_special_tokens_to_add) - assert callable(tokenizer.get_vocab) - assert callable(tokenizer.get_vocab_size) - assert callable(tokenizer.enable_truncation) - assert callable(tokenizer.no_truncation) - assert callable(tokenizer.enable_padding) - assert callable(tokenizer.no_padding) - assert callable(tokenizer.encode) - assert callable(tokenizer.encode_batch) - assert callable(tokenizer.async_encode_batch) - assert callable(tokenizer.decode) - assert callable(tokenizer.decode_batch) - assert callable(tokenizer.async_decode_batch) - assert callable(tokenizer.token_to_id) - assert callable(tokenizer.id_to_token) - assert callable(tokenizer.add_tokens) - assert callable(tokenizer.add_special_tokens) - assert callable(tokenizer.train) - assert callable(tokenizer.post_process) - assert isinstance(tokenizer.model, Model) - assert tokenizer.normalizer is None - assert tokenizer.pre_tokenizer is None - assert tokenizer.post_processor is None - assert tokenizer.decoder is None - assert isinstance(pickle.loads(pickle.dumps(Tokenizer(BPE()))), Tokenizer) - - def test_add_tokens(self): - tokenizer = Tokenizer(BPE()) - added = tokenizer.add_tokens(["my", "name", "is", "john"]) - assert added == 4 - - tokens = [AddedToken("the"), AddedToken("quick", normalized=False), AddedToken()] - assert tokens[0].normalized == True - added = tokenizer.add_tokens(tokens) - assert added == 2 - assert tokens[0].normalized == True - assert tokens[1].normalized == False - - def test_add_tokens_with_normalizer(self): - tokenizer = Tokenizer(BPE()) - tokenizer.normalizer = NormalizerByteLevel() - tokenizer.decoder = decoders.ByteLevel() - - new_tokens = [AddedToken("Začnimo", normalized=False, special=True), AddedToken("kuća"), AddedToken("međa")] - tokenizer.add_tokens(new_tokens) - enc = tokenizer.encode(new_tokens[0].content + new_tokens[1].content + " " + new_tokens[2].content) - assert tokenizer.decode(enc.ids, False) == "Za\rnimokućameđa" - - # Original content must be preserved in the decoder map regardless of normalization - decoder_map = tokenizer.get_added_tokens_decoder() - assert decoder_map[enc.ids[0]].content == "Začnimo" - assert decoder_map[enc.ids[1]].content == "kuća" - assert decoder_map[enc.ids[2]].content == "međa" - - def test_normalizer_change_refreshes_added_tokens(self): - """Changing tokenizer.normalizer must re-normalize added tokens and rebuild the trie.""" - tokenizer = Tokenizer(BPE()) - tokenizer.decoder = decoders.ByteLevel() - - # Add tokens *before* setting the normalizer — they should be re-processed - new_tokens = [AddedToken("kuća"), AddedToken("međa")] - tokenizer.add_tokens(new_tokens) - - # Now set the normalizer: refresh must happen automatically - tokenizer.normalizer = NormalizerByteLevel() - - enc = tokenizer.encode("kuća međa") - # Both tokens must be found and decode back to their original form - assert tokenizer.decode(enc.ids, False) == "kućameđa" - - # Unsetting the normalizer must also refresh (no normalization applied to the added token) - tokenizer.normalizer = None - enc2 = tokenizer.encode("kuća međa") - assert tokenizer.decode(enc2.ids, False) == "ku\x07ame\x11a" - - def test_add_special_tokens(self): - tokenizer = Tokenizer(BPE()) - - # Can add special tokens as `str` - added = tokenizer.add_special_tokens(["my", "name", "is", "john"]) - assert added == 4 - - # Can add special tokens as `AddedToken` - tokens = [AddedToken("the"), AddedToken("quick", normalized=True), AddedToken()] - assert tokens[0].normalized == True - added = tokenizer.add_special_tokens(tokens) - assert added == 2 - assert tokens[0].normalized == False - assert tokens[1].normalized == True - - def test_encode(self): - tokenizer = Tokenizer(BPE()) - tokenizer.add_tokens(["my", "name", "is", "john", "pair"]) - - # Can encode single sequence - output = tokenizer.encode("my name is john") - assert output.tokens == ["my", "name", "is", "john"] - assert type(output.ids) == list - assert type(output.type_ids) == list - assert type(output.offsets) == list - with pytest.warns(DeprecationWarning): - assert type(output.words) == list - assert type(output.word_ids) == list - assert type(output.special_tokens_mask) == list - assert type(output.attention_mask) == list - assert type(output.overflowing) == list - - # Can encode a pair of sequences - output = tokenizer.encode("my name is john", "pair") - assert output.tokens == ["my", "name", "is", "john", "pair"] - assert isinstance(pickle.loads(pickle.dumps(output)), Encoding) - - # Can encode a single pre-tokenized sequence - output = tokenizer.encode(["my", "name", "is", "john"], is_pretokenized=True) - assert output.tokens == ["my", "name", "is", "john"] - - # Can encode a batch with both a single sequence and a pair of sequences - output = tokenizer.encode_batch(["my name is john", ("my name is john", "pair")]) - assert len(output) == 2 - - @pytest.mark.network - def test_encode_formats(self, bert_files): - tokenizer = BertWordPieceTokenizer(bert_files["vocab"]) - - # Encode - output = tokenizer.encode("my name is john") - assert output.tokens == ["[CLS]", "my", "name", "is", "john", "[SEP]"] - output = tokenizer.encode("my name is john", "pair") - assert output.tokens == ["[CLS]", "my", "name", "is", "john", "[SEP]", "pair", "[SEP]"] - output = tokenizer.encode(["my", "name", "is", "john"], is_pretokenized=True) - assert output.tokens == ["[CLS]", "my", "name", "is", "john", "[SEP]"] - output = tokenizer.encode(["my", "name", "is", "john"], ["pair"], is_pretokenized=True) - assert output.tokens == ["[CLS]", "my", "name", "is", "john", "[SEP]", "pair", "[SEP]"] - - # Encode batch - result_single = [ - ["[CLS]", "my", "name", "is", "john", "[SEP]"], - ["[CLS]", "my", "name", "is", "georges", "[SEP]"], - ] - result_pair = [ - ["[CLS]", "my", "name", "is", "john", "[SEP]", "pair", "[SEP]"], - ["[CLS]", "my", "name", "is", "georges", "[SEP]", "pair", "[SEP]"], - ] - - def format(encodings): - return [e.tokens for e in encodings] - - def test_single(input, is_pretokenized=False): - output = tokenizer.encode_batch(input, is_pretokenized=is_pretokenized) - assert format(output) == result_single - - def test_pair(input, is_pretokenized=False): - output = tokenizer.encode_batch(input, is_pretokenized=is_pretokenized) - assert format(output) == result_pair - - # Classic inputs - - # Lists - test_single(["My name is John", "My name is Georges"]) - test_pair([("my name is john", "pair"), ("my name is georges", "pair")]) - test_pair([["my name is john", "pair"], ["my name is georges", "pair"]]) - - # Tuples - test_single(("My name is John", "My name is Georges")) - test_pair((("My name is John", "pair"), ("My name is Georges", "pair"))) - - # Numpy - test_single(np.array(["My name is John", "My name is Georges"])) - test_pair(np.array([("My name is John", "pair"), ("My name is Georges", "pair")])) - test_pair(np.array([["My name is John", "pair"], ["My name is Georges", "pair"]])) - - # PreTokenized inputs - - # Lists - test_single([["My", "name", "is", "John"], ["My", "name", "is", "Georges"]], True) - test_pair( - [ - (["My", "name", "is", "John"], ["pair"]), - (["My", "name", "is", "Georges"], ["pair"]), - ], - True, - ) - test_pair( - [ - [["My", "name", "is", "John"], ["pair"]], - [["My", "name", "is", "Georges"], ["pair"]], - ], - True, - ) - - # Tuples - test_single((("My", "name", "is", "John"), ("My", "name", "is", "Georges")), True) - test_pair( - ( - (("My", "name", "is", "John"), ("pair",)), - (("My", "name", "is", "Georges"), ("pair",)), - ), - True, - ) - test_pair( - ( - (["My", "name", "is", "John"], ["pair"]), - (["My", "name", "is", "Georges"], ["pair"]), - ), - True, - ) - - # Numpy - test_single( - np.array([["My", "name", "is", "John"], ["My", "name", "is", "Georges"]]), - True, - ) - test_single( - np.array((("My", "name", "is", "John"), ("My", "name", "is", "Georges"))), - True, - ) - test_pair( - np.array( - [ - [["My", "name", "is", "John"], ["pair"]], - [["My", "name", "is", "Georges"], ["pair"]], - ], - dtype=object, - ), - True, - ) - test_pair( - np.array( - ( - (("My", "name", "is", "John"), ("pair",)), - (("My", "name", "is", "Georges"), ("pair",)), - ), - dtype=object, - ), - True, - ) - - # Mal formed - with pytest.raises(TypeError, match="TextInputSequence must be str"): - tokenizer.encode([["my", "name"]]) # type: ignore[arg-type] - with pytest.raises(TypeError, match="TextInputSequence must be str"): - tokenizer.encode("My name is john", [["pair"]]) # type: ignore[arg-type] - with pytest.raises(TypeError, match="TextInputSequence must be str"): - tokenizer.encode("my name is john", ["pair"]) - - with pytest.raises(TypeError, match="InputSequence must be Union[List[str]"): - tokenizer.encode("My name is john", is_pretokenized=True) - with pytest.raises(TypeError, match="InputSequence must be Union[List[str]"): - tokenizer.encode("My name is john", ["pair"], is_pretokenized=True) - with pytest.raises(TypeError, match="InputSequence must be Union[List[str]"): - tokenizer.encode(["My", "name", "is", "John"], "pair", is_pretokenized=True) - - @pytest.mark.network - def test_encode_add_special_tokens(self, roberta_files): - tokenizer = Tokenizer(BPE(roberta_files["vocab"], roberta_files["merges"])) - tokenizer.add_special_tokens(["", ""]) - - tokenizer.pre_tokenizer = ByteLevel(add_prefix_space=True) - tokenizer.post_processor = RobertaProcessing( - ("", tokenizer.token_to_id("")), - ("", tokenizer.token_to_id("")), - ) - - # Can encode with special tokens - output_with_specials = tokenizer.encode("My name is John", add_special_tokens=True) - assert output_with_specials.tokens == ["", "ĠMy", "Ġname", "Ġis", "ĠJohn", ""] - - # Can encode without special tokens - output_without_specials = tokenizer.encode("My name is John", add_special_tokens=False) - assert output_without_specials.tokens == ["ĠMy", "Ġname", "Ġis", "ĠJohn"] - - def test_truncation(self): - tokenizer = Tokenizer(BPE()) - tokenizer.add_tokens(["my", "name", "is", "john", "pair"]) - tokenizer.enable_truncation(2) - - # Can truncate single sequences - output = tokenizer.encode("my name is john") - assert output.tokens == ["my", "name"] - - # Can truncate pair sequences as well - output = tokenizer.encode("my name is john", "pair") - assert output.tokens == ["my", "pair"] - - # Can get the params and give them to enable_truncation - trunc = tokenizer.truncation - tokenizer.enable_truncation(**trunc) - - # Left truncation direction - tokenizer.enable_truncation(2, direction="left") - output = tokenizer.encode("my name is john") - assert output.tokens == ["is", "john"] - - output = tokenizer.encode("my name is john", "pair") - assert output.tokens == ["john", "pair"] - - def test_padding(self): - tokenizer = Tokenizer(BPE()) - tokenizer.add_tokens(["my", "name", "is", "john", "pair"]) - - # By default it does nothing when encoding single sequence - tokenizer.enable_padding() - output = tokenizer.encode("my name") - assert output.tokens == ["my", "name"] - - # Can pad to the longest in a batch - output = tokenizer.encode_batch(["my name", "my name is john"]) - assert all([len(encoding) == 4 for encoding in output]) - - # Can pad to the specified length otherwise - tokenizer.enable_padding(length=4) - output = tokenizer.encode("my name") - assert output.tokens == ["my", "name", "[PAD]", "[PAD]"] - output = tokenizer.encode("my name", "pair") - assert output.tokens == ["my", "name", "pair", "[PAD]"] - - # Can get the params and give them to enable_padding - padding = tokenizer.padding - tokenizer.enable_padding(**padding) - - def test_decode(self): - tokenizer = Tokenizer(BPE()) - tokenizer.add_tokens(["my", "name", "is", "john", "pair"]) - - # Can decode single sequences - output = tokenizer.decode([0, 1, 2, 3]) - assert output == "my name is john" - - # Can decode batch - output = tokenizer.decode_batch([[0, 1, 2, 3], [4]]) - assert output == ["my name is john", "pair"] - - # Can decode stream - stream = DecodeStream(skip_special_tokens=False) - assert stream.step(tokenizer, 0) == "my" - assert stream.step(tokenizer, 1) == " name" - assert stream.step(tokenizer, 2) == " is" - assert stream.step(tokenizer, 3) == " john" - - stream = DecodeStream(ids=[0, 1, 2]) - assert stream.step(tokenizer, 3) == " john" - - def test_decode_stream_copy_and_prefix_ids(self): - tokenizer = Tokenizer(BPE()) - tokenizer.add_tokens(["my", "name", "is", "john"]) - token_ids = [0, 1, 2, 3] - - stream = DecodeStream(skip_special_tokens=False) - assert stream.step(tokenizer, token_ids[0]) == "my" - assert stream.step(tokenizer, token_ids[1]) == " name" - stream_copy = copy.copy(stream) - assert stream.step(tokenizer, token_ids[2]) == " is" - assert stream_copy.step(tokenizer, token_ids[2]) == " is" - assert stream.step(tokenizer, token_ids[3]) == " john" - assert stream_copy.step(tokenizer, token_ids[3]) == " john" - - stream_steps = DecodeStream([]) - last_chunk = None - for tid in token_ids: - last_chunk = stream_steps.step(tokenizer, tid) - stream_prefill = DecodeStream(token_ids[:-1]) - assert stream_prefill.step(tokenizer, token_ids[-1]) == last_chunk - - @pytest.mark.network - def test_decode_stream_fallback(self): - tokenizer = Tokenizer.from_pretrained("gpt2") - # tokenizer.decode([255]) fails because its a fallback - # tokenizer.encode("อั").ids = [19567, 255, 19567, 109] - stream = DecodeStream() - stream.step(tokenizer, [19567]) - stream.step(tokenizer, [255]) - stream.step(tokenizer, [19567]) - out = stream.step(tokenizer, [109]) - assert out == "ั" - - stream = DecodeStream() - out = stream.step(tokenizer, [19567, 255, 19567, 109]) - assert out == "อั" - stream = DecodeStream() - stream.step(tokenizer, [19567]) - out = stream.step(tokenizer, [255, 19567, 109]) - assert out == "อั" - - stream = DecodeStream() - stream.step(tokenizer, [19567]) - first_out = stream.step(tokenizer, [255]) - assert first_out == "อ" - # since we emitted the 'อ', we can't produce 'อั' - out = stream.step(tokenizer, [19567, 109]) - assert out == "ั" - - stream = DecodeStream([19567, 255, 19567]) - # the stream's prefix is 'อ�' which is invalid, thus all ids are kept for the next step - out = stream.step(tokenizer, [109]) - assert out == "อั" - - @pytest.mark.network - def test_decode_skip_special_tokens(self): - tokenizer = Tokenizer.from_pretrained("hf-internal-testing/Llama-3.1-8B-Instruct") - - stream = DecodeStream([40]) - out = stream.step(tokenizer, [2846, 40, 40, 40]) - assert out == "'mIII" - - stream = DecodeStream( - [ - 128000, - 128006, - 9125, - 128007, - 271, - 38766, - 1303, - 33025, - 2696, - 25, - 6790, - 220, - 2366, - 18, - 198, - 15724, - 2696, - 25, - 220, - 1627, - 10263, - 220, - 2366, - 19, - 271, - 9514, - 527, - 264, - 11190, - 18328, - 13, - 128009, - 128006, - 882, - 128007, - 271, - 15339, - 11, - 1268, - 527, - 499, - 30, - 128009, - 128006, - 78191, - 128007, - 271, - ] - ) - out = stream.step(tokenizer, 40) - assert out == "I" - - stream = DecodeStream([40]) - out = stream.step(tokenizer, 2846) - assert out == "'m" - - stream = DecodeStream([40]) - out = stream.step(tokenizer, [2846, 40, 40, 40]) - assert out == "'mIII" - - def test_decode_stream(self): - vocab = [ - ("", 0.0), - ("<0x20>", -0.1), - ("<0xC3>", -0.2), - ("<0xA9>", -0.3), - ] - tokenizer = Tokenizer(Unigram(vocab, 0, byte_fallback=True)) - tokenizer.decoder = ByteFallback() - stream = DecodeStream(skip_special_tokens=False) - assert stream.step(tokenizer, 1) == " " - assert stream.step(tokenizer, 2) == None - assert stream.step(tokenizer, 3) == "é" - - vocab = [ - ("", 0.0), - ("▁This", -0.1), - ] - tokenizer = Tokenizer(Unigram(vocab, 0, byte_fallback=False)) - tokenizer.decoder = DecoderMetaspace() - stream = DecodeStream(skip_special_tokens=False) - assert stream.step(tokenizer, 1) == "This" - assert stream.step(tokenizer, 1) == " This" - - def test_get_vocab(self): - tokenizer = Tokenizer(BPE()) - tokenizer.add_tokens(["my", "name", "is", "john", "pair"]) - - # Can retrieve vocab with added tokens - vocab = tokenizer.get_vocab(with_added_tokens=True) - assert vocab == {"is": 2, "john": 3, "my": 0, "name": 1, "pair": 4} - - # Can retrieve vocab without added tokens - vocab = tokenizer.get_vocab(with_added_tokens=False) - assert vocab == {} - - # Can retrieve added token decoder - vocab = tokenizer.get_added_tokens_decoder() - assert vocab == { - 0: AddedToken("my", rstrip=False, lstrip=False, single_word=False, normalized=True, special=False), - 1: AddedToken("name", rstrip=False, lstrip=False, single_word=False, normalized=True, special=False), - 2: AddedToken("is", rstrip=False, lstrip=False, single_word=False, normalized=True, special=False), - 3: AddedToken("john", rstrip=False, lstrip=False, single_word=False, normalized=True, special=False), - 4: AddedToken("pair", rstrip=False, lstrip=False, single_word=False, normalized=True, special=False), - } - - def test_get_vocab_size(self): - tokenizer = Tokenizer(BPE()) - tokenizer.add_tokens(["my", "name", "is", "john", "pair"]) - - # Can retrieve vocab's size with added tokens - size = tokenizer.get_vocab_size(with_added_tokens=True) - assert size == 5 - - # Can retrieve vocab's size without added tokens - size = tokenizer.get_vocab_size(with_added_tokens=False) - assert size == 0 - - def test_post_process(self): - tokenizer = Tokenizer(BPE()) - tokenizer.add_tokens(["my", "name", "is", "john", "pair"]) - tokenizer.enable_truncation(2) - tokenizer.enable_padding(length=4) - - encoding = tokenizer.encode("my name is john") - pair_encoding = tokenizer.encode("pair") - - # Can post process a single encoding - output = tokenizer.post_process(encoding) - assert output.tokens == ["my", "name", "[PAD]", "[PAD]"] - - # Can post process a pair of encodings - output = tokenizer.post_process(encoding, pair_encoding) - assert output.tokens == ["my", "pair", "[PAD]", "[PAD]"] - - def test_multiprocessing_with_parallelism(self): - tokenizer = Tokenizer(BPE()) - multiprocessing_with_parallelism(tokenizer, False) - multiprocessing_with_parallelism(tokenizer, True) - - def test_multithreaded_concurrency(self): - # Create a single shared tokenizer instance (thread-safe) - shared_tokenizer = Tokenizer(BPE()) - - # Thread worker functions that use the SAME tokenizer instance - def encode_batch(batch): - return shared_tokenizer.encode_batch(batch) - - def encode_batch_fast(batch): - return shared_tokenizer.encode_batch_fast(batch) - - # Create some significant workload - batches = [ - ["my name is john " * 50] * 20, - ["my name is paul " * 50] * 20, - ["my name is ringo " * 50] * 20, - ] - - # Many encoding operations to run concurrently using the same tokenizer - tasks = [ - (encode_batch, batches[0]), - (encode_batch_fast, batches[1]), - (encode_batch, batches[2]), - ] * 10 - - executor = concurrent.futures.ThreadPoolExecutor(max_workers=4) - - futures = [] - for function, argument in tasks: - futures.append(executor.submit(function, argument)) - - # All tasks should complete successfully - results = [f.result() for f in futures] - - # Verify results - assert len(results) == 30 - assert all(len(result) == 20 for result in results) - - @pytest.mark.network - def test_from_pretrained(self): - tokenizer = Tokenizer.from_pretrained("bert-base-cased") - output = tokenizer.encode("Hey there dear friend!", add_special_tokens=False) - assert output.tokens == ["Hey", "there", "dear", "friend", "!"] - - @pytest.mark.network - def test_from_pretrained_revision(self): - tokenizer = Tokenizer.from_pretrained("anthony/tokenizers-test") - output = tokenizer.encode("Hey there dear friend!", add_special_tokens=False) - assert output.tokens == ["hey", "there", "dear", "friend", "!"] - - tokenizer = Tokenizer.from_pretrained("anthony/tokenizers-test", revision="gpt-2") - output = tokenizer.encode("Hey there dear friend!", add_special_tokens=False) - assert output.tokens == ["Hey", "Ġthere", "Ġdear", "Ġfriend", "!"] - - def test_unigram_byte_fallback(self): - vocab = [ - ("", 0.0), - ("A", -0.03), - ("sen", -0.02), - ("te", -0.03), - ("n", -0.04), - ("ce", -0.05), - ("<0xF0>", -0.06), - ("<0x9F>", -0.06), - ("<0xA4>", -0.06), - ("<0x97>", -0.06), - (" ", -0.4), - ] - tokenizer = tokenizer = Tokenizer(Unigram(vocab, 0, byte_fallback=False)) - - output = tokenizer.encode("A sentence 🤗") - assert output.ids == [1, 10, 2, 3, 4, 5, 10, 0] - assert output.tokens == ["A", " ", "sen", "te", "n", "ce", " ", "🤗"] - - tokenizer = Tokenizer(Unigram(vocab, 0, byte_fallback=True)) - - output = tokenizer.encode("A sentence 🤗") - assert output.ids == [1, 10, 2, 3, 4, 5, 10, 6, 7, 8, 9] - assert output.tokens == ["A", " ", "sen", "te", "n", "ce", " ", "<0xF0>", "<0x9F>", "<0xA4>", "<0x97>"] - - @pytest.mark.network - def test_encode_special_tokens(self): - tokenizer = Tokenizer.from_pretrained("t5-base") - tokenizer.add_tokens([""]) - tokenizer.add_special_tokens([""]) - output = tokenizer.encode("Hey there dearfriend!", add_special_tokens=False) - assert output.tokens == ["▁Hey", "▁there", "", "▁dear", "", "▁friend", "!"] - - tokenizer.encode_special_tokens = True - assert tokenizer.encode_special_tokens == True - - output = tokenizer.encode("Hey there dearfriend!", add_special_tokens=False) - assert output.tokens == [ - "▁Hey", - "▁there", - "<", - "end", - "_", - "of", - "_", - "text", - ">", - "▁dear", - "", - "▁friend", - "!", - ] - - tokenizer.add_tokens(["of_text>"]) - output = tokenizer.encode("Hey there dearfriend!", add_special_tokens=False) - assert output.tokens == ["▁Hey", "▁there", "<", "end", "_", "of_text>", "▁dear", "", "▁friend", "!"] - - @pytest.mark.network - def test_splitting(self): - tokenizer = Tokenizer.from_pretrained("hf-internal-testing/llama-new-metaspace") - tokenizer.pre_tokenizer.split = False - tokenizer.add_tokens([AddedToken("", rstrip=True, lstrip=True)]) - assert tokenizer.encode("inform. Hey. .", add_special_tokens=False).tokens == [ - "", - "in", - "form", - "", - ".", - "▁Hey", - ".", - "▁▁▁▁▁▁", - "▁.", - ] - - assert tokenizer.encode("inform. Hey. .", add_special_tokens=False).ids == [ - 32000, - 262, - 689, - 1, - 29889, - 18637, - 29889, - 539, - 869, - ] - - assert tokenizer.encode("inform. Hey. .").tokens == [ - "", - "▁inform", - "", - ".", - "▁Hey", - ".", - "▁▁▁▁▁▁", - "▁.", - ] - assert tokenizer.encode("inform. Hey. .", add_special_tokens=False).tokens == [ - "▁inform", - "", - ".", - "▁Hey", - ".", - "▁▁▁▁▁▁", - "▁.", - ] - - def test_decode_special(self): - tokenizer = Tokenizer(BPE()) - tokenizer.add_tokens([AddedToken("my", special=True), AddedToken("name", special=False), "is", "john", "pair"]) - - # Can decode single sequences - output = tokenizer.decode([0, 1, 2, 3], skip_special_tokens=False) - assert output == "my name is john" - - output = tokenizer.decode([0, 1, 2, 3], skip_special_tokens=True) - assert output == "name is john" - assert tokenizer.get_added_tokens_decoder()[0] == AddedToken("my", special=True) - - def test_weakref_support(self): - import weakref - - tokenizer = Tokenizer(BPE()) - weak_ref = weakref.ref(tokenizer) - - assert weak_ref() is not None - assert weak_ref() is tokenizer - - del tokenizer - assert weak_ref() is None - - def test_weakref_with_multiple_references(self): - import weakref - - tokenizer = Tokenizer(BPE()) - weak_ref = weakref.ref(tokenizer) - another_ref = tokenizer - - assert weak_ref() is not None - - del tokenizer - assert weak_ref() is not None - - del another_ref - assert weak_ref() is None - - def test_setting_to_none(self): - tokenizer = Tokenizer(BPE()) - tokenizer.normalizer = Strip() - tokenizer.normalizer = None - assert tokenizer.normalizer == None - - tokenizer.pre_tokenizer = Metaspace() - tokenizer.pre_tokenizer = None - assert tokenizer.pre_tokenizer == None - - -class TestTokenizerRepr: - def test_repr(self): - tokenizer = Tokenizer(BPE()) - out = repr(tokenizer) - assert ( - out - == 'Tokenizer(version="1.0", truncation=None, padding=None, added_tokens=[], normalizer=None, pre_tokenizer=None, post_processor=None, decoder=None, model=BPE(dropout=None, unk_token=None, continuing_subword_prefix=None, end_of_word_suffix=None, fuse_unk=False, byte_fallback=False, ignore_merges=False, vocab={}, merges=[]))' - ) - - def test_repr_complete(self): - tokenizer = Tokenizer(BPE()) - tokenizer.pre_tokenizer = ByteLevel(add_prefix_space=True) - tokenizer.post_processor = TemplateProcessing( - single=["[CLS]", "$0", "[SEP]"], - pair=["[CLS]:0", "$A", "[SEP]:0", "$B:1", "[SEP]:1"], - special_tokens=[("[CLS]", 1), ("[SEP]", 0)], - ) - tokenizer.normalizer = Sequence([Lowercase(), Strip()]) - out = repr(tokenizer) - assert ( - out - == 'Tokenizer(version="1.0", truncation=None, padding=None, added_tokens=[], normalizer=Sequence(normalizers=[Lowercase(), Strip(strip_left=True, strip_right=True)]), pre_tokenizer=ByteLevel(add_prefix_space=True, trim_offsets=True, use_regex=True), post_processor=TemplateProcessing(single=[SpecialToken(id="[CLS]", type_id=0), Sequence(id=A, type_id=0), SpecialToken(id="[SEP]", type_id=0)], pair=[SpecialToken(id="[CLS]", type_id=0), Sequence(id=A, type_id=0), SpecialToken(id="[SEP]", type_id=0), Sequence(id=B, type_id=1), SpecialToken(id="[SEP]", type_id=1)], special_tokens={"[CLS]":SpecialToken(id="[CLS]", ids=[1], tokens=["[CLS]"]), "[SEP]":SpecialToken(id="[SEP]", ids=[0], tokens=["[SEP]"])}), decoder=None, model=BPE(dropout=None, unk_token=None, continuing_subword_prefix=None, end_of_word_suffix=None, fuse_unk=False, byte_fallback=False, ignore_merges=False, vocab={}, merges=[]))' - ) - - -@pytest.mark.network -class TestAsyncTokenizer: - """Tests for async methods of the Tokenizer class.""" - - def setup_method(self): - """Setup a basic tokenizer before each test.""" - self.tokenizer = Tokenizer.from_pretrained("hf-internal-testing/gpt-oss-20b") - - async def _compare_sync_async(self, input_data, is_pretokenized=False, add_special_tokens=True): - """Helper to compare sync and async results for both normal and fast encoding.""" - # Normal encoding - sync_result = self.tokenizer.encode_batch(input_data, is_pretokenized, add_special_tokens) - async_result = await self.tokenizer.async_encode_batch(input_data, is_pretokenized, add_special_tokens) - - assert len(sync_result) == len(async_result) - for s, a in zip(sync_result, async_result): - assert s.tokens == a.tokens - assert s.ids == a.ids - assert s.offsets == a.offsets - assert s.attention_mask == a.attention_mask - assert s.special_tokens_mask == a.special_tokens_mask - assert s.type_ids == a.type_ids - - # Fast encoding - sync_fast_result = self.tokenizer.encode_batch_fast(input_data, is_pretokenized, add_special_tokens) - async_fast_result = await self.tokenizer.async_encode_batch_fast( - input_data, is_pretokenized, add_special_tokens - ) - - assert len(sync_fast_result) == len(async_fast_result) - for s, a in zip(sync_fast_result, async_fast_result): - assert s.tokens == a.tokens - assert s.ids == a.ids - assert s.attention_mask == a.attention_mask - assert s.special_tokens_mask == a.special_tokens_mask - assert s.type_ids == a.type_ids - - @pytest.mark.asyncio - async def test_basic_encoding(self): - """Test basic encoding functionality.""" - # Single sequences - await self._compare_sync_async(["my name is john", "my pair"]) - - # Pair sequences - await self._compare_sync_async([("my name", "is john"), ("my", "pair")]) - - # Empty batch - await self._compare_sync_async([]) - - @pytest.mark.asyncio - async def test_encode(self): - out = await self.tokenizer.async_encode("my name is john", "my pair") - no_async_out = self.tokenizer.encode("my name is john", "my pair") - assert out.ids == no_async_out.ids - - out = await self.tokenizer.async_encode("my name is john") - no_async_out = self.tokenizer.encode("my name is john") - assert out.ids == no_async_out.ids - - @pytest.mark.asyncio - async def test_with_special_tokens(self): - """Test with special tokens handling.""" - self.tokenizer.add_special_tokens(["[CLS]", "[SEP]"]) - self.tokenizer.post_processor = TemplateProcessing( - single=["[CLS]", "$0", "[SEP]"], - pair=["[CLS]", "$A", "[SEP]", "$B", "[SEP]"], - special_tokens=[ - ("[CLS]", self.tokenizer.token_to_id("[CLS]")), - ("[SEP]", self.tokenizer.token_to_id("[SEP]")), - ], - ) - - # With special tokens - await self._compare_sync_async(["my name is john", "my pair"], add_special_tokens=True) - - # Without special tokens - await self._compare_sync_async(["my name is john", "my pair"], add_special_tokens=False) - - @pytest.mark.asyncio - async def test_with_truncation_padding(self): - """Test with truncation and padding enabled.""" - self.tokenizer.enable_truncation(2) - self.tokenizer.enable_padding(length=4) - - # Single sequences - await self._compare_sync_async(["my name is john", "pair longer"]) - - # Pair sequences - await self._compare_sync_async([("my name", "is john"), ("pair", "longer")]) - - @pytest.mark.asyncio - async def test_various_input_formats(self): - """Test with various input formats.""" - # Lists - await self._compare_sync_async(["my name", "is john"]) - - # Tuples - await self._compare_sync_async(("my name", "is john")) - - # Numpy arrays - # await self._compare_sync_async(np.array(["my name", "is john"])) - - # Mixed pairs - await self._compare_sync_async([("my name", "is john"), ["my", "pair"]]) - - @pytest.mark.asyncio - async def test_error_handling(self): - """Test that errors are handled consistently.""" - # Invalid input type for single item - with pytest.raises(TypeError): - await self.tokenizer.async_encode_batch(123) - - with pytest.raises(TypeError): - self.tokenizer.encode_batch(123) - - # Invalid pre-tokenized input - with pytest.raises(TypeError): - await self.tokenizer.async_encode_batch("my name", is_pretokenized=True) - - with pytest.raises(TypeError): - self.tokenizer.encode_batch("my name", is_pretokenized=True) - - @pytest.mark.asyncio - async def test_concurrency(self): - """Test concurrent encoding operations.""" - # Create some significant workload - large_batch = ["my name is john " * 50] * 20 - - # Run multiple encoding operations concurrently - tasks = [ - self.tokenizer.async_encode_batch(large_batch), - self.tokenizer.async_encode_batch_fast(large_batch), - self.tokenizer.async_encode_batch(large_batch), - ] - - # They should all complete successfully - results = await asyncio.gather(*tasks) - - # Verify results - assert len(results) == 3 - assert all(len(result) == 20 for result in results) - - @pytest.mark.asyncio - async def test_decode(self): - tokenizer = Tokenizer(BPE()) - tokenizer.add_tokens(["my", "name", "is", "john", "pair"]) - - # Can decode single sequences - output = tokenizer.decode([0, 1, 2, 3]) - assert output == "my name is john" - - output = tokenizer.decode_batch([[0, 1, 2, 3], [4]]) - assert output == ["my name is john", "pair"] - - output = await tokenizer.async_decode_batch([[0, 1, 2, 3], [4]]) - assert output == ["my name is john", "pair"] - - @pytest.mark.asyncio - async def test_large_batch(self): - """Test encoding a large batch of sequences.""" - large_batch = ["my name is john"] * 1000 - - # Encode large batch both ways - async_result = await self.tokenizer.async_encode_batch_fast(large_batch) - sync_result = self.tokenizer.encode_batch_fast(large_batch) - - # Results should be identical - assert len(async_result) == len(sync_result) - assert all(a.tokens == s.tokens for a, s in zip(async_result[:10], sync_result[:10])) - - @pytest.mark.asyncio - async def test_numpy_inputs(self): - """Test with numpy array inputs.""" - # Single numpy array - input_array = np.array(["my name", "is john", "pair longer"]) - await self._compare_sync_async(input_array) - - # Pre-tokenized numpy array - pretok_array = np.array([["my", "name"], ["is", "john"]], dtype=object) - await self._compare_sync_async(pretok_array, is_pretokenized=True) - - def test_async_methods_existence(self): - """Test that the async methods exist on the Tokenizer class.""" - assert hasattr(self.tokenizer, "async_encode_batch") - assert hasattr(self.tokenizer, "async_encode_batch_fast") - assert callable(self.tokenizer.async_encode_batch) - assert callable(self.tokenizer.async_encode_batch_fast) - - @pytest.mark.asyncio - async def test_performance_comparison(self): - """Compare performance between sync and async methods (informational).""" - # Create a large batch for performance comparison - large_batch = [ - "short text", - "Sometimes it helps to have a better idea", - "More short", - "Let's not delve into that habbit sir", - "I believe we can get to", - "I am going to do it. I have made up my mind. These are the first few words of the new… the best … the Longest Text In The Entire History Of The Known Universe! This Has To Have Over 35,000 words the beat the current world record set by that person who made that flaming chicken handbooky thingy. I might just be saying random things the whole time I type in this so you might get confused a lot. I just discovered something terrible. autocorrect is on!! no!!! this has to be crazy, so I will have to break all the English language rules and the basic knowledge of the average human being. I am not an average human being, however I am special. no no no, not THAT kind of special ;). Why do people send that wink face! it always gives me nightmares! it can make a completely normal sentence creepy. imagine you are going to a friend’s house, so you text this: [ see you soon 🙂 ] seems normal, right? But what is you add the word semi to that colon? (Is that right? or is it the other way around) what is you add a lorry to that briquettes? (Semi-truck to that coal-on) anyway, back to the point: [ see you soon 😉 ]THAT IS JUST SO CREEPY! is that really your friend, or is it a creepy stalker watching your every move? Or even worse, is it your friend who is a creepy stalker? maybe you thought it was your friend, but it was actually your fri end (let me explain: you are happily in McDonalds, getting fat while eating yummy food and some random dude walks up and blots out the sun (he looks like a regular here) you can’t see anything else than him, so you can’t try to avoid eye contact. he finishes eating his cheeseburger (more like horseburgher(I learned that word from the merchant of Venice(which is a good play(if you can understand it(I can cause I got a special book with all the words in readable English written on the side of the page(which is kinda funny because Shakespeare was supposed to be a good poet but no-one can understand him(and he’s racist in act 2 scene1 of the play too))))))) and sits down beside you , like you are old pals (you’ve never met him before but he looks like he could be in some weird cult) he clears his throat and asks you a very personal question. “can i have some French fries?” (I don’t know why there called French fries when I’ve never seen a French person eat fries! all they eat it is stuff like baguettes and crêpes and rats named ratty-two-ee which is a really fun game on the PlayStation 2) And you think {bubbly cloud thinking bubble} “Hahahahahhahahahahahahahaha!!!!!!!!!!!! Hehheheheheh…..heeeheehe..hehe… sigh. I remember that i was just about to eat one of my fries when I noticed something mushy and moist and [insert gross color like green or brown] on the end of one of my fries! now I can give it to this NERD!! ” (yes he is a nerd because all he does all day is watch the extended editions of the hobbit, lord of the rings and star wars and eat fat cakes (what the heck is a fat cake? I think it might be like a Twinkie or something)and twinkies(wow so is doesn’t really matter which is which because he eats both(i may have just done that so I didn’t have to Google what a fat cake is (right now I am typing on my iPhone 3gs anyway, which has a broken antenna so i can’t get internet anyway (it’s actually a really funny story that i’ll tell you sometime)))and sit in his man cave with his friend named Joe (an ACTUAL friend, not a fri end)and all Joe does is watch sports like football with bob and all bob does is gamble ferociously (don’t ask(it means he buys all those bags of chips that say “win a free monkey or something if you find a banana in your bag*”(if there is a little star it means there is fine print so I always check the back of the package) *flips over the package* okay, it says: “one of our workers accidentally threw a banana in the packing machine and we don’t want to get sued so we did this promotion thing” cool. Oh wow, this is salt and vinegar! my favourite! i hate cheese and onion.))and that’s pretty much his life, he lives in Jamaica with Naruto and his friends) so you give him that gross fri end he throws up all over you and me and the worker behind the counter who was still making an onion, and THAT is the story of the fri end, not a friend who somehow remembered your name and your phone number / email so he could text you saying he would come to your house soon. *finally takes a breath after typing a few hundred words about fri-ends* so what now? i know, i know, you think i ramble too much and use too many brackets (i don’t) but now i am going to talk about my amAZEing day. first i woke up, ate choco pops for breakfast even tho i always hate it when people say that cause i get jealous and super hungry. then i… umm… yea! that was my day. you know that other person i mentioned before? that flaming chicken person? WELL. i will steal something from that person but do it better. i will… drum roll please … badabadabadabadabadabadabummmmmmmmmmmchshchshchshchshbadabadboumboumpoopoopichypichypichypowpow-crash! *a drum roll was just playing in the background* that drumroll was so long i forget what i was talking about. *scrolls up to see what he was writing about* oh yea! i will make my own FLAMING CHICKEN HANDBOOK! what things do i like? instead of flaming it could be rainbow, instead of chicken it could be fluffysheep and instead of handbook it could be handbook (not very creative, i know) but the total complete name is now to rainbow fluffysheep handbook! to make life easier for you guys, instead of taking random rules out of book willy nilly, i will take them out using my favourite numbers! so, section 5040 of the rainbow fluffysheep handbook states that the king of all oddly coloured farm animals (thats me!) is allowed to tell you any part out of this book randomly or if it is his one of his favorite numbers! 5040 is a great number because it is divisible by 60 integers which i don’t know. i’m tired. it is 10:41 and i am getting sleepy… hey hey hey! an intruder! remember that from pokepals rulers of time and darkness or something like that! with piplup and sunflora and chimchar! whaoh piplup is really hard to write on a tiny qwerty keyboard! try it! i realised that asdf is actually written in order on the qwerty keyboard! (just in case you didn’t know, asdf is an amazing short video clips cartoony thing on youtube i first learned bout on flipnote hatena, which is now shut down 😦 ) what if one day they get rid of the qwerty keyboard completely! i will type it out for you just in case one day they get rid of it.", - ] - results_sync = [] - results_async = [] - - # Pre-initialize a thread pool executor with a reasonable number of workers - # This avoids the overhead of creating the pool for each task - - try: - executor = concurrent.futures.ThreadPoolExecutor(max_workers=2048) - loop = asyncio.get_running_loop() - - async def encode_sync_with_executor(_): - # Use the pre-initialized executor - return await loop.run_in_executor(executor, lambda: self.tokenizer.encode_batch_fast(large_batch)) - - async def encode_to_thread_sync(_): - return await asyncio.to_thread(self.tokenizer.encode_batch_fast, large_batch) - - async def encode_async(_): - return await self.tokenizer.async_encode_batch_fast(large_batch) - - await asyncio.gather(*[encode_sync_with_executor(i) for i in range(2048)]) - await asyncio.gather(*[encode_async(i) for i in range(2048)]) - - for n_tasks in [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]: - # Measure sync performance with pre-initialized executor - # Warm up - await asyncio.gather(*[encode_sync_with_executor(i) for i in range(10)]) - time.sleep(0.03) - # Actual measurement - start = time.perf_counter() - await asyncio.gather(*[encode_sync_with_executor(i) for i in range(n_tasks)]) - sync_time = time.perf_counter() - start - - # Measure async performance - # Warm up - await asyncio.gather(*[encode_async(i) for i in range(10)]) - - # Actual measurement - time.sleep(0.03) - start = time.perf_counter() - await asyncio.gather(*[encode_async(i) for i in range(n_tasks)]) - async_time = time.perf_counter() - start - - # Log times - print(f"sync vs async processing times: {sync_time:.4f}s vs {async_time:.4f}s for {n_tasks} tasks") - results_sync.append(sync_time) - results_async.append(async_time) - finally: - # Make sure we shut down the executor properly - executor.shutdown(wait=False) diff --git a/bindings/python/tests/bindings/test_trainers.py b/bindings/python/tests/bindings/test_trainers.py deleted file mode 100644 index b40eeb2d6..000000000 --- a/bindings/python/tests/bindings/test_trainers.py +++ /dev/null @@ -1,405 +0,0 @@ -import copy -import os -import pickle - -import pytest - -from tokenizers import ( - AddedToken, - SentencePieceUnigramTokenizer, - Tokenizer, - models, - normalizers, - pre_tokenizers, - trainers, -) - -from ..utils import data_dir, train_files, DATA_PATH - - -class TestBpeTrainer: - def test_can_modify(self): - trainer = trainers.BpeTrainer( - vocab_size=12345, - min_frequency=12, - show_progress=False, - special_tokens=["1", "2"], - limit_alphabet=13, - initial_alphabet=["a", "b", "c"], - continuing_subword_prefix="pref", - end_of_word_suffix="suf", - ) - - assert trainer.vocab_size == 12345 - assert trainer.min_frequency == 12 - assert trainer.show_progress == False - assert trainer.special_tokens == [ - AddedToken("1", special=True), - AddedToken("2", special=True), - ] - assert trainer.limit_alphabet == 13 - assert sorted(trainer.initial_alphabet) == ["a", "b", "c"] - assert trainer.continuing_subword_prefix == "pref" - assert trainer.end_of_word_suffix == "suf" - - # Modify these - trainer.vocab_size = 20000 - assert trainer.vocab_size == 20000 - trainer.min_frequency = 1 - assert trainer.min_frequency == 1 - trainer.show_progress = True - assert trainer.show_progress == True - trainer.special_tokens = [] - assert trainer.special_tokens == [] - trainer.limit_alphabet = None - assert trainer.limit_alphabet == None - trainer.initial_alphabet = ["d", "z"] - assert sorted(trainer.initial_alphabet) == ["d", "z"] - trainer.continuing_subword_prefix = None - assert trainer.continuing_subword_prefix == None - trainer.end_of_word_suffix = None - assert trainer.continuing_subword_prefix == None - - def test_can_pickle(self): - assert ( - trainers.BpeTrainer(min_frequency=12).__getstate__() - == b"""{"BpeTrainer":{"min_frequency":12,"vocab_size":30000,"show_progress":true,"progress_format":"Indicatif","special_tokens":[],"limit_alphabet":null,"initial_alphabet":[],"continuing_subword_prefix":null,"end_of_word_suffix":null,"max_token_length":null,"words":{}}}""" - ) - assert isinstance(pickle.loads(pickle.dumps(trainers.BpeTrainer(min_frequency=12))), trainers.BpeTrainer) - - assert isinstance(copy.deepcopy(trainers.BpeTrainer(min_frequency=12)), trainers.BpeTrainer) - # Make sure everything is correct - assert pickle.dumps(pickle.loads(pickle.dumps(trainers.BpeTrainer(min_frequency=12)))) == pickle.dumps( - trainers.BpeTrainer(min_frequency=12) - ) - - -class TestWordPieceTrainer: - def test_can_modify(self): - trainer = trainers.WordPieceTrainer( - vocab_size=12345, - min_frequency=12, - show_progress=False, - special_tokens=["1", "2"], - limit_alphabet=13, - initial_alphabet=["a", "b", "c"], - continuing_subword_prefix="pref", - end_of_word_suffix="suf", - ) - - assert trainer.vocab_size == 12345 - assert trainer.min_frequency == 12 - assert trainer.show_progress == False - assert trainer.special_tokens == [ - AddedToken("1", special=True), - AddedToken("2", special=True), - ] - assert trainer.limit_alphabet == 13 - assert sorted(trainer.initial_alphabet) == ["a", "b", "c"] - assert trainer.continuing_subword_prefix == "pref" - assert trainer.end_of_word_suffix == "suf" - - # Modify these - trainer.vocab_size = 20000 - assert trainer.vocab_size == 20000 - trainer.min_frequency = 1 - assert trainer.min_frequency == 1 - trainer.show_progress = True - assert trainer.show_progress == True - trainer.special_tokens = [] - assert trainer.special_tokens == [] - trainer.limit_alphabet = None - assert trainer.limit_alphabet == None - trainer.initial_alphabet = ["d", "z"] - assert sorted(trainer.initial_alphabet) == ["d", "z"] - trainer.continuing_subword_prefix = None - assert trainer.continuing_subword_prefix == None - trainer.end_of_word_suffix = None - assert trainer.continuing_subword_prefix == None - - def test_can_pickle(self): - assert isinstance(pickle.loads(pickle.dumps(trainers.WordPieceTrainer())), trainers.WordPieceTrainer) - - -class TestWordLevelTrainer: - def test_can_modify(self): - trainer = trainers.WordLevelTrainer( - vocab_size=12345, min_frequency=12, show_progress=False, special_tokens=["1", "2"] - ) - - assert trainer.vocab_size == 12345 - assert trainer.min_frequency == 12 - assert trainer.show_progress == False - assert trainer.special_tokens == [ - AddedToken("1", special=True), - AddedToken("2", special=True), - ] - - # Modify these - trainer.vocab_size = 20000 - assert trainer.vocab_size == 20000 - trainer.min_frequency = 1 - assert trainer.min_frequency == 1 - trainer.show_progress = True - assert trainer.show_progress == True - trainer.special_tokens = [] - assert trainer.special_tokens == [] - - def test_can_pickle(self): - assert isinstance(pickle.loads(pickle.dumps(trainers.WordLevelTrainer())), trainers.WordLevelTrainer) - - -class TestUnigram: - @pytest.mark.network - def test_train(self, train_files): - tokenizer = SentencePieceUnigramTokenizer() - tokenizer.train(train_files["small"], show_progress=False) - - filename = "tests/data/unigram_trained.json" - tokenizer.save(filename) - os.remove(filename) - - @pytest.mark.network - def test_train_parallelism_with_custom_pretokenizer(self, train_files): - class GoodCustomPretok: - def split(self, n, normalized): - # Here we just test that we can return a List[NormalizedString], it - # does not really make sense to return twice the same otherwise - return [normalized, normalized] - - def pre_tokenize(self, pretok): - pretok.split(self.split) - - custom = pre_tokenizers.PreTokenizer.custom(GoodCustomPretok()) - bpe_tokenizer = Tokenizer(models.BPE()) - bpe_tokenizer.normalizer = normalizers.Lowercase() - bpe_tokenizer.pre_tokenizer = custom - - if "TOKENIZERS_PARALLELISM" in os.environ: - del os.environ["TOKENIZERS_PARALLELISM"] - - trainer = trainers.BpeTrainer(special_tokens=[""], show_progress=False) - bpe_tokenizer.train([train_files["small"]], trainer=trainer) - - def test_can_pickle(self): - assert isinstance(pickle.loads(pickle.dumps(trainers.UnigramTrainer())), trainers.UnigramTrainer) - - def test_train_with_special_tokens(self): - filename = "tests/data/dummy-unigram-special_tokens-train.txt" - os.makedirs("tests/data", exist_ok=True) - with open(filename, "w") as f: - f.write( - """ -[CLS] The Zen of Python, by Tim Peters [SEP] -[CLS] Beautiful is better than ugly. [SEP] -[CLS] Explicit is better than implicit. [SEP] -[CLS] Simple is better than complex. [SEP] -[CLS] Complex is better than complicated. [SEP] -[CLS] Flat is better than nested. [SEP] -[CLS] Sparse is better than dense. [SEP] -[CLS] Readability counts. [SEP] -[CLS] Special cases aren't special enough to break the rules. [SEP] -[CLS] Although practicality beats purity. [SEP] -[CLS] Errors should never pass silently. [SEP] -[CLS] Unless explicitly silenced. [SEP] -[CLS] In the face of ambiguity, refuse the temptation to guess. [SEP] -[CLS] There should be one-- and preferably only one --obvious way to do it. [SEP] -[CLS] Although that way may not be obvious at first unless you're Dutch. [SEP] -[CLS] Now is better than never. [SEP] -[CLS] Although never is often better than *right* now. [SEP] -[CLS] If the implementation is hard to explain, it's a bad idea. [SEP] -[CLS] If the implementation is easy to explain, it may be a good idea. [SEP] -[CLS] Namespaces are one honking great idea -- let's do more of those! [SEP] - """ - ) - - tokenizer = Tokenizer(models.Unigram()) - trainer = trainers.UnigramTrainer( - show_progress=False, special_tokens=["[PAD]", "[SEP]", "[CLS]"], unk_token="[UNK]" - ) - - tokenizer.train([filename], trainer=trainer) - - assert tokenizer.encode("[CLS] This is a test [SEP]").tokens == [ - "[CLS]", - " T", - "h", - "i", - "s", - " is ", - "a", - " ", - "te", - "s", - "t ", - "[SEP]", - ] - - tokenizer = Tokenizer(models.Unigram()) - trainer = trainers.UnigramTrainer( - show_progress=False, - special_tokens=["[PAD]", "[SEP]", "[CLS]"], - unk_token="[UNK]", - vocab_size=100, - ) - tokenizer.train([filename], trainer=trainer) - - assert tokenizer.get_vocab_size() == 100 - - tokenizer = Tokenizer(models.Unigram()) - trainer = trainers.UnigramTrainer( - show_progress=False, - special_tokens=["[PAD]", "[SEP]", "[CLS]", "[UNK]"], - unk_token="[UNK]", - vocab_size=100, - ) - tokenizer.train([filename], trainer=trainer) - - assert tokenizer.get_vocab_size() == 100 - - def test_cannot_train_different_model(self): - tokenizer = Tokenizer(models.BPE()) - trainer = trainers.UnigramTrainer(show_progress=False) - - with pytest.raises(Exception, match="UnigramTrainer can only train a Unigram"): - tokenizer.train([], trainer) - - def test_can_modify(self): - trainer = trainers.UnigramTrainer( - vocab_size=12345, - show_progress=False, - special_tokens=["1", AddedToken("2", lstrip=True)], - initial_alphabet=["a", "b", "c"], - ) - - assert trainer.vocab_size == 12345 - assert trainer.show_progress == False - assert trainer.special_tokens == [ - AddedToken("1", normalized=False, special=True), - AddedToken("2", lstrip=True, normalized=False, special=True), - ] - assert sorted(trainer.initial_alphabet) == ["a", "b", "c"] - - # Modify these - trainer.vocab_size = 20000 - assert trainer.vocab_size == 20000 - trainer.show_progress = True - assert trainer.show_progress == True - trainer.special_tokens = [] - assert trainer.special_tokens == [] - trainer.initial_alphabet = ["d", "z"] - assert sorted(trainer.initial_alphabet) == ["d", "z"] - - @pytest.mark.network - def test_continuing_prefix_trainer_mismatch(self, train_files): - UNK = "[UNK]" - special_tokens = [UNK] - tokenizer = Tokenizer(models.BPE(unk_token=UNK, continuing_subword_prefix="##")) - trainer = trainers.BpeTrainer(special_tokens=special_tokens) - tokenizer.pre_tokenizer = pre_tokenizers.Sequence( - [pre_tokenizers.Whitespace(), pre_tokenizers.Digits(individual_digits=True)] - ) - tokenizer.train(files=[train_files["big"]], trainer=trainer) - - tokenizer_json = os.path.join(DATA_PATH, "tokenizer.json") - tokenizer.save(tokenizer_json) - - tokenizer.from_file(tokenizer_json) - - -@pytest.mark.skipif( - getattr(trainers, "ParityBpeTrainer", None) is None, - reason="built without the `parity-aware-bpe` feature", -) -class TestParityBpeTrainer: - # Two small synthetic "languages" that share most of the alphabet but have - # some language-specific character sequences, enough for a few merges to be - # computable at small num_merges. - LANG_EN = [ - "hello world", - "the quick brown fox jumps over the lazy dog", - "hello there my friend how are you", - "the cat sat on the mat and the dog barked", - "hello world hello world", - ] - LANG_ES = [ - "hola mundo", - "el rápido zorro marrón salta sobre el perro perezoso", - "hola amigo cómo estás", - "el gato se sentó sobre el tapete y el perro ladró", - "hola mundo hola mundo", - ] - - def _make_tokenizer(self): - tok = Tokenizer(models.BPE()) - tok.pre_tokenizer = pre_tokenizers.Whitespace() - return tok - - def test_instantiate_defaults(self): - trainer = trainers.ParityBpeTrainer() - assert trainer.num_merges == 32000 - assert trainer.variant == "base" - assert trainer.min_frequency == 0 - assert trainer.global_merges == 0 - assert trainer.window_size == 100 - assert trainer.alpha == 2.0 - assert trainer.total_symbols == False - - def test_instantiate_variants(self): - base = trainers.ParityBpeTrainer(variant="base") - assert base.variant == "base" - window = trainers.ParityBpeTrainer(variant="window", window_size=50, alpha=1.5) - assert window.variant == "window" - assert window.window_size == 50 - assert window.alpha == 1.5 - - def test_train_from_iterator(self): - tokenizer = self._make_tokenizer() - trainer = trainers.ParityBpeTrainer(num_merges=50, variant="base", show_progress=False) - trainer.train_from_iterator( - tokenizer, - train_iterators=[self.LANG_EN, self.LANG_ES], - ) - assert tokenizer.get_vocab_size() > 0 - encoded = tokenizer.encode("hello world hola mundo") - assert len(encoded.tokens) > 0 - - def test_train_from_iterator_with_dev(self): - tokenizer = self._make_tokenizer() - trainer = trainers.ParityBpeTrainer(num_merges=50, variant="base", show_progress=False) - # Use the same sequences as dev data just to exercise the path; real - # training would use held-out data. - trainer.train_from_iterator( - tokenizer, - train_iterators=[self.LANG_EN, self.LANG_ES], - dev_iterators=[self.LANG_EN, self.LANG_ES], - ) - assert tokenizer.get_vocab_size() > 0 - - def test_train_from_iterator_with_ratio(self): - tokenizer = self._make_tokenizer() - trainer = trainers.ParityBpeTrainer(num_merges=50, variant="base", show_progress=False) - trainer.train_from_iterator( - tokenizer, - train_iterators=[self.LANG_EN, self.LANG_ES], - ratio=[1.0, 1.0], - ) - assert tokenizer.get_vocab_size() > 0 - - def test_can_pickle(self): - original = trainers.ParityBpeTrainer(num_merges=123, variant="window", window_size=42) - restored = pickle.loads(pickle.dumps(original)) - assert isinstance(restored, trainers.ParityBpeTrainer) - assert restored.num_merges == 123 - assert restored.variant == "window" - assert restored.window_size == 42 - - def test_train_iterators_dev_iterators_length_mismatch(self): - tokenizer = self._make_tokenizer() - trainer = trainers.ParityBpeTrainer(num_merges=10, show_progress=False) - with pytest.raises(ValueError): - trainer.train_from_iterator( - tokenizer, - train_iterators=[self.LANG_EN, self.LANG_ES], - dev_iterators=[self.LANG_EN], # length mismatch - ) diff --git a/bindings/python/tests/documentation/__init__.py b/bindings/python/tests/documentation/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/bindings/python/tests/documentation/test_pipeline.py b/bindings/python/tests/documentation/test_pipeline.py deleted file mode 100644 index a94f9cc61..000000000 --- a/bindings/python/tests/documentation/test_pipeline.py +++ /dev/null @@ -1,190 +0,0 @@ -import pytest -from tokenizers import Tokenizer - -from ..utils import data_dir, doc_pipeline_bert_tokenizer, doc_wiki_tokenizer - -disable_printing = True -original_print = print - - -def print(*args, **kwargs): - if not disable_printing: - original_print(*args, **kwargs) - - -class TestPipeline: - @pytest.mark.network - def test_pipeline(self, doc_wiki_tokenizer): - try: - # START reload_tokenizer - from tokenizers import Tokenizer - - tokenizer = Tokenizer.from_file("data/tokenizer-wiki.json") - # END reload_tokenizer - except Exception: - tokenizer = Tokenizer.from_file(doc_wiki_tokenizer) - - # START setup_normalizer - from tokenizers import normalizers - from tokenizers.normalizers import NFD, StripAccents - - normalizer = normalizers.Sequence([NFD(), StripAccents()]) - # END setup_normalizer - # START test_normalizer - normalizer.normalize_str("Héllò hôw are ü?") - # "Hello how are u?" - # END test_normalizer - assert normalizer.normalize_str("Héllò hôw are ü?") == "Hello how are u?" - # START replace_normalizer - tokenizer.normalizer = normalizer - # END replace_normalizer - # START setup_pre_tokenizer - from tokenizers.pre_tokenizers import Whitespace - - pre_tokenizer = Whitespace() - pre_tokenizer.pre_tokenize_str("Hello! How are you? I'm fine, thank you.") - # [("Hello", (0, 5)), ("!", (5, 6)), ("How", (7, 10)), ("are", (11, 14)), ("you", (15, 18)), - # ("?", (18, 19)), ("I", (20, 21)), ("'", (21, 22)), ('m', (22, 23)), ("fine", (24, 28)), - # (",", (28, 29)), ("thank", (30, 35)), ("you", (36, 39)), (".", (39, 40))] - # END setup_pre_tokenizer - assert pre_tokenizer.pre_tokenize_str("Hello! How are you? I'm fine, thank you.") == [ - ("Hello", (0, 5)), - ("!", (5, 6)), - ("How", (7, 10)), - ("are", (11, 14)), - ("you", (15, 18)), - ("?", (18, 19)), - ("I", (20, 21)), - ("'", (21, 22)), - ("m", (22, 23)), - ("fine", (24, 28)), - (",", (28, 29)), - ("thank", (30, 35)), - ("you", (36, 39)), - (".", (39, 40)), - ] - # START combine_pre_tokenizer - from tokenizers import pre_tokenizers - from tokenizers.pre_tokenizers import Digits - - pre_tokenizer = pre_tokenizers.Sequence([Whitespace(), Digits(individual_digits=True)]) - pre_tokenizer.pre_tokenize_str("Call 911!") - # [("Call", (0, 4)), ("9", (5, 6)), ("1", (6, 7)), ("1", (7, 8)), ("!", (8, 9))] - # END combine_pre_tokenizer - assert pre_tokenizer.pre_tokenize_str("Call 911!") == [ - ("Call", (0, 4)), - ("9", (5, 6)), - ("1", (6, 7)), - ("1", (7, 8)), - ("!", (8, 9)), - ] - # START replace_pre_tokenizer - tokenizer.pre_tokenizer = pre_tokenizer - # END replace_pre_tokenizer - # START setup_processor - from tokenizers.processors import TemplateProcessing - - tokenizer.post_processor = TemplateProcessing( - single="[CLS] $A [SEP]", - pair="[CLS] $A [SEP] $B:1 [SEP]:1", - special_tokens=[("[CLS]", 1), ("[SEP]", 2)], - ) - # END setup_processor - # START test_decoding - output = tokenizer.encode("Hello, y'all! How are you 😁 ?") - print(output.ids) - # [1, 27253, 16, 93, 11, 5097, 5, 7961, 5112, 6218, 0, 35, 2] - - tokenizer.decode([1, 27253, 16, 93, 11, 5097, 5, 7961, 5112, 6218, 0, 35, 2]) - # "Hello , y ' all ! How are you ?" - # END test_decoding - assert output.ids == [1, 27253, 16, 93, 11, 5097, 5, 7961, 5112, 6218, 0, 35, 2] - assert ( - tokenizer.decode([1, 27253, 16, 93, 11, 5097, 5, 7961, 5112, 6218, 0, 35, 2]) - == "Hello , y ' all ! How are you ?" - ) - - @staticmethod - def slow_train(): - # START bert_setup_tokenizer - from tokenizers import Tokenizer - from tokenizers.models import WordPiece - - bert_tokenizer = Tokenizer(WordPiece(unk_token="[UNK]")) - # END bert_setup_tokenizer - # START bert_setup_normalizer - from tokenizers import normalizers - from tokenizers.normalizers import NFD, Lowercase, StripAccents - - bert_tokenizer.normalizer = normalizers.Sequence([NFD(), Lowercase(), StripAccents()]) - # END bert_setup_normalizer - # START bert_setup_pre_tokenizer - from tokenizers.pre_tokenizers import Whitespace - - bert_tokenizer.pre_tokenizer = Whitespace() - # END bert_setup_pre_tokenizer - # START bert_setup_processor - from tokenizers.processors import TemplateProcessing - - bert_tokenizer.post_processor = TemplateProcessing( - single="[CLS] $A [SEP]", - pair="[CLS] $A [SEP] $B:1 [SEP]:1", - special_tokens=[ - ("[CLS]", 1), - ("[SEP]", 2), - ], - ) - # END bert_setup_processor - # START bert_train_tokenizer - from tokenizers.trainers import WordPieceTrainer - - trainer = WordPieceTrainer(vocab_size=30522, special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"]) - files = [f"data/wikitext-103-raw/wiki.{split}.raw" for split in ["test", "train", "valid"]] - bert_tokenizer.train(files, trainer) - - bert_tokenizer.save("data/bert-wiki.json") - # END bert_train_tokenizer - - @pytest.mark.network - def test_bert_example(self, doc_pipeline_bert_tokenizer): - try: - bert_tokenizer = Tokenizer.from_file("data/bert-wiki.json") - except Exception: - bert_tokenizer = Tokenizer.from_file(doc_pipeline_bert_tokenizer) - - # START bert_test_decoding - output = bert_tokenizer.encode("Welcome to the 🤗 Tokenizers library.") - print(output.tokens) - # ["[CLS]", "welcome", "to", "the", "[UNK]", "tok", "##eni", "##zer", "##s", "library", ".", "[SEP]"] - - bert_tokenizer.decode(output.ids) - # "welcome to the tok ##eni ##zer ##s library ." - # END bert_test_decoding - assert bert_tokenizer.decode(output.ids) == "welcome to the tok ##eni ##zer ##s library ." - # START bert_proper_decoding - from tokenizers import decoders - - bert_tokenizer.decoder = decoders.WordPiece() - bert_tokenizer.decode(output.ids) - # "welcome to the tokenizers library." - # END bert_proper_decoding - assert bert_tokenizer.decode(output.ids) == "welcome to the tokenizers library." - - -if __name__ == "__main__": - import os - from urllib import request - from zipfile import ZipFile - - disable_printing = False - if not os.path.isdir("data/wikitext-103-raw"): - print("Downloading wikitext-103...") - wiki_text, _ = request.urlretrieve( - "https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-103-raw-v1.zip" - ) - with ZipFile(wiki_text, "r") as z: - print("Unzipping in data...") - z.extractall("data") - - print("Now training...") - TestPipeline.slow_train() diff --git a/bindings/python/tests/documentation/test_quicktour.py b/bindings/python/tests/documentation/test_quicktour.py deleted file mode 100644 index 59d5cceec..000000000 --- a/bindings/python/tests/documentation/test_quicktour.py +++ /dev/null @@ -1,199 +0,0 @@ -import pytest -from tokenizers import Tokenizer -from ..utils import data_dir, doc_wiki_tokenizer - - -disable_printing = True -original_print = print - - -def print(*args, **kwargs): - if not disable_printing: - original_print(*args, **kwargs) - - -class TestQuicktour: - # This method contains everything we don't want to run - @staticmethod - def slow_train(): - tokenizer, trainer = TestQuicktour.get_tokenizer_trainer() - - # START train - files = [f"data/wikitext-103-raw/wiki.{split}.raw" for split in ["test", "train", "valid"]] - tokenizer.train(files, trainer) - # END train - # START save - tokenizer.save("data/tokenizer-wiki.json") - # END save - - @staticmethod - def get_tokenizer_trainer(): - # START init_tokenizer - from tokenizers import Tokenizer - from tokenizers.models import BPE - - tokenizer = Tokenizer(BPE(unk_token="[UNK]")) - # END init_tokenizer - # START init_trainer - from tokenizers.trainers import BpeTrainer - - trainer = BpeTrainer(special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"]) - # END init_trainer - # START init_pretok - from tokenizers.pre_tokenizers import Whitespace - - tokenizer.pre_tokenizer = Whitespace() - # END init_pretok - return tokenizer, trainer - - @pytest.mark.network - def test_quicktour(self, doc_wiki_tokenizer): - def print(*args, **kwargs): - pass - - try: - # START reload_tokenizer - tokenizer = Tokenizer.from_file("data/tokenizer-wiki.json") - # END reload_tokenizer - except Exception: - tokenizer = Tokenizer.from_file(doc_wiki_tokenizer) - # START encode - output = tokenizer.encode("Hello, y'all! How are you 😁 ?") - # END encode - # START print_tokens - print(output.tokens) - # ["Hello", ",", "y", "'", "all", "!", "How", "are", "you", "[UNK]", "?"] - # END print_tokens - assert output.tokens == [ - "Hello", - ",", - "y", - "'", - "all", - "!", - "How", - "are", - "you", - "[UNK]", - "?", - ] - # START print_ids - print(output.ids) - # [27253, 16, 93, 11, 5097, 5, 7961, 5112, 6218, 0, 35] - # END print_ids - assert output.ids == [27253, 16, 93, 11, 5097, 5, 7961, 5112, 6218, 0, 35] - # START print_offsets - print(output.offsets[9]) - # (26, 27) - # END print_offsets - assert output.offsets[9] == (26, 27) - # START use_offsets - sentence = "Hello, y'all! How are you 😁 ?" - sentence[26:27] - # "😁" - # END use_offsets - assert sentence[26:27] == "😁" - # START check_sep - tokenizer.token_to_id("[SEP]") - # 2 - # END check_sep - assert tokenizer.token_to_id("[SEP]") == 2 - # START init_template_processing - from tokenizers.processors import TemplateProcessing - - tokenizer.post_processor = TemplateProcessing( - single="[CLS] $A [SEP]", - pair="[CLS] $A [SEP] $B:1 [SEP]:1", - special_tokens=[ - ("[CLS]", tokenizer.token_to_id("[CLS]")), - ("[SEP]", tokenizer.token_to_id("[SEP]")), - ], - ) - # END init_template_processing - # START print_special_tokens - output = tokenizer.encode("Hello, y'all! How are you 😁 ?") - print(output.tokens) - # ["[CLS]", "Hello", ",", "y", "'", "all", "!", "How", "are", "you", "[UNK]", "?", "[SEP]"] - # END print_special_tokens - assert output.tokens == [ - "[CLS]", - "Hello", - ",", - "y", - "'", - "all", - "!", - "How", - "are", - "you", - "[UNK]", - "?", - "[SEP]", - ] - # START print_special_tokens_pair - output = tokenizer.encode("Hello, y'all!", "How are you 😁 ?") - print(output.tokens) - # ["[CLS]", "Hello", ",", "y", "'", "all", "!", "[SEP]", "How", "are", "you", "[UNK]", "?", "[SEP]"] - # END print_special_tokens_pair - assert output.tokens == [ - "[CLS]", - "Hello", - ",", - "y", - "'", - "all", - "!", - "[SEP]", - "How", - "are", - "you", - "[UNK]", - "?", - "[SEP]", - ] - # START print_type_ids - print(output.type_ids) - # [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1] - # END print_type_ids - assert output.type_ids == [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1] - # START encode_batch - output = tokenizer.encode_batch(["Hello, y'all!", "How are you 😁 ?"]) - # END encode_batch - # START encode_batch_pair - output = tokenizer.encode_batch( - [["Hello, y'all!", "How are you 😁 ?"], ["Hello to you too!", "I'm fine, thank you!"]] - ) - # END encode_batch_pair - # START enable_padding - tokenizer.enable_padding(pad_id=3, pad_token="[PAD]") - # END enable_padding - # START print_batch_tokens - output = tokenizer.encode_batch(["Hello, y'all!", "How are you 😁 ?"]) - print(output[1].tokens) - # ["[CLS]", "How", "are", "you", "[UNK]", "?", "[SEP]", "[PAD]"] - # END print_batch_tokens - assert output[1].tokens == ["[CLS]", "How", "are", "you", "[UNK]", "?", "[SEP]", "[PAD]"] - # START print_attention_mask - print(output[1].attention_mask) - # [1, 1, 1, 1, 1, 1, 1, 0] - # END print_attention_mask - assert output[1].attention_mask == [1, 1, 1, 1, 1, 1, 1, 0] - - -if __name__ == "__main__": - import os - from urllib import request - from zipfile import ZipFile - - disable_printing = False - if not os.path.isdir("data/wikitext-103-raw"): - print("Downloading wikitext-103...") - wiki_text, _ = request.urlretrieve( - "https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-103-raw-v1.zip" - ) - with ZipFile(wiki_text, "r") as z: - print("Unzipping in data...") - z.extractall("data") - - print("Now training...") - TestQuicktour.slow_train() diff --git a/bindings/python/tests/documentation/test_tutorial_train_from_iterators.py b/bindings/python/tests/documentation/test_tutorial_train_from_iterators.py deleted file mode 100644 index d6868b00a..000000000 --- a/bindings/python/tests/documentation/test_tutorial_train_from_iterators.py +++ /dev/null @@ -1,106 +0,0 @@ -# flake8: noqa -import gzip -import os - -import datasets # type: ignore[import-not-found] -import pytest - -from ..utils import data_dir, train_files - - -class TestTrainFromIterators: - @staticmethod - def get_tokenizer_trainer(): - # START init_tokenizer_trainer - from tokenizers import Tokenizer, decoders, models, normalizers, pre_tokenizers, trainers - - tokenizer = Tokenizer(models.Unigram()) - tokenizer.normalizer = normalizers.NFKC() - tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel() - tokenizer.decoder = decoders.ByteLevel() - - trainer = trainers.UnigramTrainer( - vocab_size=20000, - initial_alphabet=pre_tokenizers.ByteLevel.alphabet(), - special_tokens=["", "", ""], - ) - # END init_tokenizer_trainer - trainer.show_progress = False - - return tokenizer, trainer - - @staticmethod - def load_dummy_dataset(): - # START load_dataset - import datasets # type: ignore[import-not-found] - - dataset = datasets.load_dataset("Salesforce/wikitext", "wikitext-103-raw-v1", split="train+test+validation") - # END load_dataset - - @pytest.fixture(scope="class") - def setup_gzip_files(self, train_files): - with open(train_files["small"], "rt") as small: - for n in range(3): - path = f"data/my-file.{n}.gz" - with gzip.open(path, "wt") as f: - f.write(small.read()) - - def test_train_basic(self): - tokenizer, trainer = self.get_tokenizer_trainer() - - # START train_basic - # First few lines of the "Zen of Python" https://www.python.org/dev/peps/pep-0020/ - data = [ - "Beautiful is better than ugly." - "Explicit is better than implicit." - "Simple is better than complex." - "Complex is better than complicated." - "Flat is better than nested." - "Sparse is better than dense." - "Readability counts." - ] - tokenizer.train_from_iterator(data, trainer=trainer) - # END train_basic - - @pytest.mark.network - def test_datasets(self): - tokenizer, trainer = self.get_tokenizer_trainer() - - # In order to keep tests fast, we only use the first 100 examples - os.environ["TOKENIZERS_PARALLELISM"] = "true" - dataset = datasets.load_dataset("Salesforce/wikitext", "wikitext-103-raw-v1", split="train[0:100]") - - # START def_batch_iterator - def batch_iterator(batch_size=1000): - # Only keep the text column to avoid decoding the rest of the columns unnecessarily - tok_dataset = dataset.select_columns("text") - for batch in tok_dataset.iter(batch_size): - yield batch["text"] - - # END def_batch_iterator - - # START train_datasets - tokenizer.train_from_iterator(batch_iterator(), trainer=trainer, length=len(dataset)) - # END train_datasets - - @pytest.mark.network - def test_gzip(self, setup_gzip_files): - tokenizer, trainer = self.get_tokenizer_trainer() - - # START single_gzip - import gzip - - with gzip.open("data/my-file.0.gz", "rt") as f: - tokenizer.train_from_iterator(f, trainer=trainer) - # END single_gzip - # START multi_gzip - files = ["data/my-file.0.gz", "data/my-file.1.gz", "data/my-file.2.gz"] - - def gzip_iterator(): - for path in files: - with gzip.open(path, "rt") as f: - for line in f: - yield line - - tokenizer.train_from_iterator(gzip_iterator(), trainer=trainer) - # END multi_gzip diff --git a/bindings/python/tests/implementations/__init__.py b/bindings/python/tests/implementations/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/bindings/python/tests/implementations/test_base_tokenizer.py b/bindings/python/tests/implementations/test_base_tokenizer.py deleted file mode 100644 index 535964656..000000000 --- a/bindings/python/tests/implementations/test_base_tokenizer.py +++ /dev/null @@ -1,30 +0,0 @@ -from tokenizers import Tokenizer, decoders, models, normalizers, pre_tokenizers, processors -from tokenizers.implementations import BaseTokenizer - - -class TestBaseTokenizer: - def test_get_set_components(self): - toki = Tokenizer(models.BPE()) - toki.normalizer = normalizers.NFC() - toki.pre_tokenizer = pre_tokenizers.ByteLevel() - toki.post_processor = processors.BertProcessing(("A", 0), ("B", 1)) - toki.decoder = decoders.ByteLevel() - - tokenizer = BaseTokenizer(toki) - - assert isinstance(tokenizer.model, models.BPE) - assert isinstance(tokenizer.normalizer, normalizers.NFC) - assert isinstance(tokenizer.pre_tokenizer, pre_tokenizers.ByteLevel) - assert isinstance(tokenizer.post_processor, processors.BertProcessing) - assert isinstance(tokenizer.decoder, decoders.ByteLevel) - - tokenizer.model = models.Unigram() - assert isinstance(tokenizer.model, models.Unigram) - tokenizer.normalizer = normalizers.NFD() - assert isinstance(tokenizer.normalizer, normalizers.NFD) - tokenizer.pre_tokenizer = pre_tokenizers.Whitespace() - assert isinstance(tokenizer.pre_tokenizer, pre_tokenizers.Whitespace) - tokenizer.post_processor = processors.ByteLevel() - assert isinstance(tokenizer.post_processor, processors.ByteLevel) - tokenizer.decoder = decoders.WordPiece() - assert isinstance(tokenizer.decoder, decoders.WordPiece) diff --git a/bindings/python/tests/implementations/test_bert_wordpiece.py b/bindings/python/tests/implementations/test_bert_wordpiece.py deleted file mode 100644 index a7fefc1d3..000000000 --- a/bindings/python/tests/implementations/test_bert_wordpiece.py +++ /dev/null @@ -1,56 +0,0 @@ -import pytest -from tokenizers import BertWordPieceTokenizer - -from ..utils import bert_files, data_dir, multiprocessing_with_parallelism - - -class TestBertWordPieceTokenizer: - @pytest.mark.network - def test_basic_encode(self, bert_files): - tokenizer = BertWordPieceTokenizer.from_file(bert_files["vocab"]) - - # Encode with special tokens by default - output = tokenizer.encode("My name is John", "pair") - assert output.ids == [101, 2026, 2171, 2003, 2198, 102, 3940, 102] - assert output.tokens == [ - "[CLS]", - "my", - "name", - "is", - "john", - "[SEP]", - "pair", - "[SEP]", - ] - assert output.offsets == [ - (0, 0), - (0, 2), - (3, 7), - (8, 10), - (11, 15), - (0, 0), - (0, 4), - (0, 0), - ] - assert output.type_ids == [0, 0, 0, 0, 0, 0, 1, 1] - - # Can encode without the special tokens - output = tokenizer.encode("My name is John", "pair", add_special_tokens=False) - assert output.ids == [2026, 2171, 2003, 2198, 3940] - assert output.tokens == ["my", "name", "is", "john", "pair"] - assert output.offsets == [(0, 2), (3, 7), (8, 10), (11, 15), (0, 4)] - assert output.type_ids == [0, 0, 0, 0, 1] - - @pytest.mark.network - def test_multiprocessing_with_parallelism(self, bert_files): - tokenizer = BertWordPieceTokenizer.from_file(bert_files["vocab"]) - multiprocessing_with_parallelism(tokenizer, False) - multiprocessing_with_parallelism(tokenizer, True) - - def test_train_from_iterator(self): - text = ["A first sentence", "Another sentence", "And a last one"] - tokenizer = BertWordPieceTokenizer() - tokenizer.train_from_iterator(text, show_progress=False) - - output = tokenizer.encode("A sentence") - assert output.tokens == ["a", "sentence"] diff --git a/bindings/python/tests/implementations/test_byte_level_bpe.py b/bindings/python/tests/implementations/test_byte_level_bpe.py deleted file mode 100644 index 51410ac8e..000000000 --- a/bindings/python/tests/implementations/test_byte_level_bpe.py +++ /dev/null @@ -1,103 +0,0 @@ -import pytest -from tokenizers import ByteLevelBPETokenizer - -from ..utils import data_dir, multiprocessing_with_parallelism, roberta_files - - -class TestByteLevelBPE: - @pytest.mark.network - def test_basic_encode(self, roberta_files): - tokenizer = ByteLevelBPETokenizer.from_file(roberta_files["vocab"], roberta_files["merges"]) - output = tokenizer.encode("The quick brown fox jumps over the lazy dog") - - assert output.ids == [133, 2119, 6219, 23602, 13855, 81, 5, 22414, 2335] - assert output.tokens == [ - "The", - "Ġquick", - "Ġbrown", - "Ġfox", - "Ġjumps", - "Ġover", - "Ġthe", - "Ġlazy", - "Ġdog", - ] - assert output.offsets == [ - (0, 3), - (3, 9), - (9, 15), - (15, 19), - (19, 25), - (25, 30), - (30, 34), - (34, 39), - (39, 43), - ] - - @pytest.mark.network - def test_add_prefix_space(self, roberta_files): - tokenizer = ByteLevelBPETokenizer.from_file( - roberta_files["vocab"], roberta_files["merges"], add_prefix_space=True - ) - output = tokenizer.encode("The quick brown fox jumps over the lazy dog") - - assert output.ids == [20, 2119, 6219, 23602, 13855, 81, 5, 22414, 2335] - assert output.tokens == [ - "ĠThe", - "Ġquick", - "Ġbrown", - "Ġfox", - "Ġjumps", - "Ġover", - "Ġthe", - "Ġlazy", - "Ġdog", - ] - assert output.offsets == [ - (0, 3), - (3, 9), - (9, 15), - (15, 19), - (19, 25), - (25, 30), - (30, 34), - (34, 39), - (39, 43), - ] - - @pytest.mark.network - def test_lowerspace(self, roberta_files): - tokenizer = ByteLevelBPETokenizer.from_file( - roberta_files["vocab"], - roberta_files["merges"], - add_prefix_space=True, - lowercase=True, - ) - output = tokenizer.encode("The Quick Brown Fox Jumps Over The Lazy Dog") - - assert output.ids == [5, 2119, 6219, 23602, 13855, 81, 5, 22414, 2335] - assert output.tokens == [ - "Ġthe", - "Ġquick", - "Ġbrown", - "Ġfox", - "Ġjumps", - "Ġover", - "Ġthe", - "Ġlazy", - "Ġdog", - ] - - @pytest.mark.network - def test_multiprocessing_with_parallelism(self, roberta_files): - tokenizer = ByteLevelBPETokenizer.from_file(roberta_files["vocab"], roberta_files["merges"]) - multiprocessing_with_parallelism(tokenizer, False) - multiprocessing_with_parallelism(tokenizer, True) - - def test_train_from_iterator(self): - text = ["A first sentence", "Another sentence", "And a last one"] - tokenizer = ByteLevelBPETokenizer() - tokenizer.train_from_iterator(text, show_progress=False) - - output = tokenizer.encode("A sentence") - assert output.tokens == ["A", "Ġsentence"] diff --git a/bindings/python/tests/implementations/test_char_bpe.py b/bindings/python/tests/implementations/test_char_bpe.py deleted file mode 100644 index 3449e8b2a..000000000 --- a/bindings/python/tests/implementations/test_char_bpe.py +++ /dev/null @@ -1,63 +0,0 @@ -import pytest -from tokenizers import CharBPETokenizer - -from ..utils import data_dir, multiprocessing_with_parallelism, openai_files - - -class TestCharBPETokenizer: - @pytest.mark.network - def test_basic_encode(self, openai_files): - tokenizer = CharBPETokenizer.from_file(openai_files["vocab"], openai_files["merges"]) - - output = tokenizer.encode("My name is John", "pair") - assert output.ids == [0, 253, 1362, 544, 0, 7, 12662, 2688] - assert output.tokens == [ - "", - "y", - "name", - "is", - "", - "o", - "hn", - "pair", - ] - assert output.offsets == [ - (0, 1), - (1, 2), - (3, 7), - (8, 10), - (11, 12), - (12, 13), - (13, 15), - (0, 4), - ] - assert output.type_ids == [0, 0, 0, 0, 0, 0, 0, 1] - - @pytest.mark.network - def test_lowercase(self, openai_files): - tokenizer = CharBPETokenizer.from_file(openai_files["vocab"], openai_files["merges"], lowercase=True) - output = tokenizer.encode("My name is John", "pair", add_special_tokens=False) - assert output.ids == [547, 1362, 544, 2476, 2688] - assert output.tokens == ["my", "name", "is", "john", "pair"] - assert output.offsets == [(0, 2), (3, 7), (8, 10), (11, 15), (0, 4)] - assert output.type_ids == [0, 0, 0, 0, 1] - - @pytest.mark.network - def test_decoding(self, openai_files): - tokenizer = CharBPETokenizer.from_file(openai_files["vocab"], openai_files["merges"], lowercase=True) - decoded = tokenizer.decode(tokenizer.encode("my name is john").ids) - assert decoded == "my name is john" - - @pytest.mark.network - def test_multiprocessing_with_parallelism(self, openai_files): - tokenizer = CharBPETokenizer.from_file(openai_files["vocab"], openai_files["merges"]) - multiprocessing_with_parallelism(tokenizer, False) - multiprocessing_with_parallelism(tokenizer, True) - - def test_train_from_iterator(self): - text = ["A first sentence", "Another sentence", "And a last one"] - tokenizer = CharBPETokenizer() - tokenizer.train_from_iterator(text, show_progress=False) - - output = tokenizer.encode("A sentence") - assert output.tokens == ["A", "sentence"] diff --git a/bindings/python/tests/implementations/test_sentencepiece.py b/bindings/python/tests/implementations/test_sentencepiece.py deleted file mode 100644 index 1da41fec0..000000000 --- a/bindings/python/tests/implementations/test_sentencepiece.py +++ /dev/null @@ -1,61 +0,0 @@ -import pytest - -from tokenizers import SentencePieceBPETokenizer, SentencePieceUnigramTokenizer - - -class TestSentencePieceBPE: - def test_train_from_iterator(self): - text = ["A first sentence", "Another sentence", "And a last one"] - tokenizer = SentencePieceBPETokenizer() - tokenizer.train_from_iterator(text, show_progress=False) - - output = tokenizer.encode("A sentence") - assert output.tokens == ["▁A", "▁sentence"] - - -class TestSentencePieceUnigram: - def test_train(self, tmpdir): - p = tmpdir.mkdir("tmpdir").join("file.txt") - p.write("A first sentence\nAnother sentence\nAnd a last one") - - tokenizer = SentencePieceUnigramTokenizer() - tokenizer.train(files=str(p), show_progress=False) - - output = tokenizer.encode("A sentence") - assert output.tokens == ["▁A", "▁", "s", "en", "t", "en", "c", "e"] - - with pytest.raises(Exception) as excinfo: - _ = tokenizer.encode("A sentence 🤗") - assert str(excinfo.value) == "Encountered an unknown token but `unk_id` is missing" - - def test_train_with_unk_token(self, tmpdir): - p = tmpdir.mkdir("tmpdir").join("file.txt") - p.write("A first sentence\nAnother sentence\nAnd a last one") - - tokenizer = SentencePieceUnigramTokenizer() - tokenizer.train(files=str(p), show_progress=False, special_tokens=[""], unk_token="") - output = tokenizer.encode("A sentence 🤗") - assert output.ids[-1] == 0 - assert output.tokens == ["▁A", "▁", "s", "en", "t", "en", "c", "e", "▁", "🤗"] - - def test_train_from_iterator(self): - text = ["A first sentence", "Another sentence", "And a last one"] - tokenizer = SentencePieceUnigramTokenizer() - tokenizer.train_from_iterator(text, show_progress=False) - - output = tokenizer.encode("A sentence") - assert output.tokens == ["▁A", "▁", "s", "en", "t", "en", "c", "e"] - - with pytest.raises(Exception) as excinfo: - _ = tokenizer.encode("A sentence 🤗") - assert str(excinfo.value) == "Encountered an unknown token but `unk_id` is missing" - - def test_train_from_iterator_with_unk_token(self): - text = ["A first sentence", "Another sentence", "And a last one"] - tokenizer = SentencePieceUnigramTokenizer() - tokenizer.train_from_iterator( - text, vocab_size=100, show_progress=False, special_tokens=[""], unk_token="" - ) - output = tokenizer.encode("A sentence 🤗") - assert output.ids[-1] == 0 - assert output.tokens == ["▁A", "▁", "s", "en", "t", "en", "c", "e", "▁", "🤗"] diff --git a/bindings/python/tests/test_benchmarks.py b/bindings/python/tests/test_benchmarks.py deleted file mode 100644 index e8a3828c2..000000000 --- a/bindings/python/tests/test_benchmarks.py +++ /dev/null @@ -1,204 +0,0 @@ -""" -Benchmark suite for Python tokenizer bindings. - -Measures the overhead of the Python ↔ Rust FFI layer by exercising the same -operations benchmarked on the Rust side. Run with: - - pytest tests/test_benchmarks.py --benchmark-columns=mean,stddev,rounds -v - -Requires: pytest-benchmark, tokenizers (built with maturin develop --release) -""" - -import asyncio -import concurrent.futures -import os -from pathlib import Path - -import pytest - -pytest.importorskip("pytest_benchmark") - -from tokenizers import Tokenizer, AddedToken -from tokenizers.models import BPE -from tokenizers.pre_tokenizers import ByteLevel as ByteLevelPreTokenizer -from tokenizers.trainers import BpeTrainer - -# --------------------------------------------------------------------------- -# Fixtures -# --------------------------------------------------------------------------- - -DATA_DIR = Path( - os.environ.get( - "TOKENIZERS_DATA_DIR", - Path(__file__).resolve().parent.parent.parent.parent / "tokenizers" / "data", - ) -) - - -@pytest.fixture(scope="module") -def big_text(): - return (DATA_DIR / "big.txt").read_text() - - -@pytest.fixture(scope="module") -def lines(big_text): - return big_text.splitlines() - - -@pytest.fixture(scope="module") -def gpt2_tokenizer(): - bpe = BPE.from_file( - str(DATA_DIR / "gpt2-vocab.json"), - str(DATA_DIR / "gpt2-merges.txt"), - ) - tok = Tokenizer(bpe) - tok.pre_tokenizer = ByteLevelPreTokenizer() - return tok - - -@pytest.fixture(scope="module") -def llama3_tokenizer(): - return Tokenizer.from_file(str(DATA_DIR / "llama-3-tokenizer.json")) - - -@pytest.fixture(scope="module") -def roberta_tokenizer(): - return Tokenizer.from_file(str(DATA_DIR / "roberta.json")) - - -@pytest.fixture(scope="module") -def albert_tokenizer(): - return Tokenizer.from_file(str(DATA_DIR / "albert-base-v1-tokenizer.json")) - - -# --------------------------------------------------------------------------- -# Encoding benchmarks — GPT-2 -# --------------------------------------------------------------------------- - - -class TestBPEGPT2: - def test_encode(self, benchmark, gpt2_tokenizer, lines): - def run(): - for line in lines: - gpt2_tokenizer.encode(line) - - benchmark(run) - - def test_encode_batch(self, benchmark, gpt2_tokenizer, lines): - benchmark(gpt2_tokenizer.encode_batch, lines) - - def test_encode_batch_multithreaded(self, benchmark, gpt2_tokenizer, lines): - """encode_batch with multiple OS threads via concurrent.futures.""" - n_workers = 4 - chunk_size = len(lines) // n_workers - - def run(): - with concurrent.futures.ThreadPoolExecutor(max_workers=n_workers) as pool: - futures = [] - for i in range(n_workers): - chunk = lines[i * chunk_size : (i + 1) * chunk_size] - futures.append(pool.submit(gpt2_tokenizer.encode_batch, chunk)) - for f in futures: - f.result() - - benchmark(run) - - -# --------------------------------------------------------------------------- -# Encoding benchmarks — Llama-3 -# --------------------------------------------------------------------------- - - -class TestLlama3: - def test_encode(self, benchmark, llama3_tokenizer, lines): - def run(): - for line in lines: - llama3_tokenizer.encode(line) - - benchmark(run) - - def test_encode_batch(self, benchmark, llama3_tokenizer, lines): - benchmark(llama3_tokenizer.encode_batch, lines) - - def test_encode_fast(self, benchmark, llama3_tokenizer, lines): - """encode without offset tracking.""" - benchmark(llama3_tokenizer.encode_batch_fast, lines) - - def test_encode_batch_multithreaded(self, benchmark, llama3_tokenizer, lines): - n_workers = 4 - chunk_size = len(lines) // n_workers - - def run(): - with concurrent.futures.ThreadPoolExecutor(max_workers=n_workers) as pool: - futures = [] - for i in range(n_workers): - chunk = lines[i * chunk_size : (i + 1) * chunk_size] - futures.append(pool.submit(llama3_tokenizer.encode_batch, chunk)) - for f in futures: - f.result() - - benchmark(run) - - def test_decode_batch(self, benchmark, llama3_tokenizer, lines): - # Pre-encode to get token IDs for decoding - encoded = llama3_tokenizer.encode_batch(lines[:1000]) - ids_list = [enc.ids for enc in encoded] - benchmark(llama3_tokenizer.decode_batch, ids_list) - - -# --------------------------------------------------------------------------- -# Async encoding benchmarks -# --------------------------------------------------------------------------- - - -class TestAsync: - def test_async_encode_batch(self, benchmark, llama3_tokenizer, lines): - async def run(): - return await llama3_tokenizer.async_encode_batch(lines) - - benchmark(lambda: asyncio.run(run())) - - def test_async_encode_batch_fast(self, benchmark, llama3_tokenizer, lines): - async def run(): - return await llama3_tokenizer.async_encode_batch_fast(lines) - - benchmark(lambda: asyncio.run(run())) - - -# --------------------------------------------------------------------------- -# Serialization benchmarks -# --------------------------------------------------------------------------- - - -class TestSerialization: - def test_from_file_roberta(self, benchmark): - benchmark(Tokenizer.from_file, str(DATA_DIR / "roberta.json")) - - def test_from_file_llama3(self, benchmark): - benchmark(Tokenizer.from_file, str(DATA_DIR / "llama-3-tokenizer.json")) - - def test_from_file_albert(self, benchmark): - benchmark(Tokenizer.from_file, str(DATA_DIR / "albert-base-v1-tokenizer.json")) - - def test_to_str_llama3(self, benchmark, llama3_tokenizer): - benchmark(llama3_tokenizer.to_str) - - def test_from_str_llama3(self, benchmark, llama3_tokenizer): - json_str = llama3_tokenizer.to_str() - benchmark(Tokenizer.from_str, json_str) - - -# --------------------------------------------------------------------------- -# Training benchmark -# --------------------------------------------------------------------------- - - -class TestTraining: - def test_train_bpe_small(self, benchmark): - def run(): - tok = Tokenizer(BPE()) - tok.pre_tokenizer = ByteLevelPreTokenizer() - trainer = BpeTrainer(vocab_size=1000, show_progress=False) - tok.train([str(DATA_DIR / "small.txt")], trainer) - - benchmark(run) diff --git a/bindings/python/tests/test_freethreaded.py b/bindings/python/tests/test_freethreaded.py deleted file mode 100644 index 017fafd6c..000000000 --- a/bindings/python/tests/test_freethreaded.py +++ /dev/null @@ -1,163 +0,0 @@ -"""Concurrency stress tests for free-threaded Python (3.14t). - -Pairs with docs/free-threading-audit.md. Designed to catch: - - data races on RwLock-guarded component state, - - RwLock poisoning from a panicking setter, - - segfaults / SystemErrors from mismatched lifetimes. - -These tests run on regular CPython too — under the GIL they're a -no-op for race detection, but they verify the non-racey behavior is -unchanged. -""" - -import sys -import threading -from concurrent.futures import ThreadPoolExecutor - -import pytest - -from tokenizers import Tokenizer -from tokenizers.models import BPE -from tokenizers.normalizers import Lowercase, NFD, Sequence as NormalizerSequence -from tokenizers.pre_tokenizers import Whitespace -from tokenizers.processors import ByteLevel -from tokenizers.trainers import BpeTrainer - - -def _is_free_threaded() -> bool: - is_gil_enabled = getattr(sys, "_is_gil_enabled", None) - return callable(is_gil_enabled) and not is_gil_enabled() - - -pytestmark = pytest.mark.timeout(60) # any of these hanging means a deadlock - - -def _make_tokenizer() -> Tokenizer: - tok = Tokenizer(BPE()) - tok.pre_tokenizer = Whitespace() - tok.normalizer = NormalizerSequence([NFD(), Lowercase()]) - return tok - - -class TestEncodeUnderConcurrentSetters: - """N encoders + M setters racing on the same Tokenizer. - - Audit references: §1 (single-field setter), §2 (top-level swap), - §6 (encode read guard). - """ - - def test_encode_while_swapping_post_processor(self): - tok = _make_tokenizer() - stop = threading.Event() - encoded_count = 0 - encode_lock = threading.Lock() - - def encoder(): - nonlocal encoded_count - local_count = 0 - while not stop.is_set(): - enc = tok.encode("the quick brown fox jumps over the lazy dog") - # Tokens list must always be well-formed (no torn read). - assert isinstance(enc.tokens, list) - assert all(isinstance(t, str) for t in enc.tokens) - local_count += 1 - with encode_lock: - encoded_count += local_count - - def setter(): - while not stop.is_set(): - tok.post_processor = ByteLevel(trim_offsets=True) - - with ThreadPoolExecutor(max_workers=8) as ex: - futures = [ex.submit(encoder) for _ in range(6)] - futures += [ex.submit(setter) for _ in range(2)] - threading.Event().wait(2.0) # let them race for 2 seconds - stop.set() - for f in futures: - f.result() # surface any exception - - assert encoded_count > 0, "encoders should make progress" - - def test_encode_while_mutating_trainer_fields(self): - """Audit §5: trainer field mutation should not race with encode. - - train() is not called here — we just verify that mutating trainer - fields doesn't poison locks observed by other operations. - """ - tok = _make_tokenizer() - trainer = BpeTrainer(vocab_size=1000) - stop = threading.Event() - - def encoder(): - while not stop.is_set(): - tok.encode("hello world") - - def trainer_mutator(): - n = 0 - while not stop.is_set(): - trainer.vocab_size = 1000 + (n % 4096) - trainer.min_frequency = n % 5 - n += 1 - - with ThreadPoolExecutor(max_workers=6) as ex: - futures = [ex.submit(encoder) for _ in range(4)] - futures += [ex.submit(trainer_mutator) for _ in range(2)] - threading.Event().wait(1.5) - stop.set() - for f in futures: - f.result() - - # Final state must still be readable — a poisoned RwLock would - # raise here. - _ = trainer.vocab_size - - def test_concurrent_setters_no_lock_poisoning(self): - """Audit §1: concurrent setters serialize through RwLock. - - If any setter panics inside the guarded scope, the lock is poisoned - and the next reader raises. This test asserts neither happens - under heavy contention. - """ - tok = _make_tokenizer() - stop = threading.Event() - - def setter_a(): - while not stop.is_set(): - tok.pre_tokenizer = Whitespace() - - def setter_b(): - while not stop.is_set(): - tok.normalizer = NFD() - - def reader(): - while not stop.is_set(): - _ = tok.pre_tokenizer - _ = tok.normalizer - - with ThreadPoolExecutor(max_workers=8) as ex: - futures = [ex.submit(setter_a) for _ in range(2)] - futures += [ex.submit(setter_b) for _ in range(2)] - futures += [ex.submit(reader) for _ in range(4)] - threading.Event().wait(1.5) - stop.set() - for f in futures: - f.result() - - # Final assignments succeed → locks are healthy. - tok.pre_tokenizer = Whitespace() - tok.normalizer = NFD() - - -@pytest.mark.skipif(not _is_free_threaded(), reason="3.14t-only check") -class TestFreeThreadedSpecific: - """Asserts the 3.14t-specific properties: GIL truly off + module - declares Py_MOD_GIL_NOT_USED.""" - - def test_gil_actually_disabled_on_import(self): - """If the wheel were misconfigured with gil_used=true, importing - tokenizers would silently re-enable the GIL on 3.14t.""" - import tokenizers # noqa: F401 (re-import is a no-op) - - assert sys._is_gil_enabled() is False, ( - "tokenizers re-enabled the GIL on free-threaded Python — wheel was built without gil_used=false" - ) diff --git a/bindings/python/tests/test_serialization.py b/bindings/python/tests/test_serialization.py deleted file mode 100644 index c9060f4ad..000000000 --- a/bindings/python/tests/test_serialization.py +++ /dev/null @@ -1,153 +0,0 @@ -import json -import os -import pytest -import unittest - -import tqdm -from huggingface_hub import hf_hub_download -from tokenizers import Tokenizer -from tokenizers.models import BPE, Unigram - -from .utils import albert_base, data_dir - - -class TestSerialization: - @pytest.mark.network - def test_full_serialization_albert(self, albert_base): - # Check we can read this file. - # This used to fail because of BufReader that would fail because the - # file exceeds the buffer capacity - Tokenizer.from_file(albert_base) - - @pytest.mark.network - def test_str_big(self, albert_base): - tokenizer = Tokenizer.from_file(albert_base) - assert ( - str(tokenizer) - == """Tokenizer(version="1.0", truncation=None, padding=None, added_tokens=[{"id":0, "content":"", "single_word":False, "lstrip":False, "rstrip":False, ...}, {"id":1, "content":"", "single_word":False, "lstrip":False, "rstrip":False, ...}, {"id":2, "content":"[CLS]", "single_word":False, "lstrip":False, "rstrip":False, ...}, {"id":3, "content":"[SEP]", "single_word":False, "lstrip":False, "rstrip":False, ...}, {"id":4, "content":"[MASK]", "single_word":False, "lstrip":False, "rstrip":False, ...}], normalizer=Sequence(normalizers=[Replace(pattern=String("``"), content="\""), Replace(pattern=String("''"), content="\""), NFKD(), StripAccents(), Lowercase(), ...]), pre_tokenizer=Sequence(pretokenizers=[WhitespaceSplit(), Metaspace(replacement="▁", prepend_scheme=always, split=True)]), post_processor=TemplateProcessing(single=[SpecialToken(id="[CLS]", type_id=0), Sequence(id=A, type_id=0), SpecialToken(id="[SEP]", type_id=0)], pair=[SpecialToken(id="[CLS]", type_id=0), Sequence(id=A, type_id=0), SpecialToken(id="[SEP]", type_id=0), Sequence(id=B, type_id=1), SpecialToken(id="[SEP]", type_id=1)], special_tokens={"[CLS]":SpecialToken(id="[CLS]", ids=[2], tokens=["[CLS]"]), "[SEP]":SpecialToken(id="[SEP]", ids=[3], tokens=["[SEP]"])}), decoder=Metaspace(replacement="▁", prepend_scheme=always, split=True), model=Unigram(unk_id=1, vocab=[("", 0), ("", 0), ("[CLS]", 0), ("[SEP]", 0), ("[MASK]", 0), ...], byte_fallback=False))""" - ) - - def test_repr_str(self): - tokenizer = Tokenizer(BPE()) - tokenizer.add_tokens(["my"]) - assert ( - repr(tokenizer) - == """Tokenizer(version="1.0", truncation=None, padding=None, added_tokens=[{"id":0, "content":"my", "single_word":False, "lstrip":False, "rstrip":False, "normalized":True, "special":False}], normalizer=None, pre_tokenizer=None, post_processor=None, decoder=None, model=BPE(dropout=None, unk_token=None, continuing_subword_prefix=None, end_of_word_suffix=None, fuse_unk=False, byte_fallback=False, ignore_merges=False, vocab={}, merges=[]))""" - ) - assert ( - str(tokenizer) - == """Tokenizer(version="1.0", truncation=None, padding=None, added_tokens=[{"id":0, "content":"my", "single_word":False, "lstrip":False, "rstrip":False, ...}], normalizer=None, pre_tokenizer=None, post_processor=None, decoder=None, model=BPE(dropout=None, unk_token=None, continuing_subword_prefix=None, end_of_word_suffix=None, fuse_unk=False, byte_fallback=False, ignore_merges=False, vocab={}, merges=[]))""" - ) - - def test_repr_str_ellipsis(self): - model = BPE() - assert ( - repr(model) - == """BPE(dropout=None, unk_token=None, continuing_subword_prefix=None, end_of_word_suffix=None, fuse_unk=False, byte_fallback=False, ignore_merges=False, vocab={}, merges=[])""" - ) - assert ( - str(model) - == """BPE(dropout=None, unk_token=None, continuing_subword_prefix=None, end_of_word_suffix=None, fuse_unk=False, byte_fallback=False, ignore_merges=False, vocab={}, merges=[])""" - ) - - vocab = [ - ("A", 0.0), - ("B", -0.01), - ("C", -0.02), - ("D", -0.03), - ("E", -0.04), - ] - # No ellispsis yet - model = Unigram(vocab, 0, byte_fallback=False) - assert ( - repr(model) - == """Unigram(unk_id=0, vocab=[("A", 0), ("B", -0.01), ("C", -0.02), ("D", -0.03), ("E", -0.04)], byte_fallback=False)""" - ) - assert ( - str(model) - == """Unigram(unk_id=0, vocab=[("A", 0), ("B", -0.01), ("C", -0.02), ("D", -0.03), ("E", -0.04)], byte_fallback=False)""" - ) - - # Ellispis for longer than 5 elements only on `str`. - vocab = [ - ("A", 0.0), - ("B", -0.01), - ("C", -0.02), - ("D", -0.03), - ("E", -0.04), - ("F", -0.04), - ] - model = Unigram(vocab, 0, byte_fallback=False) - assert ( - repr(model) - == """Unigram(unk_id=0, vocab=[("A", 0), ("B", -0.01), ("C", -0.02), ("D", -0.03), ("E", -0.04), ("F", -0.04)], byte_fallback=False)""" - ) - assert ( - str(model) - == """Unigram(unk_id=0, vocab=[("A", 0), ("B", -0.01), ("C", -0.02), ("D", -0.03), ("E", -0.04), ...], byte_fallback=False)""" - ) - - -def check(tokenizer_file) -> bool: - with open(tokenizer_file, "r") as f: - data = json.load(f) - if "pre_tokenizer" not in data: - return True - if "type" not in data["pre_tokenizer"]: - return False - if data["pre_tokenizer"]["type"] == "Sequence": - for pre_tok in data["pre_tokenizer"]["pretokenizers"]: - if "type" not in pre_tok: - return False - return True - - -def slow(test_case): - """ - Decorator marking a test as slow. - - Slow tests are skipped by default. Set the RUN_SLOW environment variable to a truthy value to run them. - - """ - if os.getenv("RUN_SLOW") != "1": - return unittest.skip("use `RUN_SLOW=1` to run")(test_case) - else: - return test_case - - -@slow -class TestFullDeserialization(unittest.TestCase): - def test_full_deserialization_hub(self): - # Check we can read this file. - # This used to fail because of BufReader that would fail because the - # file exceeds the buffer capacity - not_loadable = [] - invalid_pre_tokenizer = [] - - # models = api.list_models(filter="transformers") - # for model in tqdm.tqdm(models): - # model_id = model.modelId - # for model_file in model.siblings: - # filename = model_file.rfilename - # if filename == "tokenizer.json": - # all_models.append((model_id, filename)) - - all_models = [("HueyNemud/das22-10-camembert_pretrained", "tokenizer.json")] - for model_id, filename in tqdm.tqdm(all_models): - tokenizer_file = hf_hub_download(model_id, filename=filename) - - is_ok = check(tokenizer_file) - if not is_ok: - print(f"{model_id} is affected by no type") - invalid_pre_tokenizer.append(model_id) - try: - Tokenizer.from_file(tokenizer_file) - except Exception as e: - print(f"{model_id} is not loadable: {e}") - not_loadable.append(model_id) - except: # noqa: E722 - print(f"{model_id} is not loadable: Rust error") - not_loadable.append(model_id) - - self.assertEqual(invalid_pre_tokenizer, []) - self.assertEqual(not_loadable, []) diff --git a/bindings/python/tests/utils.py b/bindings/python/tests/utils.py deleted file mode 100644 index 0932a3aae..000000000 --- a/bindings/python/tests/utils.py +++ /dev/null @@ -1,105 +0,0 @@ -import multiprocessing as mp -import os - -import pytest - -from huggingface_hub import hf_hub_download - - -DATA_PATH = os.path.join("tests", "data") -HF_TEST_REPO = "hf-internal-testing/tokenizers-test-data" - - -def download(filename): - # huggingface_hub handles auth (HF_TOKEN), retry/backoff and ETag-based caching. - return hf_hub_download(repo_id=HF_TEST_REPO, filename=filename, repo_type="dataset") - - -@pytest.fixture(scope="session") -def data_dir(): - assert os.getcwd().endswith("python") - exist = os.path.exists(DATA_PATH) and os.path.isdir(DATA_PATH) - if not exist: - os.mkdir(DATA_PATH) - - -@pytest.fixture(scope="session") -def roberta_files(data_dir): - return { - "vocab": download("roberta-base-vocab.json"), - "merges": download("roberta-base-merges.txt"), - } - - -@pytest.fixture(scope="session") -def bert_files(data_dir): - return { - "vocab": download("bert-base-uncased-vocab.txt"), - } - - -@pytest.fixture(scope="session") -def openai_files(data_dir): - return { - "vocab": download("openai-gpt-vocab.json"), - "merges": download("openai-gpt-merges.txt"), - } - - -@pytest.fixture(scope="session") -def train_files(data_dir): - big = download("big.txt") - small = download("small.txt") - return { - "small": small, - "big": big, - } - - -@pytest.fixture(scope="session") -def albert_base(data_dir): - return download("albert-base-v1-tokenizer.json") - - -@pytest.fixture(scope="session") -def doc_wiki_tokenizer(data_dir): - return download("tokenizer-wiki.json") - - -@pytest.fixture(scope="session") -def doc_pipeline_bert_tokenizer(data_dir): - return download("bert-wiki.json") - - -# On MacOS Python 3.8+ the default was modified to `spawn`, we need `fork` in tests. -mp.set_start_method("fork") - - -def multiprocessing_with_parallelism(tokenizer, enabled: bool): - """ - This helper can be used to test that disabling parallelism avoids dead locks when the - same tokenizer is used after forking. - """ - # It's essential to this test that we call 'encode' or 'encode_batch' - # before the fork. This causes the main process to "lock" some resources - # provided by the Rust "rayon" crate that are needed for parallel processing. - tokenizer.encode("Hi") - tokenizer.encode_batch(["hi", "there"]) - - def encode(tokenizer): - tokenizer.encode("Hi") - tokenizer.encode_batch(["hi", "there"]) - - # Make sure this environment variable is set before the fork happens - os.environ["TOKENIZERS_PARALLELISM"] = str(enabled) - p = mp.Process(target=encode, args=(tokenizer,)) - p.start() - p.join(timeout=1) - - # At this point the process should have successfully exited, depending on whether parallelism - # was activated or not. So we check the status and kill it if needed - alive = p.is_alive() - if alive: - p.terminate() - - assert (alive and mp.get_start_method() == "fork") == enabled diff --git a/bindings/python/tools/stub-gen/Cargo.lock b/bindings/python/tools/stub-gen/Cargo.lock index 01116d4cf..c00f814df 100644 --- a/bindings/python/tools/stub-gen/Cargo.lock +++ b/bindings/python/tools/stub-gen/Cargo.lock @@ -2,182 +2,40 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "aho-corasick" -version = "1.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" -dependencies = [ - "memchr", -] - -[[package]] -name = "anstream" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "is_terminal_polyfill", - "utf8parse", -] - -[[package]] -name = "anstyle" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" - -[[package]] -name = "anstyle-parse" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-query" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" -dependencies = [ - "windows-sys", -] - -[[package]] -name = "anstyle-wincon" -version = "3.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" -dependencies = [ - "anstyle", - "once_cell_polyfill", - "windows-sys", -] - [[package]] name = "anyhow" -version = "1.0.102" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" - -[[package]] -name = "colorchoice" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" - -[[package]] -name = "env_filter" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32e90c2accc4b07a8456ea0debdc2e7587bdd890680d71173a15d4ae604f6eef" -dependencies = [ - "log", - "regex", -] - -[[package]] -name = "env_logger" -version = "0.11.10" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0621c04f2196ac3f488dd583365b9c09be011a4ab8b9f37248ffcc8f6198b56a" -dependencies = [ - "anstream", - "anstyle", - "env_filter", - "jiff", - "log", -] +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "goblin" -version = "0.10.5" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "983a6aafb3b12d4c41ea78d39e189af4298ce747353945ff5105b54a056e5cd9" +checksum = "17582616a7718cca54cec18e534a76c7c4aec11a8b9a85695712f262fd15a4c8" dependencies = [ "log", "plain", "scroll", ] -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - -[[package]] -name = "is_terminal_polyfill" -version = "1.70.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" - [[package]] name = "itoa" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" -[[package]] -name = "jiff" -version = "0.2.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" -dependencies = [ - "jiff-static", - "log", - "portable-atomic", - "portable-atomic-util", - "serde_core", -] - -[[package]] -name = "jiff-static" -version = "0.2.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "libc" -version = "0.2.183" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" - [[package]] name = "log" -version = "0.4.29" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "memchr" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" - -[[package]] -name = "once_cell" -version = "1.21.4" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" - -[[package]] -name = "once_cell_polyfill" -version = "1.70.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "plain" @@ -185,63 +43,15 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" -[[package]] -name = "portable-atomic" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" - -[[package]] -name = "portable-atomic-util" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3" -dependencies = [ - "portable-atomic", -] - [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] -[[package]] -name = "pyo3" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd274650b21d4bfc26a0a47587962c1edb425f69287324355cd040c3ea66071c" -dependencies = [ - "libc", - "once_cell", - "portable-atomic", - "pyo3-build-config", - "pyo3-ffi", - "pyo3-macros", -] - -[[package]] -name = "pyo3-build-config" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5e2a7d2f0d013342f295c048ad19237add5154a55b1c5a254c0ec93d4109078" -dependencies = [ - "target-lexicon", -] - -[[package]] -name = "pyo3-ffi" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca85c467da1bbc8d866eea5deff9cf29ea5f7785054a17da36e65bda9c05845b" -dependencies = [ - "libc", - "pyo3-build-config", -] - [[package]] name = "pyo3-introspection" version = "0.29.0" @@ -254,68 +64,15 @@ dependencies = [ "serde_json", ] -[[package]] -name = "pyo3-macros" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ac53762fd065daa3194dd09337a38bd793a188100fd1a9304c4ab312d901771" -dependencies = [ - "proc-macro2", - "pyo3-macros-backend", - "quote", - "syn", -] - -[[package]] -name = "pyo3-macros-backend" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca3a1557399783172dc5bf39cfca835157732532cba56b71d2292161e53b362" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "quote" -version = "1.0.45" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] -[[package]] -name = "regex" -version = "1.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.8.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" - [[package]] name = "scroll" version = "0.13.0" @@ -333,14 +90,14 @@ checksum = "ed76efe62313ab6610570951494bdaa81568026e0318eaa55f167de70eeea67d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -348,29 +105,29 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "itoa", "memchr", @@ -383,16 +140,14 @@ dependencies = [ name = "stub-gen" version = "0.1.0" dependencies = [ - "env_logger", - "pyo3", "pyo3-introspection", ] [[package]] name = "syn" -version = "2.0.117" +version = "2.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" dependencies = [ "proc-macro2", "quote", @@ -400,10 +155,15 @@ dependencies = [ ] [[package]] -name = "target-lexicon" -version = "0.13.5" +name = "syn" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] [[package]] name = "unicode-ident" @@ -411,29 +171,8 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" -[[package]] -name = "utf8parse" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" - -[[package]] -name = "windows-link" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" - -[[package]] -name = "windows-sys" -version = "0.61.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" -dependencies = [ - "windows-link", -] - [[package]] name = "zmij" -version = "1.0.21" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/bindings/python/tools/stub-gen/Cargo.toml b/bindings/python/tools/stub-gen/Cargo.toml index d7e3cb437..afb83b1bc 100644 --- a/bindings/python/tools/stub-gen/Cargo.toml +++ b/bindings/python/tools/stub-gen/Cargo.toml @@ -2,13 +2,9 @@ name = "stub-gen" version = "0.1.0" edition = "2024" -description = "Stub generation tool for tokenizers Python bindings" - -[[bin]] -name = "stub-gen" -path = "src/main.rs" +description = "Generates py_src .pyi stubs by introspecting the built cdylib" [dependencies] -env_logger = "0.11" -pyo3 = { version="0.29", default-features = false, features = ["auto-initialize", "experimental-inspect"] } -pyo3-introspection = { version="0.29"} +pyo3-introspection = "0.29" + +[workspace] diff --git a/bindings/python/tools/stub-gen/src/main.rs b/bindings/python/tools/stub-gen/src/main.rs index 282e589eb..012aeec7f 100644 --- a/bindings/python/tools/stub-gen/src/main.rs +++ b/bindings/python/tools/stub-gen/src/main.rs @@ -1,270 +1,122 @@ -use pyo3::prelude::*; -use pyo3::types::PyList; -use pyo3_introspection::model::{Class, Module}; +//! Generates the `.pyi` stubs under `py_src/tokenizers/` from the +//! introspection metadata pyo3 embeds in the built extension (the +//! `experimental-inspect` feature). Run after `maturin develop --release`: +//! +//! ```sh +//! cargo run --manifest-path tools/stub-gen/Cargo.toml +//! ``` +//! +//! Return types beyond introspection's reach come from the +//! `#[pyo3(signature = (...) -> "Type")]` annotations in the sources; numpy +//! imports for those annotations are injected here. + use std::path::{Path, PathBuf}; -use std::process::Command; -fn main() -> Result<(), Box> { - env_logger::try_init().ok(); +const MODULE: &str = "tokenizers"; +/// The `#[pymodule]` name inside the cdylib. +const NATIVE_MODULE: &str = "_native"; - let manifest_dir = find_manifest_dir()?; - let cdylib = manifest_dir.join("tokenizers.abi3.so"); - let out_dir = manifest_dir.join("py_src/tokenizers"); - println!("Using manifest directory: {}", manifest_dir.display()); - println!("Using cdylib: {}", cdylib.display()); - println!("Using output directory: {}", out_dir.display()); - build_extension(&manifest_dir)?; - refresh_cdylib(&manifest_dir, &cdylib)?; - setup_python_env()?; - generate_stubs(&cdylib, &out_dir)?; - Ok(()) -} +fn main() -> Result<(), Box> { + let crate_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .ancestors() + .nth(2) + .expect("tools/stub-gen sits two levels under the crate root") + .to_path_buf(); + let cdylib = crate_dir.join(format!( + "target/release/{}{NATIVE_MODULE}.{}", + std::env::consts::DLL_PREFIX, + std::env::consts::DLL_EXTENSION + )); + let out_dir = crate_dir.join("py_src").join(MODULE); -/// Set up PYTHONHOME environment variable if not already set. -/// This is needed for PyO3 embedded Python to find the standard library, -/// especially when using virtual environments created by uv. -fn setup_python_env() -> Result<(), Box> { - if std::env::var_os("PYTHONHOME").is_some() { - return Ok(()); + if !cdylib.is_file() { + return Err(format!( + "no cdylib at {} — run `maturin develop --release` first", + cdylib.display() + ) + .into()); } - // Query Python for its base_prefix (the actual Python installation, not venv) - let output = Command::new("python3") - .args(["-c", "import sys; print(sys.base_prefix, end='')"]) - .output()?; - - if !output.status.success() { - return Err("Failed to query Python base_prefix".into()); - } + let module = pyo3_introspection::introspect_cdylib(&cdylib, NATIVE_MODULE)?; + assert_has_docstrings(&module); - let base_prefix = String::from_utf8(output.stdout)?; - if !base_prefix.is_empty() { - println!("Setting PYTHONHOME={}", base_prefix); - // SAFETY: main is still single-threaded at this point — no other - // threads can race on the environment, and the embedded Python - // interpreter (which reads PYTHONHOME) hasn't been initialized yet. - // FIXME: doesn't look great - unsafe { std::env::set_var("PYTHONHOME", &base_prefix) }; + for (rel_path, contents) in pyo3_introspection::module_stub_files(&module) { + let out_path = out_dir.join(place(&rel_path)); + if let Some(parent) = out_path.parent() { + std::fs::create_dir_all(parent)?; + } + let mut contents = postprocess(&contents); + if rel_path == Path::new("__init__.pyi") { + // `create_exception!` types carry no introspection metadata. + contents.push_str("\nclass TokenizersError(Exception): ...\n"); + } + std::fs::write(&out_path, &contents)?; + println!("generated {}", out_path.display()); } - Ok(()) } -fn find_manifest_dir() -> Result> { - // Look for the bindings/python directory relative to current working directory - // or from the tool's location - let cwd = std::env::current_dir()?; - - // Check if we're already in bindings/python - if cwd.join("pyproject.toml").exists() && cwd.join("py_src").exists() { - return Ok(cwd); +/// Map the introspected layout onto the package layout: the root module stub +/// becomes `__init__.pyi`, and each submodule stub lands inside its runtime +/// shim package (`models.pyi` -> `models/__init__.pyi`) so it shadows the +/// `.py` re-exports for type checkers. +fn place(rel_path: &Path) -> PathBuf { + let name = rel_path + .file_name() + .and_then(|n| n.to_str()) + .expect("stub paths are utf-8 files"); + match name.strip_suffix(".pyi") { + Some("__init__") | None => rel_path.to_path_buf(), + Some(module) => rel_path.with_file_name(module).join("__init__.pyi"), } - - // Check if bindings/python exists relative to cwd - let bindings_python = cwd.join("bindings/python"); - if bindings_python.join("pyproject.toml").exists() { - return Ok(bindings_python); - } - - // Try to find it from the executable location - if let Ok(exe) = std::env::current_exe() { - // Go up from tools/stub-gen/target/... to bindings/python - let mut path = exe.as_path(); - for _ in 0..10 { - if let Some(parent) = path.parent() { - if parent.join("pyproject.toml").exists() && parent.join("py_src").exists() { - return Ok(parent.to_path_buf()); - } - path = parent; - } - } - } - - Err("Could not find bindings/python directory. Run from the tokenizers root or bindings/python directory.".into()) } -fn generate_stubs(cdylib: &Path, out_dir: &Path) -> Result<(), Box> { - if !cdylib.is_file() { - return Err(format!("Failed to locate cdylib at {}", cdylib.display()).into()); - } - - println!("Initializing python"); - Python::initialize(); - let cdylib = cdylib.to_path_buf(); - let out_dir = out_dir.to_path_buf(); - - Python::attach(|py| -> PyResult<()> { - println!("Gathering Python environment information..."); - let sys = py.import("sys")?; - println!("sys.version = {}", sys.getattr("version")?); - println!("sys.executable = {}", sys.getattr("executable")?); - println!("sys.prefix = {}", sys.getattr("prefix")?); - println!("sys.base_prefix = {}", sys.getattr("base_prefix")?); - - let so_dir = cdylib - .parent() - .unwrap_or_else(|| Path::new(".")) - .to_path_buf(); - - let bindings = sys.getattr("path")?; - let sys_path = bindings.cast::()?; - sys_path.insert(0, so_dir.to_str().unwrap())?; - - let sysconfig = PyModule::import(py, "sysconfig")?; - let python_version = sysconfig.call_method0("get_python_version")?; - println!("Using python version: {}", python_version); - let python_lib = sysconfig.call_method("get_config_var", ("LIBDEST",), None)?; - println!("Using python lib: {}", python_lib); - let python_site_packages = sysconfig.call_method("get_path", ("purelib",), None)?; - println!("Using python site-packages: {}", python_site_packages); - py.run( - c"import tokenizers; import sys; print('import ok:', tokenizers.__file__); print('sys.path[0]=', sys.path[0])", - None, - None, - ) - .unwrap_or_else(|e| panic!("Failed to import tokenizers: {:?}", e)); - - println!("Generating stub files"); - assert!( - cdylib.is_file(), - "Failed to locate cdylib at {}", - cdylib.display() +fn postprocess(contents: &str) -> String { + // Cross-submodule references come out relative to the extension root; + // absolutize them to the package. + let mut contents = contents + .replace("from . import", &format!("from {MODULE} import")) + .replace("from .", &format!("from {MODULE}.")); + // Annotated numpy return types need their imports. + if contents.contains("npt.") || contents.contains("np.") { + contents = format!( + "import numpy as np\nimport numpy.typing as npt\n\n{contents}" ); - println!("Found cdylib at {}", cdylib.display()); - - let main_module_name = "tokenizers"; - let python_module = pyo3_introspection::introspect_cdylib(&cdylib, main_module_name) - .unwrap_or_else(|_| panic!("Failed introspection of {}", main_module_name)); - - // Sanity check: if docstrings are missing the patched pyo3 in - // .cargo/config.toml didn't actually apply to the cdylib build (most - // common cause: the rev pinned in tools/stub-gen/Cargo.toml and the - // version requirement in bindings/python/Cargo.toml have drifted, so - // cargo silently ignores `[patch.crates-io]`). Check that at least - // one well-known class still carries its docstring before writing - // out otherwise-empty stubs. - assert_introspection_has_docstrings(&python_module); - - let type_stubs = pyo3_introspection::module_stub_files(&python_module); - - for (rel_path, contents) in type_stubs { - let out_path = out_dir.join(&rel_path); - if let Some(parent) = out_path.parent() { - std::fs::create_dir_all(parent) - .unwrap_or_else(|_| panic!("Failed introspection of {}", main_module_name)) - } - let contents = absolutize_local_imports(&contents, main_module_name); - std::fs::write(&out_path, contents).expect("Failed to write stubs file"); - println!("Generated stub: {}", out_path.display()); - } - - Ok(()) - })?; - - Ok(()) -} - -fn absolutize_local_imports(contents: &str, root_module: &str) -> String { + } contents - .replace("from . import", &format!("from {root_module} import")) - .replace("from .", &format!("from {root_module}.")) } -/// Walk the introspected module tree and count classes, functions, and -/// attributes that carry a docstring. Returns `(with_docstring, total)`. -fn count_docstrings(module: &Module) -> (usize, usize) { - let (mut with_doc, mut total) = (0, 0); - for f in &module.functions { - total += 1; - if f.docstring.is_some() { - with_doc += 1; - } - } - for a in &module.attributes { - total += 1; - if a.docstring.is_some() { - with_doc += 1; - } - } - fn walk_class(c: &Class, with_doc: &mut usize, total: &mut usize) { - *total += 1; - if c.docstring.is_some() { - *with_doc += 1; +/// Fail loudly if introspection came back without docstrings — that means the +/// cdylib was built without `experimental-inspect` (or the feature broke) and +/// the stubs would silently lose all documentation. +fn assert_has_docstrings(module: &pyo3_introspection::model::Module) { + fn count(module: &pyo3_introspection::model::Module) -> (usize, usize) { + let mut with_doc = 0; + let mut total = 0; + for f in &module.functions { + total += 1; + with_doc += f.docstring.is_some() as usize; } - for m in &c.methods { - *total += 1; - if m.docstring.is_some() { - *with_doc += 1; + for c in &module.classes { + total += 1; + with_doc += c.docstring.is_some() as usize; + for m in &c.methods { + total += 1; + with_doc += m.docstring.is_some() as usize; } } - for inner in &c.inner_classes { - walk_class(inner, with_doc, total); + for sub in &module.modules { + let (w, t) = count(sub); + with_doc += w; + total += t; } + (with_doc, total) } - for c in &module.classes { - walk_class(c, &mut with_doc, &mut total); - } - for sub in &module.modules { - let (sw, st) = count_docstrings(sub); - with_doc += sw; - total += st; - } - (with_doc, total) -} - -/// Abort if the introspected module tree is missing docstrings — usually -/// caused by `[patch.crates-io]` in `.cargo/config.toml` not actually -/// applying to the cdylib build. Better to fail loudly here than silently -/// strip every docstring from the published stubs. -fn assert_introspection_has_docstrings(module: &Module) { - let (with_doc, total) = count_docstrings(module); - println!( - "Docstring coverage: {}/{} items carry a docstring", - with_doc, total - ); + let (with_doc, total) = count(module); + println!("docstring coverage: {with_doc}/{total}"); assert!( with_doc > 0, - "stub-gen produced 0/{} docstrings — pyo3-introspection is reading \ - the cdylib but every docstring slot is empty. Most likely the \ - `[patch.crates-io]` in `.cargo/config.toml` (injected by `make \ - style`) did not actually apply: the rev pinned in \ - tools/stub-gen/Cargo.toml must match the version requirement in \ - bindings/python/Cargo.toml. Check `cargo build` output for \ - `warning: patch \\`pyo3 vX.Y.Z\\` was not used in the crate graph` \ - and align the versions before re-running.", - total, - ); -} - -fn build_extension(manifest_dir: &Path) -> Result<(), Box> { - println!("Building and installing extension (release)..."); - match Command::new("maturin").current_dir(manifest_dir).args(["develop", "--release"]).status() { - Ok(_) => {} - Err(e) => { eprintln!("Hint: Failed to run `maturin develop`: {:?}. Is maturin even installed? ;)", e) } - } ; - - Ok(()) -} - -fn refresh_cdylib(manifest_dir: &Path, cdylib: &Path) -> Result<(), Box> { - let built_cdylib = manifest_dir.join(format!( - "target/release/{}tokenizers.{}", - std::env::consts::DLL_PREFIX, - std::env::consts::DLL_EXTENSION - )); - - if !built_cdylib.is_file() { - return Err(format!( - "Could not find built cdylib at {}.", - built_cdylib.display() - ) - .into()); - } - - println!( - "Refreshing cdylib used for introspection: {}", - cdylib.display() + "introspection returned 0/{total} docstrings — was the cdylib built \ + with the `experimental-inspect` pyo3 feature?" ); - std::fs::copy(&built_cdylib, cdylib)?; - Ok(()) } From 3f5bb5577f8d6c01d4a182750d3d0e275ebeb405 Mon Sep 17 00:00:00 2001 From: SBrandeis <33657802+SBrandeis@users.noreply.github.com> Date: Wed, 22 Jul 2026 14:22:12 +0200 Subject: [PATCH 08/19] iteration --- .github/workflows/python.yml | 28 +- bindings/python/CHANGELOG.md | 533 ++++++++++++++++++ bindings/python/Cargo.toml | 2 +- bindings/python/Makefile | 17 +- bindings/python/README.md | 5 +- bindings/python/benches/bench_vs_release.py | 21 +- .../python/examples/01_train_and_encode.py | 2 +- bindings/python/examples/03_threading.py | 14 +- .../examples/04_train_bert_wordpiece.py | 50 ++ .../python/examples/05_train_bytelevel_bpe.py | 48 ++ .../python/examples/06_train_with_datasets.py | 27 + .../tokenizers/pre_tokenizers/__init__.pyi | 7 + .../py_src/tokenizers/trainers/__init__.py | 2 + .../py_src/tokenizers/trainers/__init__.pyi | 28 +- bindings/python/pyproject.toml | 9 + bindings/python/rust-toolchain | 1 + bindings/python/src/added_token.rs | 13 +- bindings/python/src/pre_tokenizers.rs | 8 + bindings/python/src/tokenizer.rs | 102 ++++ bindings/python/src/trainers.rs | 157 +++++- bindings/python/tests/__init__.py | 0 bindings/python/tests/conftest.py | 56 ++ bindings/python/tests/test_added_token.py | 30 + bindings/python/tests/test_components.py | 97 ++++ bindings/python/tests/test_parity_trainer.py | 70 +++ bindings/python/tests/test_pretrained.py | 32 ++ bindings/python/tests/test_threading.py | 44 ++ bindings/python/tests/test_tokenizer.py | 117 ++++ bindings/python/tests/test_trainers.py | 68 +++ 29 files changed, 1558 insertions(+), 30 deletions(-) create mode 100644 bindings/python/CHANGELOG.md create mode 100644 bindings/python/examples/04_train_bert_wordpiece.py create mode 100644 bindings/python/examples/05_train_bytelevel_bpe.py create mode 100644 bindings/python/examples/06_train_with_datasets.py create mode 100644 bindings/python/rust-toolchain create mode 100644 bindings/python/tests/__init__.py create mode 100644 bindings/python/tests/conftest.py create mode 100644 bindings/python/tests/test_added_token.py create mode 100644 bindings/python/tests/test_components.py create mode 100644 bindings/python/tests/test_parity_trainer.py create mode 100644 bindings/python/tests/test_pretrained.py create mode 100644 bindings/python/tests/test_threading.py create mode 100644 bindings/python/tests/test_tokenizer.py create mode 100644 bindings/python/tests/test_trainers.py diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 11dee20aa..1076d61be 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -43,7 +43,7 @@ jobs: working-directory: ./bindings/python run: | uv venv .venv --python ${{ matrix.python }} - uv pip install --python .venv/bin/python maturin numpy + uv pip install --python .venv/bin/python maturin numpy pytest datasets source .venv/bin/activate && maturin develop --release # The examples read real tokenizer.json files + a text corpus. Cached @@ -52,7 +52,9 @@ jobs: - name: Cache test data uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: - path: tokenizers/data + path: | + tokenizers/data + ~/.cache/huggingface key: python-test-data-${{ hashFiles('tokenizers/Makefile', 'tokenizers/tk-encode/examples/bench_models.json') }} - name: Download model tokenizers and corpus @@ -61,12 +63,25 @@ jobs: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: make bench-models data/big.txt HF="uvx --from huggingface_hub hf" + # Network-marked tests (Hub downloads) are skipped: they'd flake on + # rate limits and from_file covers the same code paths. + - name: Run the tests + working-directory: ./bindings/python + run: .venv/bin/python -m pytest -q -m "not network" + + # Every example runs here so they can never rot. 06 pulls wikitext-2 + # from the Hub (cached above). - name: Run the examples working-directory: ./bindings/python + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | .venv/bin/python examples/01_train_and_encode.py .venv/bin/python examples/02_pretrained.py .venv/bin/python examples/03_threading.py + .venv/bin/python examples/04_train_bert_wordpiece.py + .venv/bin/python examples/05_train_bytelevel_bpe.py + .venv/bin/python examples/06_train_with_datasets.py # The .pyi stubs are generated from the built extension; a diff here # means someone changed the Rust API without running `make stubs`. @@ -118,6 +133,15 @@ jobs: - name: Lint with Clippy run: cargo clippy --manifest-path ./bindings/python/Cargo.toml --all-targets --all-features -- -D warnings + - name: Install uv + uses: astral-sh/setup-uv@v6 + + - name: Lint the Python sources with ruff + working-directory: ./bindings/python + run: | + uvx ruff check benches examples tests + uvx ruff format --check benches examples tests + audit: name: Audit dependencies runs-on: ubuntu-latest diff --git a/bindings/python/CHANGELOG.md b/bindings/python/CHANGELOG.md new file mode 100644 index 000000000..ce8a79b2b --- /dev/null +++ b/bindings/python/CHANGELOG.md @@ -0,0 +1,533 @@ +# Changelog +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [1.0.0] (unreleased) + +Ground-up rewrite of the bindings on the `PipelineTokenizer` encode path. +Same `tokenizer.json` files and the same ids as 0.x, much faster through +Python: encode never holds the GIL, batches run multi-threaded in Rust, +inputs are borrowed instead of copied, and ids come back as `numpy.uint32` +arrays without a copy. + +Breaking changes: + +- `encode` returns a numpy array of ids, not an `Encoding` object. Offsets, + type ids, attention masks, truncation and padding are gone from the encode + path. +- Not implemented yet (loud errors, never wrong ids): `decode`, + post-processor templates (pass `add_special_tokens=False`), and the + `Metaspace` pre-tokenizer. +- Custom Python components are not supported; components are plain values. +- `decoders`, `processors`, and the `implementations` helpers are gone. +- Wheels are abi3-py310 only; free-threaded interpreters (3.14t) are not + supported. + +## [0.13.2] + +- [#1096] Python 3.11 support + +## [0.13.1] + +- [#1072] Fixing Roberta type ids. + +## [0.13.0] + +- [#956] PyO3 version upgrade +- [#1055] M1 automated builds +- [#1008] `Decoder` is now a composable trait, but without being backward incompatible +- [#1047, #1051, #1052] `Processor` is now a composable trait, but without being backward incompatible + +Both trait changes warrant a "major" number since, despite best efforts to not break backward + compatibility, the code is different enough that we cannot be exactly sure. + +## [0.12.1] + +- [#938] **Reverted breaking change**. https://github.com/huggingface/transformers/issues/16520 + +## [0.12.0] YANKED + +Bump minor version because of a breaking change. + +- [#938] [REVERTED IN 0.12.1] **Breaking change**. Decoder trait is modified to be composable. This is only breaking if you are using decoders on their own. tokenizers should be error free. +- [#939] Making the regex in `ByteLevel` pre_tokenizer optional (necessary for BigScience) + +- [#952] Fixed the vocabulary size of UnigramTrainer output (to respect added tokens) +- [#954] Fixed not being able to save vocabularies with holes in vocab (ConvBert). Yell warnings instead, but stop panicking. +- [#962] Fix tests for python 3.10 +- [#961] Added link for Ruby port of `tokenizers` + +## [0.11.6] + +- [#919] Fixing single_word AddedToken. (regression from 0.11.2) +- [#916] Deserializing faster `added_tokens` by loading them in batch. + +## [0.11.5] + +- [#895] Build `python 3.10` wheels. + +## [0.11.4] + +- [#884] Fixing bad deserialization following inclusion of a default for Punctuation + +## [0.11.3] + +- [#882] Fixing Punctuation deserialize without argument. +- [#868] Fixing missing direction in TruncationParams +- [#860] Adding TruncationSide to TruncationParams + +## [0.11.0] + +### Fixed + +- [#585] Conda version should now work on old CentOS +- [#844] Fixing interaction between `is_pretokenized` and `trim_offsets`. +- [#851] Doc links + +### Added +- [#657]: Add SplitDelimiterBehavior customization to Punctuation constructor +- [#845]: Documentation for `Decoders`. + +### Changed +- [#850]: Added a feature gate to enable disabling `http` features +- [#718]: Fix `WordLevel` tokenizer determinism during training +- [#762]: Add a way to specify the unknown token in `SentencePieceUnigramTokenizer` +- [#770]: Improved documentation for `UnigramTrainer` +- [#780]: Add `Tokenizer.from_pretrained` to load tokenizers from the Hugging Face Hub +- [#793]: Saving a pretty JSON file by default when saving a tokenizer + +## [0.10.3] + +### Fixed +- [#686]: Fix SPM conversion process for whitespace deduplication +- [#707]: Fix stripping strings containing Unicode characters + +### Added +- [#693]: Add a CTC Decoder for Wave2Vec models + +### Removed +- [#714]: Removed support for Python 3.5 + +## [0.10.2] + +### Fixed +- [#652]: Fix offsets for `Precompiled` corner case +- [#656]: Fix BPE `continuing_subword_prefix` +- [#674]: Fix `Metaspace` serialization problems + +## [0.10.1] + +### Fixed +- [#616]: Fix SentencePiece tokenizers conversion +- [#617]: Fix offsets produced by Precompiled Normalizer (used by tokenizers converted from SPM) +- [#618]: Fix Normalizer.normalize with `PyNormalizedStringRefMut` +- [#620]: Fix serialization/deserialization for overlapping models +- [#621]: Fix `ByteLevel` instantiation from a previously saved state (using `__getstate__()`) + +## [0.10.0] + +### Added +- [#508]: Add a Visualizer for notebooks to help understand how the tokenizers work +- [#519]: Add a `WordLevelTrainer` used to train a `WordLevel` model +- [#533]: Add support for conda builds +- [#542]: Add Split pre-tokenizer to easily split using a pattern +- [#544]: Ability to train from memory. This also improves the integration with `datasets` +- [#590]: Add getters/setters for components on BaseTokenizer +- [#574]: Add `fust_unk` option to SentencePieceBPETokenizer + +### Changed +- [#509]: Automatically stubbing the `.pyi` files +- [#519]: Each `Model` can return its associated `Trainer` with `get_trainer()` +- [#530]: The various attributes on each component can be get/set (ie. +`tokenizer.model.dropout = 0.1`) +- [#538]: The API Reference has been improved and is now up-to-date. + +### Fixed +- [#519]: During training, the `Model` is now trained in-place. This fixes several bugs that were +forcing to reload the `Model` after a training. +- [#539]: Fix `BaseTokenizer` enable_truncation docstring + +## [0.9.4] + +### Fixed +- [#492]: Fix `from_file` on `BertWordPieceTokenizer` +- [#498]: Fix the link to download `sentencepiece_model_pb2.py` +- [#500]: Fix a typo in the docs quicktour + +### Changed +- [#506]: Improve Encoding mappings for pairs of sequence + +## [0.9.3] + +### Fixed +- [#470]: Fix hanging error when training with custom component +- [#476]: TemplateProcessing serialization is now deterministic +- [#481]: Fix SentencePieceBPETokenizer.from_files + +### Added +- [#477]: UnicodeScripts PreTokenizer to avoid merges between various scripts +- [#480]: Unigram now accepts an `initial_alphabet` and handles `special_tokens` correctly + +## [0.9.2] + +### Fixed +- [#464]: Fix a problem with RobertaProcessing being deserialized as BertProcessing + +## [0.9.1] + +### Fixed +- [#459]: Fix a problem with deserialization + +## [0.9.0] + +### Fixed +- [#362]: Fix training deadlock with Python components. +- [#363]: Fix a crash when calling `.train` with some non-existent files +- [#355]: Remove a lot of possible crashes +- [#389]: Improve truncation (crash and consistency) + +### Added +- [#379]: Add the ability to call `encode`/`encode_batch` with numpy arrays +- [#292]: Support for the Unigram algorithm +- [#378], [#394], [#416], [#417]: Many new Normalizer and PreTokenizer +- [#403]: Add `TemplateProcessing` `PostProcessor`. +- [#420]: Ability to fuse the "unk" token in BPE. + +### Changed +- [#360]: Lots of improvements related to words/alignment tracking +- [#426]: Improvements on error messages thanks to PyO3 0.12 + +## [0.8.1] + +### Fixed +- [#333]: Fix deserialization of `AddedToken`, where the content was not restored properly + +### Changed +- [#329]: Improved warning and behavior when we detect a fork +- [#330]: BertNormalizer now keeps the same behavior than the original implementation when +`strip_accents` is not specified. + +## [0.8.0] + +### Highlights of this release +- We can now encode both pre-tokenized inputs, and raw strings. This is especially usefull when +processing datasets that are already pre-tokenized like for NER (Name Entity Recognition), and helps +while applying labels to each word. +- Full tokenizer serialization. It is now easy to save a tokenizer to a single JSON file, to later +load it back with just one line of code. That's what sharing a Tokenizer means now: 1 line of code. +- With the serialization comes the compatibility with `Pickle`! The Tokenizer, all of its components, +Encodings, everything can be pickled! +- Training a tokenizer is now even faster (up to 5-10x) than before! +- Compatibility with `multiprocessing`, even when using the `fork` start method. Since this library +makes heavy use of the multithreading capacities of our computers to allows a very fast tokenization, +this led to problems (deadlocks) when used with `multiprocessing`. This version now allows to +disable the parallelism, and will warn you if this is necessary. +- And a lot of other improvements, and fixes. + +### Fixed +- [#286]: Fix various crash when training a BPE model +- [#309]: Fixed a few bugs related to additional vocabulary/tokens + +### Added +- [#272]: Serialization of the `Tokenizer` and all the parts (`PreTokenizer`, `Normalizer`, ...). +This adds some methods to easily save/load an entire tokenizer (`from_str`, `from_file`). +- [#273]: `Tokenizer` and its parts are now pickable +- [#289]: Ability to pad to a multiple of a specified value. This is especially useful to ensure +activation of the Tensor Cores, while ensuring padding to a multiple of 8. Use with +`enable_padding(pad_to_multiple_of=8)` for example. +- [#298]: Ability to get the currently set truncation/padding params +- [#311]: Ability to enable/disable the parallelism using the `TOKENIZERS_PARALLELISM` environment +variable. This is especially usefull when using `multiprocessing` capabilities, with the `fork` +start method, which happens to be the default on Linux systems. Without disabling the parallelism, +the process dead-locks while encoding. (Cf [#187] for more information) + +### Changed +- Improved errors generated during truncation: When the provided max length is too low are +now handled properly. +- [#249] `encode` and `encode_batch` now accept pre-tokenized inputs. When the input is pre-tokenized, +the argument `is_pretokenized=True` must be specified. +- [#276]: Improve BPE training speeds, by reading files sequentially, but parallelizing the +processing of each file +- [#280]: Use `onig` for byte-level pre-tokenization to remove all the differences with the original +implementation from GPT-2 +- [#309]: Improved the management of the additional vocabulary. This introduces an option +`normalized`, controlling whether a token should be extracted from the normalized version of the +input text. + +## [0.7.0] + +### Changed +- Only one progress bar while reading files during training. This is better for use-cases with +a high number of files as it avoids having too many progress bars on screen. Also avoids reading the +size of each file before starting to actually read these files, as this process could take really +long. +- [#193]: `encode` and `encode_batch` now take a new optional argument, specifying whether we +should add the special tokens. This is activated by default. +- [#197]: `original_str` and `normalized_str` have been removed from the `Encoding` returned by +`encode` and `encode_batch`. This brings a reduction of 70% of the memory footprint. +- [#197]: The offsets provided on `Encoding` are now relative to the original string, and not the +normalized one anymore. +- The added token given to `add_special_tokens` or `add_tokens` on a `Tokenizer`, or while using +`train(special_tokens=...)` can now be instances of `AddedToken` to provide more control over these +tokens. +- [#136]: Updated Pyo3 version +- [#136]: Static methods `Model.from_files` and `Model.empty` are removed in favor of using +constructors. +- [#239]: `CharBPETokenizer` now corresponds to OpenAI GPT BPE implementation by default. + +### Added +- [#188]: `ByteLevel` is also a `PostProcessor` now and handles trimming the offsets if activated. +This avoids the unintuitive inclusion of the whitespaces in the produced offsets, even if these +whitespaces are part of the actual token. +It has been added to `ByteLevelBPETokenizer` but it is off by default (`trim_offsets=False`). +- [#236]: `RobertaProcessing` also handles trimming the offsets. +- [#234]: New alignment mappings on the `Encoding`. Provide methods to easily convert between `char` +or `word` (input space) and `token` (output space). +- `post_process` can be called on the `Tokenizer` +- [#208]: Ability to retrieve the vocabulary from the `Tokenizer` with +`get_vocab(with_added_tokens: bool)` +- [#136] Models can now be instantiated through object constructors. + +### Fixed +- [#193]: Fix some issues with the offsets being wrong with the `ByteLevel` BPE: + - when `add_prefix_space=True` + - [#156]: when a Unicode character gets split-up in multiple byte-level characters +- Fix a bug where offsets were wrong when there was any added tokens in the sequence being encoded. +- [#175]: Fix a bug that prevented the addition of more than a certain amount of tokens (even if +not advised, but that's not the question). +- [#205]: Trim the decoded string in `BPEDecoder` used by `CharBPETokenizer` + +### How to migrate +- Add the `ByteLevel` `PostProcessor` to your byte-level BPE tokenizers if relevant. If you are +using `ByteLevelBPETokenizer`, this option is disabled by default (`trim_offsets=False`). +- `BertWordPieceTokenizer` option to `add_special_tokens` must now be given to `encode` or +`encode_batch` +- Access to the `original_str` on the `Encoding` has been removed. The original string is the input +of `encode` so it didn't make sense to keep it here. +- No need to call `original_str.offsets(offsets[N])` to convert offsets to the original string. They +are now relative to the original string by default. +- Access to the `normalized_str` on the `Encoding` has been removed. Can be retrieved by calling +`normalize(sequence)` on the `Tokenizer` +- Change `Model.from_files` and `Model.empty` to use constructor. The model constructor should take +the same arguments as the old methods. (ie `BPE(vocab, merges)` or `BPE()`) +- If you were using the `CharBPETokenizer` and want to keep the same behavior as before, set +`bert_normalizer=False` and `split_on_whitespace_only=True`. + +## [0.6.0] + +### Changed +- [#165]: Big improvements in speed for BPE (Both training and tokenization) + +### Fixed +- [#160]: Some default tokens were missing from `BertWordPieceTokenizer` +- [#156]: There was a bug in ByteLevel PreTokenizer that caused offsets to be wrong if a char got +split up in multiple bytes. +- [#174]: The `longest_first` truncation strategy had a bug + +## [0.5.2] +- [#163]: Do not open all files directly while training + +### Fixed +- We introduced a bug related to the saving of the WordPiece model in 0.5.1: The `vocab.txt` file +was named `vocab.json`. This is now fixed. +- The `WordLevel` model was also saving its vocabulary to the wrong format. + +## [0.5.1] + +### Changed +- `name` argument is now optional when saving a `Model`'s vocabulary. When the name is not +specified, the files get a more generic naming, like `vocab.json` or `merges.txt`. + +## [0.5.0] + +### Changed +- [#145]: `BertWordPieceTokenizer` now cleans up some tokenization artifacts while decoding +- [#149]: `ByteLevelBPETokenizer` now has `dropout`. +- `do_lowercase` has been changed to `lowercase` for consistency between the different tokenizers. +(Especially `ByteLevelBPETokenizer` and `CharBPETokenizer`) +- [#139]: Expose `__len__` on `Encoding` +- Improved padding performances. + +### Added +- Added a new `Strip` normalizer + +### Fixed +- [#145]: Decoding was buggy on `BertWordPieceTokenizer`. +- [#152]: Some documentation and examples were still using the old `BPETokenizer` + +### How to migrate +- Use `lowercase` when initializing `ByteLevelBPETokenizer` or `CharBPETokenizer` instead of +`do_lowercase`. + +## [0.4.2] + +### Fixed +- [#137]: Fix a bug in the class `WordPieceTrainer` that prevented `BertWordPieceTokenizer` from +being trained. + +## [0.4.1] + +### Fixed +- [#134]: Fix a bug related to the punctuation in BertWordPieceTokenizer + +## [0.4.0] + +### Changed +- [#131]: Replaced all .new() class methods by a proper __new__ implementation +- Improved typings + +### How to migrate +- Remove all `.new` on all classe instanciations + +## [0.3.0] + +### Changed +- BPETokenizer has been renamed to CharBPETokenizer for clarity. +- Improve truncation/padding and the handling of overflowing tokens. Now when a sequence gets +truncated, we provide a list of overflowing `Encoding` that are ready to be processed by a language +model, just as the main `Encoding`. +- Provide mapping to the original string offsets using: +``` +output = tokenizer.encode(...) +print(output.original_str.offsets(output.offsets[3])) +``` +- [#99]: Exposed the vocabulary size on all tokenizers + +### Added +- Added `CharDelimiterSplit`: a new `PreTokenizer` that allows splitting sequences on the given +delimiter (Works like `.split(delimiter)`) +- Added `WordLevel`: a new model that simply maps `tokens` to their `ids`. + +### Fixed +- Fix a bug with IndexableString +- Fix a bug with truncation + +### How to migrate +- Rename `BPETokenizer` to `CharBPETokenizer` +- `Encoding.overflowing` is now a List instead of a `Optional[Encoding]` + +## [0.2.1] + +### Fixed +- Fix a bug with the IDs associated with added tokens. +- Fix a bug that was causing crashes in Python 3.5 + +[#1096]: https://github.com/huggingface/tokenizers/pull/1096 +[#1072]: https://github.com/huggingface/tokenizers/pull/1072 +[#956]: https://github.com/huggingface/tokenizers/pull/956 +[#1008]: https://github.com/huggingface/tokenizers/pull/1008 +[#1009]: https://github.com/huggingface/tokenizers/pull/1009 +[#1047]: https://github.com/huggingface/tokenizers/pull/1047 +[#1055]: https://github.com/huggingface/tokenizers/pull/1055 +[#1051]: https://github.com/huggingface/tokenizers/pull/1051 +[#1052]: https://github.com/huggingface/tokenizers/pull/1052 +[#938]: https://github.com/huggingface/tokenizers/pull/938 +[#939]: https://github.com/huggingface/tokenizers/pull/939 +[#952]: https://github.com/huggingface/tokenizers/pull/952 +[#954]: https://github.com/huggingface/tokenizers/pull/954 +[#962]: https://github.com/huggingface/tokenizers/pull/962 +[#961]: https://github.com/huggingface/tokenizers/pull/961 +[#960]: https://github.com/huggingface/tokenizers/pull/960 +[#919]: https://github.com/huggingface/tokenizers/pull/919 +[#916]: https://github.com/huggingface/tokenizers/pull/916 +[#895]: https://github.com/huggingface/tokenizers/pull/895 +[#884]: https://github.com/huggingface/tokenizers/pull/884 +[#882]: https://github.com/huggingface/tokenizers/pull/882 +[#868]: https://github.com/huggingface/tokenizers/pull/868 +[#860]: https://github.com/huggingface/tokenizers/pull/860 +[#850]: https://github.com/huggingface/tokenizers/pull/850 +[#844]: https://github.com/huggingface/tokenizers/pull/844 +[#845]: https://github.com/huggingface/tokenizers/pull/845 +[#851]: https://github.com/huggingface/tokenizers/pull/851 +[#585]: https://github.com/huggingface/tokenizers/pull/585 +[#793]: https://github.com/huggingface/tokenizers/pull/793 +[#780]: https://github.com/huggingface/tokenizers/pull/780 +[#770]: https://github.com/huggingface/tokenizers/pull/770 +[#762]: https://github.com/huggingface/tokenizers/pull/762 +[#718]: https://github.com/huggingface/tokenizers/pull/718 +[#714]: https://github.com/huggingface/tokenizers/pull/714 +[#707]: https://github.com/huggingface/tokenizers/pull/707 +[#693]: https://github.com/huggingface/tokenizers/pull/693 +[#686]: https://github.com/huggingface/tokenizers/pull/686 +[#674]: https://github.com/huggingface/tokenizers/pull/674 +[#657]: https://github.com/huggingface/tokenizers/pull/657 +[#656]: https://github.com/huggingface/tokenizers/pull/656 +[#652]: https://github.com/huggingface/tokenizers/pull/652 +[#621]: https://github.com/huggingface/tokenizers/pull/621 +[#620]: https://github.com/huggingface/tokenizers/pull/620 +[#618]: https://github.com/huggingface/tokenizers/pull/618 +[#617]: https://github.com/huggingface/tokenizers/pull/617 +[#616]: https://github.com/huggingface/tokenizers/pull/616 +[#590]: https://github.com/huggingface/tokenizers/pull/590 +[#574]: https://github.com/huggingface/tokenizers/pull/574 +[#544]: https://github.com/huggingface/tokenizers/pull/544 +[#542]: https://github.com/huggingface/tokenizers/pull/542 +[#539]: https://github.com/huggingface/tokenizers/pull/539 +[#538]: https://github.com/huggingface/tokenizers/pull/538 +[#533]: https://github.com/huggingface/tokenizers/pull/533 +[#530]: https://github.com/huggingface/tokenizers/pull/530 +[#519]: https://github.com/huggingface/tokenizers/pull/519 +[#509]: https://github.com/huggingface/tokenizers/pull/509 +[#508]: https://github.com/huggingface/tokenizers/pull/508 +[#506]: https://github.com/huggingface/tokenizers/pull/506 +[#500]: https://github.com/huggingface/tokenizers/pull/500 +[#498]: https://github.com/huggingface/tokenizers/pull/498 +[#492]: https://github.com/huggingface/tokenizers/pull/492 +[#481]: https://github.com/huggingface/tokenizers/pull/481 +[#480]: https://github.com/huggingface/tokenizers/pull/480 +[#477]: https://github.com/huggingface/tokenizers/pull/477 +[#476]: https://github.com/huggingface/tokenizers/pull/476 +[#470]: https://github.com/huggingface/tokenizers/pull/470 +[#464]: https://github.com/huggingface/tokenizers/pull/464 +[#459]: https://github.com/huggingface/tokenizers/pull/459 +[#420]: https://github.com/huggingface/tokenizers/pull/420 +[#417]: https://github.com/huggingface/tokenizers/pull/417 +[#416]: https://github.com/huggingface/tokenizers/pull/416 +[#403]: https://github.com/huggingface/tokenizers/pull/403 +[#394]: https://github.com/huggingface/tokenizers/pull/394 +[#389]: https://github.com/huggingface/tokenizers/pull/389 +[#379]: https://github.com/huggingface/tokenizers/pull/379 +[#378]: https://github.com/huggingface/tokenizers/pull/378 +[#363]: https://github.com/huggingface/tokenizers/pull/363 +[#362]: https://github.com/huggingface/tokenizers/pull/362 +[#360]: https://github.com/huggingface/tokenizers/pull/360 +[#355]: https://github.com/huggingface/tokenizers/pull/355 +[#333]: https://github.com/huggingface/tokenizers/pull/333 +[#330]: https://github.com/huggingface/tokenizers/pull/330 +[#329]: https://github.com/huggingface/tokenizers/pull/329 +[#311]: https://github.com/huggingface/tokenizers/pull/311 +[#309]: https://github.com/huggingface/tokenizers/pull/309 +[#292]: https://github.com/huggingface/tokenizers/pull/292 +[#289]: https://github.com/huggingface/tokenizers/pull/289 +[#286]: https://github.com/huggingface/tokenizers/pull/286 +[#280]: https://github.com/huggingface/tokenizers/pull/280 +[#276]: https://github.com/huggingface/tokenizers/pull/276 +[#273]: https://github.com/huggingface/tokenizers/pull/273 +[#272]: https://github.com/huggingface/tokenizers/pull/272 +[#249]: https://github.com/huggingface/tokenizers/pull/249 +[#239]: https://github.com/huggingface/tokenizers/pull/239 +[#236]: https://github.com/huggingface/tokenizers/pull/236 +[#234]: https://github.com/huggingface/tokenizers/pull/234 +[#208]: https://github.com/huggingface/tokenizers/pull/208 +[#205]: https://github.com/huggingface/tokenizers/issues/205 +[#197]: https://github.com/huggingface/tokenizers/pull/197 +[#193]: https://github.com/huggingface/tokenizers/pull/193 +[#190]: https://github.com/huggingface/tokenizers/pull/190 +[#188]: https://github.com/huggingface/tokenizers/pull/188 +[#187]: https://github.com/huggingface/tokenizers/issues/187 +[#175]: https://github.com/huggingface/tokenizers/issues/175 +[#174]: https://github.com/huggingface/tokenizers/issues/174 +[#165]: https://github.com/huggingface/tokenizers/pull/165 +[#163]: https://github.com/huggingface/tokenizers/issues/163 +[#160]: https://github.com/huggingface/tokenizers/issues/160 +[#156]: https://github.com/huggingface/tokenizers/pull/156 +[#152]: https://github.com/huggingface/tokenizers/issues/152 +[#149]: https://github.com/huggingface/tokenizers/issues/149 +[#145]: https://github.com/huggingface/tokenizers/issues/145 +[#139]: https://github.com/huggingface/tokenizers/issues/139 +[#137]: https://github.com/huggingface/tokenizers/issues/137 +[#134]: https://github.com/huggingface/tokenizers/issues/134 +[#131]: https://github.com/huggingface/tokenizers/issues/131 +[#99]: https://github.com/huggingface/tokenizers/pull/99 diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index bd728de19..eb9552fc3 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -16,7 +16,7 @@ rayon = "1.10" serde = "1.0" serde_json = "1.0" tk-encode = { path = "../../tokenizers/tk-encode", features = ["fancy-regex"] } -tk-train = { path = "../../tokenizers/tk-train" } +tk-train = { path = "../../tokenizers/tk-train", features = ["parity-aware-bpe"] } [target.'cfg(unix)'.dependencies] libc = "0.2" diff --git a/bindings/python/Makefile b/bindings/python/Makefile index 67f13839b..896436d05 100644 --- a/bindings/python/Makefile +++ b/bindings/python/Makefile @@ -7,19 +7,28 @@ dev: .venv .venv: uv venv .venv - uv pip install --python $(PYTHON) maturin numpy + uv pip install --python $(PYTHON) maturin numpy pytest ruff # Regenerate the .pyi stubs from the built extension. Run after `make dev`. .PHONY: stubs stubs: cargo run --manifest-path tools/stub-gen/Cargo.toml -# Run the end-to-end examples (train, pretrained parity, threading). +.PHONY: test +test: dev + $(PYTHON) -m pytest -q + +# Run the end-to-end examples. `datasets` (for 06) is installed on demand so +# `make dev` stays lean; 06 downloads wikitext-2 (~12 MB) on first run. .PHONY: examples examples: dev + uv pip install --python $(PYTHON) datasets $(PYTHON) examples/01_train_and_encode.py $(PYTHON) examples/02_pretrained.py $(PYTHON) examples/03_threading.py + $(PYTHON) examples/04_train_bert_wordpiece.py + $(PYTHON) examples/05_train_bytelevel_bpe.py + $(PYTHON) examples/06_train_with_datasets.py # The released PyPI wheel shares our package name, so it lives in its own # directory that the bench subprocess puts on PYTHONPATH. @@ -32,9 +41,11 @@ bench: dev .release $(PYTHON) benches/bench_vs_release.py .PHONY: lint -lint: +lint: .venv cargo fmt --check cargo clippy --all-targets -- -D warnings + .venv/bin/ruff check benches examples tests + .venv/bin/ruff format --check benches examples tests .PHONY: clean clean: diff --git a/bindings/python/README.md b/bindings/python/README.md index 21967f5f5..efa25afe1 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -64,10 +64,11 @@ To build a distributable wheel instead: `maturin build --release` (find it in Other targets: ```sh -make examples # run the three end-to-end examples (needs ../../tokenizers/data) +make test # pytest suite in tests/ +make examples # run the end-to-end examples (needs ../../tokenizers/data) make bench # benchmark against the released tokenizers wheel from PyPI make stubs # regenerate the .pyi type stubs from the built extension -make lint # cargo fmt --check + clippy -D warnings +make lint # cargo fmt + clippy, ruff over the python sources ``` The examples and the benchmark read test data from `../../tokenizers/data`. diff --git a/bindings/python/benches/bench_vs_release.py b/bindings/python/benches/bench_vs_release.py index a07bff242..577d9c8e2 100644 --- a/bindings/python/benches/bench_vs_release.py +++ b/bindings/python/benches/bench_vs_release.py @@ -167,8 +167,7 @@ def bench_local_side(tok, fixtures: list[dict], release_row: dict, iters: int) - def render_markdown(report: dict) -> str: lines = [ - "## Python bindings: this branch vs `tokenizers` " - f"{report['release_version']} (PyPI)", + f"## Python bindings: this branch vs `tokenizers` {report['release_version']} (PyPI)", "", f"{report['fixture_count']} fixtures (~10 KiB chunks, ≤100/fixture), median of " f"{report['iters']} runs, {report['cpus']} CPUs. Single-thread numbers aggregate " @@ -250,12 +249,18 @@ def main() -> int: release_out = Path(td) / "release.json" subprocess.run( [ - sys.executable, __file__, - "--side", "release", - "--models-json", str(models_json), - "--out", str(release_out), - "--data-dir", str(args.data_dir), - "--iters", str(args.iters), + sys.executable, + __file__, + "--side", + "release", + "--models-json", + str(models_json), + "--out", + str(release_out), + "--data-dir", + str(args.data_dir), + "--iters", + str(args.iters), ], env=os.environ | {"PYTHONPATH": str(args.release_dir)}, check=True, diff --git a/bindings/python/examples/01_train_and_encode.py b/bindings/python/examples/01_train_and_encode.py index 436a794b9..674c36d63 100644 --- a/bindings/python/examples/01_train_and_encode.py +++ b/bindings/python/examples/01_train_and_encode.py @@ -41,7 +41,7 @@ def corpus(): print(f"ids: {ids.dtype} {ids}") assert isinstance(ids, np.ndarray) and ids.dtype == np.uint32 assert tok.token_to_id("") in ids -assert [tok.id_to_token(int(i)) for i in ids[:2]] is not None +assert all(tok.id_to_token(int(i)) is not None for i in ids) # 4. Mutate a component in place: dropping the lowercasing normalizer changes ids tok_ids_lower = tok.encode("HELLO WORLD") diff --git a/bindings/python/examples/03_threading.py b/bindings/python/examples/03_threading.py index cbe5a8b74..b18211f9b 100644 --- a/bindings/python/examples/03_threading.py +++ b/bindings/python/examples/03_threading.py @@ -36,9 +36,11 @@ def encode_once(): threaded = time.perf_counter() - start speedup = sequential / threaded -print(f"{N_THREADS} encodes of {len(text) / 1e6:.1f}MB: " - f"sequential {sequential:.2f}s, {N_THREADS} threads {threaded:.2f}s " - f"({speedup:.1f}x)") +print( + f"{N_THREADS} encodes of {len(text) / 1e6:.1f}MB: " + f"sequential {sequential:.2f}s, {N_THREADS} threads {threaded:.2f}s " + f"({speedup:.1f}x)" +) assert speedup > 1.5, f"threads did not scale ({speedup:.2f}x): is the GIL held?" # 2. encode_batch: rayon parallelism inside one call, toggled by env var @@ -55,8 +57,10 @@ def encode_once(): assert all(a.tolist() == b.tolist() for a, b in zip(serial_ids, parallel_ids, strict=True)) mbps = len(text) / parallel / 1e6 -print(f"encode_batch {len(lines)} lines: serial {serial:.2f}s, " - f"rayon {parallel:.2f}s ({serial / parallel:.1f}x, {mbps:.0f} MB/s)") +print( + f"encode_batch {len(lines)} lines: serial {serial:.2f}s, " + f"rayon {parallel:.2f}s ({serial / parallel:.1f}x, {mbps:.0f} MB/s)" +) assert serial / parallel > 1.5, "rayon batch did not scale" print("OK") diff --git a/bindings/python/examples/04_train_bert_wordpiece.py b/bindings/python/examples/04_train_bert_wordpiece.py new file mode 100644 index 000000000..20a30bb32 --- /dev/null +++ b/bindings/python/examples/04_train_bert_wordpiece.py @@ -0,0 +1,50 @@ +"""Train a BERT-style WordPiece tokenizer from text files — the 1.x version of +the old `BertWordPieceTokenizer` helper, built from explicit components.""" + +import argparse +import glob +import tempfile +from pathlib import Path + +from tokenizers import Tokenizer, models, normalizers, pre_tokenizers, trainers + +DEFAULT_CORPUS = Path(__file__).resolve().parents[3] / "tokenizers" / "data" / "big.txt" + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument( + "--files", + default=str(DEFAULT_CORPUS), + metavar="path", + help="training files; '**/*.txt' patterns work if enclosed in quotes", +) +parser.add_argument("--out", default=tempfile.mkdtemp(), help="output directory") +parser.add_argument("--name", default="bert-wordpiece", help="name of the saved tokenizer file") +args = parser.parse_args() + +files = glob.glob(args.files) +assert files, f"no files match {args.files}" + +tokenizer = Tokenizer(models.WordPiece(unk_token="[UNK]")) +tokenizer.normalizer = normalizers.BertNormalizer( + clean_text=True, handle_chinese_chars=True, strip_accents=True, lowercase=True +) +tokenizer.pre_tokenizer = pre_tokenizers.BertPreTokenizer() + +tokenizer.train( + files, + trainer=trainers.WordPieceTrainer( + vocab_size=10000, + min_frequency=2, + special_tokens=["[PAD]", "[UNK]", "[CLS]", "[SEP]", "[MASK]"], + limit_alphabet=1000, + continuing_subword_prefix="##", + ), +) + +out = Path(args.out) / f"{args.name}.json" +tokenizer.save(out) +print(f"saved {out} ({tokenizer.get_vocab_size()} tokens)") + +reloaded = Tokenizer.from_file(out) +ids = reloaded.encode("Training a WordPiece tokenizer is very easy", add_special_tokens=False) +print([reloaded.id_to_token(int(i)) for i in ids]) diff --git a/bindings/python/examples/05_train_bytelevel_bpe.py b/bindings/python/examples/05_train_bytelevel_bpe.py new file mode 100644 index 000000000..0e0eec252 --- /dev/null +++ b/bindings/python/examples/05_train_bytelevel_bpe.py @@ -0,0 +1,48 @@ +"""Train a byte-level BPE tokenizer (GPT-2 style) from text files — the 1.x +version of the old `ByteLevelBPETokenizer` helper, built from explicit +components. One difference: the encode pipeline does not support +`add_prefix_space`, so it is always off.""" + +import argparse +import glob +import tempfile +from pathlib import Path + +from tokenizers import Tokenizer, models, pre_tokenizers, trainers + +DEFAULT_CORPUS = Path(__file__).resolve().parents[3] / "tokenizers" / "data" / "big.txt" + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument( + "--files", + default=str(DEFAULT_CORPUS), + metavar="path", + help="training files; '**/*.txt' patterns work if enclosed in quotes", +) +parser.add_argument("--out", default=tempfile.mkdtemp(), help="output directory") +parser.add_argument("--name", default="bpe-bytelevel", help="name of the saved tokenizer file") +args = parser.parse_args() + +files = glob.glob(args.files) +assert files, f"no files match {args.files}" + +tokenizer = Tokenizer(models.BPE()) +tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel() + +tokenizer.train( + files, + trainer=trainers.BpeTrainer( + vocab_size=10000, + min_frequency=2, + special_tokens=["", "", ""], + initial_alphabet=pre_tokenizers.ByteLevel.alphabet(), + ), +) + +out = Path(args.out) / f"{args.name}.json" +tokenizer.save(out) +print(f"saved {out} ({tokenizer.get_vocab_size()} tokens)") + +reloaded = Tokenizer.from_file(out) +ids = reloaded.encode("Training ByteLevel BPE is very easy", add_special_tokens=False) +print([reloaded.id_to_token(int(i)) for i in ids]) diff --git a/bindings/python/examples/06_train_with_datasets.py b/bindings/python/examples/06_train_with_datasets.py new file mode 100644 index 000000000..a8eaeea03 --- /dev/null +++ b/bindings/python/examples/06_train_with_datasets.py @@ -0,0 +1,27 @@ +"""Train from a Hugging Face dataset without writing it to disk — +`train_from_iterator` streams text straight into the Rust trainer, which runs +multi-threaded while the iterator is drained in 256-line gulps. + +Needs the `datasets` package; downloads wikitext-2 (~12 MB) on first run.""" + +import datasets + +from tokenizers import Tokenizer, models, normalizers, pre_tokenizers + +tokenizer = Tokenizer(models.BPE()) +tokenizer.normalizer = normalizers.Lowercase() +tokenizer.pre_tokenizer = pre_tokenizers.Whitespace() + +dataset = datasets.load_dataset("Salesforce/wikitext", "wikitext-2-raw-v1", split="train") + + +def texts(batch_size=1000): + for batch in dataset.iter(batch_size=batch_size): + yield from batch["text"] + + +tokenizer.train_from_iterator(texts()) + +print(f"trained: {tokenizer.get_vocab_size()} tokens") +ids = tokenizer.encode("the quick brown fox", add_special_tokens=False) +print([tokenizer.id_to_token(int(i)) for i in ids]) diff --git a/bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi b/bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi index 63ecde851..d42e35fe9 100644 --- a/bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi +++ b/bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi @@ -20,6 +20,13 @@ class ByteLevel(PreTokenizer): is always off. """ def __new__(cls, /, *, use_regex: bool = True) -> ByteLevel: ... + @staticmethod + def alphabet() -> list[str]: + """ + The 256 characters byte-level tokens are spelled with, one per byte + value. Pass it as a trainer's `initial_alphabet` so every byte gets a + token even if it never appears in the training data. + """ @final class CharDelimiterSplit(PreTokenizer): diff --git a/bindings/python/py_src/tokenizers/trainers/__init__.py b/bindings/python/py_src/tokenizers/trainers/__init__.py index 99cc2a3eb..054fadd54 100644 --- a/bindings/python/py_src/tokenizers/trainers/__init__.py +++ b/bindings/python/py_src/tokenizers/trainers/__init__.py @@ -4,6 +4,7 @@ Trainer = _trainers.Trainer BpeTrainer = _trainers.BpeTrainer +ParityBpeTrainer = _trainers.ParityBpeTrainer UnigramTrainer = _trainers.UnigramTrainer WordLevelTrainer = _trainers.WordLevelTrainer WordPieceTrainer = _trainers.WordPieceTrainer @@ -11,6 +12,7 @@ __all__ = [ "Trainer", "BpeTrainer", + "ParityBpeTrainer", "UnigramTrainer", "WordLevelTrainer", "WordPieceTrainer", diff --git a/bindings/python/py_src/tokenizers/trainers/__init__.pyi b/bindings/python/py_src/tokenizers/trainers/__init__.pyi index c754b2452..182e68883 100644 --- a/bindings/python/py_src/tokenizers/trainers/__init__.pyi +++ b/bindings/python/py_src/tokenizers/trainers/__init__.pyi @@ -2,9 +2,9 @@ Recipes for learning a vocabulary from text. """ -from tokenizers import AddedToken +from tokenizers import AddedToken, Tokenizer from collections.abc import Sequence -from typing import final +from typing import Any, final @final class BpeTrainer(Trainer): @@ -17,6 +17,30 @@ class BpeTrainer(Trainer): """ def __new__(cls, /, *, vocab_size: int = 30000, min_frequency: int = 0, special_tokens: Sequence[str |AddedToken] = ..., limit_alphabet: int |None = None, initial_alphabet: Sequence[str] = ..., continuing_subword_prefix: str |None = None, end_of_word_suffix: str |None = None, max_token_length: int |None = None, show_progress: bool = True) -> BpeTrainer: ... +@final +class ParityBpeTrainer: + """ + Learns a BPE vocabulary from several languages at once, keeping the + compression rate fair across them (parity-aware BPE, Foroutan et al. 2026). + Unlike the other trainers it is not passed to `Tokenizer.train`: call its + own `train_from_iterator` with one iterator of text per language. + `variant` picks how merges are selected — "base" enforces parity on every + merge, "window" relaxes it to every `window_size` merges. Fairness is + measured on the per-language `dev_iterators`, or against target + compression `ratio`s when no dev data is given. `num_merges` replaces + `vocab_size`; the remaining knobs match `BpeTrainer`. + """ + def __new__(cls, /, *, num_merges: int = 32000, variant: str = "base", min_frequency: int = 0, ratio: Sequence[float] |None = None, global_merges: int = 0, window_size: int = 100, alpha: float = 2.0, total_symbols: bool = False, special_tokens: Sequence[str |AddedToken] = ..., show_progress: bool = True, limit_alphabet: int |None = None, initial_alphabet: Sequence[str] = ..., continuing_subword_prefix: str |None = None, end_of_word_suffix: str |None = None, max_token_length: int |None = None) -> ParityBpeTrainer: ... + def __repr__(self, /) -> str: ... + def train_from_iterator(self, /, tokenizer: Tokenizer, train_iterators: Sequence[Any], *, dev_iterators: Sequence[Any] = ..., ratio: Sequence[float] |None = None) -> None: + """ + Train `tokenizer`'s vocabulary with parity-aware BPE. `train_iterators` + holds one iterator of `str` per language; `dev_iterators` (same length) + drives the fairness measurement, or pass per-language `ratio` targets + instead. The tokenizer's normalizer and pre-tokenizer are applied to + every sequence, and its model is replaced by the trained BPE. + """ + class Trainer: """ Base class for all trainers. diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml index 51ecd2ad9..7cb58e069 100644 --- a/bindings/python/pyproject.toml +++ b/bindings/python/pyproject.toml @@ -17,9 +17,18 @@ Source = "https://github.com/huggingface/tokenizers" [project.optional-dependencies] hub = ["huggingface_hub>=0.16.4"] +testing = ["pytest", "ruff"] [tool.maturin] python-source = "py_src" module-name = "tokenizers._native" bindings = "pyo3" features = ["ext-module"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +markers = ["network: needs access to the Hugging Face Hub"] + +[tool.ruff] +line-length = 119 +extend-exclude = ["py_src"] diff --git a/bindings/python/rust-toolchain b/bindings/python/rust-toolchain new file mode 100644 index 000000000..2bf5ad044 --- /dev/null +++ b/bindings/python/rust-toolchain @@ -0,0 +1 @@ +stable diff --git a/bindings/python/src/added_token.rs b/bindings/python/src/added_token.rs index 0deca0c3a..06c23d5c8 100644 --- a/bindings/python/src/added_token.rs +++ b/bindings/python/src/added_token.rs @@ -64,14 +64,17 @@ impl PyAddedToken { } fn __repr__(&self) -> String { + fn py(b: bool) -> &'static str { + if b { "True" } else { "False" } + } format!( "AddedToken({:?}, single_word={}, lstrip={}, rstrip={}, normalized={}, special={})", self.inner.content, - self.inner.single_word, - self.inner.lstrip, - self.inner.rstrip, - self.inner.normalized, - self.inner.special + py(self.inner.single_word), + py(self.inner.lstrip), + py(self.inner.rstrip), + py(self.inner.normalized), + py(self.inner.special) ) } } diff --git a/bindings/python/src/pre_tokenizers.rs b/bindings/python/src/pre_tokenizers.rs index 46f1fd1cb..01cebf531 100644 --- a/bindings/python/src/pre_tokenizers.rs +++ b/bindings/python/src/pre_tokenizers.rs @@ -154,6 +154,14 @@ impl PyByteLevel { }) .add_subclass(PyByteLevel) } + + /// The 256 characters byte-level tokens are spelled with, one per byte + /// value. Pass it as a trainer's `initial_alphabet` so every byte gets a + /// token even if it never appears in the training data. + #[staticmethod] + fn alphabet() -> Vec { + ByteLevel::alphabet().iter().map(char::to_string).collect() + } } /// Splits on one fixed character, dropping it. diff --git a/bindings/python/src/tokenizer.rs b/bindings/python/src/tokenizer.rs index 917e1a843..e9f502970 100644 --- a/bindings/python/src/tokenizer.rs +++ b/bindings/python/src/tokenizer.rs @@ -104,6 +104,108 @@ impl PyTokenizer { Ok(result) }) } + + /// Drive a parity-aware BPE run: stream one buffered iterator per language + /// into the trainer, train a fresh BPE model, and install it (plus the + /// trainer's special tokens) into the spec. Lives here because it needs + /// `BufferedPyIterator` and the lock internals; the Python-facing class is + /// `trainers::PyParityBpeTrainer`. + pub(crate) fn train_parity( + &self, + py: Python<'_>, + mut trainer: tk_train::trainers::bpe::ParityBpeTrainer, + train_iterators: Vec>, + dev_iterators: Vec>, + ) -> PyResult<()> { + let train_seqs = train_iterators + .iter() + .map(BufferedPyIterator::new) + .collect::>>()?; + let dev_seqs = dev_iterators + .iter() + .map(BufferedPyIterator::new) + .collect::>>()?; + let errors: Vec<_> = train_seqs + .iter() + .chain(&dev_seqs) + .map(|s| s.error.clone()) + .collect(); + + self.inner.with(py, |lock| { + USED_PARALLELISM.store(true, Ordering::SeqCst); + let mut guard = lock.write().map_err(poisoned)?; + let normalizer = guard.spec.get_normalizer().cloned(); + let pre_tokenizer = guard.spec.get_pre_tokenizer().cloned(); + let process = + |text: &str| pretokenize(text, normalizer.as_ref(), pre_tokenizer.as_ref()); + + for (lang, seqs) in train_seqs.into_iter().enumerate() { + trainer + .feed_language_from_iter(lang, seqs, process) + .map_err(to_pyerr)?; + } + for (lang, seqs) in dev_seqs.into_iter().enumerate() { + trainer + .feed_dev_language_from_iter(lang, seqs, process) + .map_err(to_pyerr)?; + } + + let mut model = tk_encode::models::bpe::BPE::default(); + let (special_tokens, _) = trainer.do_train(&mut model).map_err(to_pyerr)?; + guard.spec.with_model(model); + guard + .spec + .add_special_tokens(special_tokens) + .map_err(to_pyerr)?; + guard.compiled = None; + Ok::<_, PyErr>(()) + })?; + for error in errors { + if let Some(err) = error.lock().expect("error slot poisoned").take() { + return Err(err); + } + } + Ok(()) + } +} + +/// Normalize and pre-tokenize one sequence into word strings — the same +/// splitting `Tokenizer.train` applies before counting words. +fn pretokenize( + text: &str, + normalizer: Option<&tk_encode::normalizers::NormalizerWrapper>, + pre_tokenizer: Option<&tk_encode::pre_tokenizers::PreTokenizerWrapper>, +) -> tk_encode::tokenizer::Result> { + use tk_encode::tokenizer::{ + NormalizedString, Normalizer as _, OffsetReferential, OffsetType, PreTokenizedString, + PreTokenizer as _, + }; + + let normalized_text = if let Some(norm) = normalizer { + let mut normalized = NormalizedString::from(text); + norm.normalize(&mut normalized)?; + normalized.get().to_string() + } else { + text.to_string() + }; + + if let Some(pretok) = pre_tokenizer { + let mut pretokenized = PreTokenizedString::from(normalized_text.as_str()); + pretok.pre_tokenize(&mut pretokenized)?; + Ok(pretokenized + .get_splits(OffsetReferential::Original, OffsetType::Byte) + .into_iter() + .filter(|(word, _, _)| !word.is_empty()) + .map(|(word, _, _)| word.to_string()) + .collect()) + } else { + let trimmed = normalized_text.trim(); + Ok(if trimmed.is_empty() { + Vec::new() + } else { + vec![trimmed.to_string()] + }) + } } /// Get the compiled pipeline, building it from the spec on first use after a diff --git a/bindings/python/src/trainers.rs b/bindings/python/src/trainers.rs index 90bb4e83b..8c6f1aca0 100644 --- a/bindings/python/src/trainers.rs +++ b/bindings/python/src/trainers.rs @@ -1,10 +1,13 @@ +use pyo3::exceptions::PyValueError; use pyo3::prelude::*; +use tk_train::trainers::bpe::{ParityBpeTrainer, ParityVariant}; use tk_train::trainers::{ BpeTrainer, TrainerWrapper, UnigramTrainer, WordLevelTrainer, WordPieceTrainer, }; use crate::added_token::{TokenInput, parse_tokens}; use crate::error::to_pyerr; +use crate::tokenizer::PyTokenizer; /// Base class for all trainers. /// @@ -180,11 +183,163 @@ impl PyWordLevelTrainer { } } +/// Learns a BPE vocabulary from several languages at once, keeping the +/// compression rate fair across them (parity-aware BPE, Foroutan et al. 2026). +/// Unlike the other trainers it is not passed to `Tokenizer.train`: call its +/// own `train_from_iterator` with one iterator of text per language. +/// `variant` picks how merges are selected — "base" enforces parity on every +/// merge, "window" relaxes it to every `window_size` merges. Fairness is +/// measured on the per-language `dev_iterators`, or against target +/// compression `ratio`s when no dev data is given. `num_merges` replaces +/// `vocab_size`; the remaining knobs match `BpeTrainer`. +#[pyclass(frozen, name = "ParityBpeTrainer", module = "tokenizers.trainers")] +pub struct PyParityBpeTrainer { + num_merges: usize, + variant: ParityVariant, + min_frequency: u64, + ratio: Option>, + global_merges: usize, + window_size: usize, + alpha: f64, + total_symbols: bool, + special_tokens: Vec, + show_progress: bool, + limit_alphabet: Option, + initial_alphabet: Vec, + continuing_subword_prefix: Option, + end_of_word_suffix: Option, + max_token_length: Option, +} + +fn parse_variant(variant: &str) -> PyResult { + match variant { + "base" => Ok(ParityVariant::Base), + "window" => Ok(ParityVariant::Window), + _ => Err(PyValueError::new_err(format!( + "unknown variant {variant:?}: use \"base\" or \"window\"" + ))), + } +} + +#[pymethods] +impl PyParityBpeTrainer { + #[new] + #[pyo3(signature = (*, num_merges = 32000, variant = "base", min_frequency = 0, ratio = None, global_merges = 0, window_size = 100, alpha = 2.0, total_symbols = false, special_tokens = vec![], show_progress = true, limit_alphabet = None, initial_alphabet = vec![], continuing_subword_prefix = None, end_of_word_suffix = None, max_token_length = None))] + #[allow(clippy::too_many_arguments)] + fn new( + num_merges: usize, + variant: &str, + min_frequency: u64, + ratio: Option>, + global_merges: usize, + window_size: usize, + alpha: f64, + total_symbols: bool, + special_tokens: Vec, + show_progress: bool, + limit_alphabet: Option, + initial_alphabet: Vec, + continuing_subword_prefix: Option, + end_of_word_suffix: Option, + max_token_length: Option, + ) -> PyResult { + Ok(Self { + num_merges, + variant: parse_variant(variant)?, + min_frequency, + ratio, + global_merges, + window_size, + alpha, + total_symbols, + special_tokens: parse_tokens(special_tokens, true), + show_progress, + limit_alphabet, + initial_alphabet, + continuing_subword_prefix, + end_of_word_suffix, + max_token_length, + }) + } + + /// Train `tokenizer`'s vocabulary with parity-aware BPE. `train_iterators` + /// holds one iterator of `str` per language; `dev_iterators` (same length) + /// drives the fairness measurement, or pass per-language `ratio` targets + /// instead. The tokenizer's normalizer and pre-tokenizer are applied to + /// every sequence, and its model is replaced by the trained BPE. + #[pyo3(signature = (tokenizer, train_iterators, *, dev_iterators = vec![], ratio = None))] + fn train_from_iterator( + &self, + py: Python<'_>, + tokenizer: PyRef<'_, PyTokenizer>, + train_iterators: Vec>, + dev_iterators: Vec>, + ratio: Option>, + ) -> PyResult<()> { + if train_iterators.is_empty() { + return Err(PyValueError::new_err("train_iterators must not be empty")); + } + if !dev_iterators.is_empty() && dev_iterators.len() != train_iterators.len() { + return Err(PyValueError::new_err(format!( + "dev_iterators length ({}) must match train_iterators length ({})", + dev_iterators.len(), + train_iterators.len() + ))); + } + + let mut builder = ParityBpeTrainer::builder() + .num_merges(self.num_merges) + .variant(self.variant) + .min_frequency(self.min_frequency) + .global_merges(self.global_merges) + .window_size(self.window_size) + .alpha(self.alpha) + .total_symbols(self.total_symbols) + .special_tokens(self.special_tokens.clone()) + .show_progress(self.show_progress) + .max_token_length(self.max_token_length); + if let Some(limit) = self.limit_alphabet { + builder = builder.limit_alphabet(limit); + } + if !self.initial_alphabet.is_empty() { + builder = builder.initial_alphabet(self.initial_alphabet.iter().copied().collect()); + } + if let Some(prefix) = &self.continuing_subword_prefix { + builder = builder.continuing_subword_prefix(prefix.clone()); + } + if let Some(suffix) = &self.end_of_word_suffix { + builder = builder.end_of_word_suffix(suffix.clone()); + } + // Dev data beats ratio targets; a call-site ratio beats the configured one. + if dev_iterators.is_empty() + && let Some(r) = ratio.or_else(|| self.ratio.clone()) + { + builder = builder.ratio(r); + } + + tokenizer.train_parity(py, builder.build(), train_iterators, dev_iterators) + } + + fn __repr__(&self) -> String { + format!( + "ParityBpeTrainer(num_merges={}, variant=\"{}\", alpha={}, window_size={})", + self.num_merges, + match self.variant { + ParityVariant::Base => "base", + ParityVariant::Window => "window", + }, + self.alpha, + self.window_size + ) + } +} + /// Recipes for learning a vocabulary from text. #[pymodule(gil_used = false)] pub mod trainers { #[pymodule_export] pub use super::{ - PyBpeTrainer, PyTrainer, PyUnigramTrainer, PyWordLevelTrainer, PyWordPieceTrainer, + PyBpeTrainer, PyParityBpeTrainer, PyTrainer, PyUnigramTrainer, PyWordLevelTrainer, + PyWordPieceTrainer, }; } diff --git a/bindings/python/tests/__init__.py b/bindings/python/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/bindings/python/tests/conftest.py b/bindings/python/tests/conftest.py new file mode 100644 index 000000000..ce515e14c --- /dev/null +++ b/bindings/python/tests/conftest.py @@ -0,0 +1,56 @@ +from pathlib import Path + +import pytest + +from tokenizers import Tokenizer, models, pre_tokenizers, trainers + +DATA = Path(__file__).resolve().parents[3] / "tokenizers" / "data" + +SENTENCES = [ + "the quick brown fox jumps over the lazy dog", + "pack my box with five dozen liquor jugs", + "how vexingly quick daft zebras jump", + "the five boxing wizards jump quickly", +] * 8 + + +def data_file(name: str) -> Path: + path = DATA / name + if not path.is_file(): + pytest.skip(f"{path} missing — run `make -C ../../tokenizers fixtures bench-models data/big.txt`") + return path + + +@pytest.fixture(scope="session") +def corpus(): + text = data_file("big.txt").read_text(encoding="utf-8")[:200_000] + return [line for line in text.splitlines() if line.strip()] + + +@pytest.fixture(scope="session") +def gpt2_file(): + return data_file("gpt2.json") + + +@pytest.fixture(scope="session") +def bert_file(): + return data_file("bert-base-uncased.json") + + +@pytest.fixture(scope="session") +def t5_file(): + return data_file("t5-base.json") + + +def train_word_tokenizer() -> Tokenizer: + tok = Tokenizer(models.WordLevel(unk_token="[UNK]")) + tok.pre_tokenizer = pre_tokenizers.Whitespace() + tok.train_from_iterator(SENTENCES, trainer=trainers.WordLevelTrainer(special_tokens=["[UNK]"])) + return tok + + +@pytest.fixture(scope="session") +def word_tokenizer(): + """A small trained tokenizer shared by read-only tests. Tests that mutate + build their own with `train_word_tokenizer()`.""" + return train_word_tokenizer() diff --git a/bindings/python/tests/test_added_token.py b/bindings/python/tests/test_added_token.py new file mode 100644 index 000000000..aecfb51ed --- /dev/null +++ b/bindings/python/tests/test_added_token.py @@ -0,0 +1,30 @@ +from tokenizers import AddedToken + + +def test_defaults(): + token = AddedToken("") + assert token.content == "" + assert token.single_word is False + assert token.lstrip is False + assert token.rstrip is False + assert token.special is False + assert token.normalized is True + + +def test_special_flips_normalized_default(): + assert AddedToken("", special=True).normalized is False + assert AddedToken("", special=True, normalized=True).normalized is True + + +def test_flags(): + token = AddedToken("", single_word=True, lstrip=True, rstrip=True) + assert token.single_word is True + assert token.lstrip is True + assert token.rstrip is True + + +def test_repr(): + assert ( + repr(AddedToken("")) + == 'AddedToken("", single_word=False, lstrip=False, rstrip=False, normalized=True, special=False)' + ) diff --git a/bindings/python/tests/test_components.py b/bindings/python/tests/test_components.py new file mode 100644 index 000000000..01a6e137e --- /dev/null +++ b/bindings/python/tests/test_components.py @@ -0,0 +1,97 @@ +import numpy as np +import pytest + +from tokenizers import Tokenizer, models, normalizers, pre_tokenizers, trainers + +from .conftest import train_word_tokenizer + + +def test_all_models_construct_and_assign(): + for model in [ + models.BPE(), + models.BPE(unk_token="", dropout=0.1, byte_fallback=True), + models.WordPiece(unk_token="[UNK]", continuing_subword_prefix="##"), + models.WordLevel(unk_token="[UNK]"), + models.Unigram(), + ]: + tok = Tokenizer(model) + assert type(model).__name__ in repr(tok.model) + + +def test_all_normalizers_construct(): + for normalizer in [ + normalizers.NFC(), + normalizers.NFD(), + normalizers.NFKC(), + normalizers.NFKD(), + normalizers.Lowercase(), + normalizers.StripAccents(), + normalizers.Strip(), + normalizers.Replace("a", "b"), + normalizers.Prepend("_"), + normalizers.BertNormalizer(strip_accents=True, lowercase=False), + normalizers.Sequence([normalizers.NFD(), normalizers.Lowercase()]), + ]: + tok = Tokenizer(models.BPE()) + tok.normalizer = normalizer + assert isinstance(tok.normalizer, normalizers.Normalizer) + + +def test_all_pre_tokenizers_construct(): + for pre_tokenizer in [ + pre_tokenizers.Whitespace(), + pre_tokenizers.WhitespaceSplit(), + pre_tokenizers.BertPreTokenizer(), + pre_tokenizers.UnicodeScripts(), + pre_tokenizers.ByteLevel(), + pre_tokenizers.ByteLevel(use_regex=False), + pre_tokenizers.CharDelimiterSplit(","), + pre_tokenizers.Digits(individual_digits=True), + pre_tokenizers.FixedLength(length=4), + pre_tokenizers.Punctuation(), + pre_tokenizers.Split(" ", behavior="removed"), + pre_tokenizers.Sequence([pre_tokenizers.Whitespace(), pre_tokenizers.Digits()]), + ]: + tok = Tokenizer(models.BPE()) + tok.pre_tokenizer = pre_tokenizer + assert isinstance(tok.pre_tokenizer, pre_tokenizers.PreTokenizer) + + +def test_byte_level_alphabet(): + alphabet = pre_tokenizers.ByteLevel.alphabet() + assert len(alphabet) == 256 + assert len(set(alphabet)) == 256 + assert all(len(c) == 1 for c in alphabet) + + +def test_lowercase_normalizer_changes_ids(): + tok = train_word_tokenizer() + assert tok.token_to_id("THE") is None + before = tok.encode("THE", add_special_tokens=False) + assert [tok.id_to_token(int(i)) for i in before] == ["[UNK]"] + + tok.normalizer = normalizers.Lowercase() + after = tok.encode("THE", add_special_tokens=False) + assert [tok.id_to_token(int(i)) for i in after] == ["the"] + + +def test_char_delimiter_split_effect(): + tok = Tokenizer(models.WordLevel(unk_token="[UNK]")) + tok.pre_tokenizer = pre_tokenizers.CharDelimiterSplit(",") + tok.train_from_iterator(["a,b", "b,c"], trainer=trainers.WordLevelTrainer(special_tokens=["[UNK]"])) + ids = tok.encode("a,c", add_special_tokens=False) + assert [tok.id_to_token(int(i)) for i in ids] == ["a", "c"] + + +def test_component_assignment_invalidates_pipeline(): + tok = train_word_tokenizer() + the_id = tok.encode("the", add_special_tokens=False) + tok.pre_tokenizer = pre_tokenizers.FixedLength(length=1) + per_char = tok.encode("the", add_special_tokens=False) + assert len(per_char) == 3 + assert not np.array_equal(the_id, per_char) + + +def test_split_rejects_bad_behavior(): + with pytest.raises(Exception): + pre_tokenizers.Split(" ", behavior="not-a-behavior") diff --git a/bindings/python/tests/test_parity_trainer.py b/bindings/python/tests/test_parity_trainer.py new file mode 100644 index 000000000..25717e68e --- /dev/null +++ b/bindings/python/tests/test_parity_trainer.py @@ -0,0 +1,70 @@ +import pytest + +from tokenizers import Tokenizer, models, pre_tokenizers, trainers + +EN = ["the cat sat on the mat", "the dog ate the food", "a cat and a dog"] * 6 +DE = ["die katze sitzt auf der matte", "der hund frisst das futter", "eine katze und ein hund"] * 6 + + +def fresh(): + tok = Tokenizer(models.BPE()) + tok.pre_tokenizer = pre_tokenizers.Whitespace() + return tok + + +def test_trains_with_dev_iterators(): + tok = fresh() + trainer = trainers.ParityBpeTrainer(num_merges=30, special_tokens=[""], show_progress=False) + trainer.train_from_iterator(tok, [iter(EN), iter(DE)], dev_iterators=[iter(EN[:6]), iter(DE[:6])]) + assert tok.token_to_id("") is not None + for line in (EN[0], DE[0]): + ids = tok.encode(line, add_special_tokens=False) + assert len(ids) > 0 + assert all(tok.id_to_token(int(i)) is not None for i in ids) + + +def test_trains_with_ratio_targets(): + tok = fresh() + trainer = trainers.ParityBpeTrainer(num_merges=20, show_progress=False) + trainer.train_from_iterator(tok, [iter(EN), iter(DE)], ratio=[1.0, 1.0]) + assert tok.get_vocab_size() > 0 + + +def test_window_variant(): + tok = fresh() + trainer = trainers.ParityBpeTrainer(num_merges=20, variant="window", window_size=5, show_progress=False) + trainer.train_from_iterator(tok, [iter(EN), iter(DE)], dev_iterators=[iter(EN[:6]), iter(DE[:6])]) + assert tok.get_vocab_size() > 0 + + +def test_rejects_unknown_variant(): + with pytest.raises(ValueError, match="variant"): + trainers.ParityBpeTrainer(variant="strict") + + +def test_rejects_empty_train_iterators(): + trainer = trainers.ParityBpeTrainer(num_merges=10, show_progress=False) + with pytest.raises(ValueError, match="must not be empty"): + trainer.train_from_iterator(fresh(), []) + + +def test_rejects_mismatched_dev_length(): + trainer = trainers.ParityBpeTrainer(num_merges=10, show_progress=False) + with pytest.raises(ValueError, match="must match"): + trainer.train_from_iterator(fresh(), [iter(EN), iter(DE)], dev_iterators=[iter(EN)]) + + +def test_iterator_error_propagates(): + def broken(): + yield "fine" + raise RuntimeError("boom") + + trainer = trainers.ParityBpeTrainer(num_merges=10, show_progress=False) + with pytest.raises(RuntimeError, match="boom"): + trainer.train_from_iterator(fresh(), [broken(), iter(DE)], ratio=[1.0, 1.0]) + + +def test_repr(): + trainer = trainers.ParityBpeTrainer(num_merges=100, variant="window") + assert "num_merges=100" in repr(trainer) + assert 'variant="window"' in repr(trainer) diff --git a/bindings/python/tests/test_pretrained.py b/bindings/python/tests/test_pretrained.py new file mode 100644 index 000000000..ebf0d4ac6 --- /dev/null +++ b/bindings/python/tests/test_pretrained.py @@ -0,0 +1,32 @@ +import numpy as np +import pytest + +from tokenizers import Tokenizer, TokenizersError + + +def test_gpt2_encodes_corpus(gpt2_file, corpus): + tok = Tokenizer.from_file(gpt2_file) + batch = tok.encode_batch(corpus, add_special_tokens=False) + assert sum(len(ids) for ids in batch) > 0 + assert all(ids.dtype == np.uint32 for ids in batch) + + +def test_bert_special_tokens_gate(bert_file): + tok = Tokenizer.from_file(bert_file) + with pytest.raises(NotImplementedError, match="post-process"): + tok.encode("hello") + ids = tok.encode("hello", add_special_tokens=False) + assert len(ids) > 0 + + +def test_metaspace_fails_loudly_at_compile(t5_file): + tok = Tokenizer.from_file(t5_file) + with pytest.raises(TokenizersError, match="Metaspace"): + tok.encode("hello", add_special_tokens=False) + + +@pytest.mark.network +def test_from_pretrained(): + tok = Tokenizer.from_pretrained("bert-base-uncased") + ids = tok.encode("hello world", add_special_tokens=False) + assert [tok.id_to_token(int(i)) for i in ids] == ["hello", "world"] diff --git a/bindings/python/tests/test_threading.py b/bindings/python/tests/test_threading.py new file mode 100644 index 000000000..ed0a1d100 --- /dev/null +++ b/bindings/python/tests/test_threading.py @@ -0,0 +1,44 @@ +import concurrent.futures +import os + +import numpy as np + +from .conftest import SENTENCES, train_word_tokenizer + + +def test_concurrent_encode_matches_serial(): + tok = train_word_tokenizer() + expected = [tok.encode(line, add_special_tokens=False) for line in SENTENCES] + with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: + results = list(pool.map(lambda s: tok.encode(s, add_special_tokens=False), SENTENCES)) + for got, want in zip(results, expected): + assert np.array_equal(got, want) + + +def test_parallel_encode_batch_matches_serial(): + tok = train_word_tokenizer() + os.environ["TOKENIZERS_PARALLELISM"] = "false" + serial = tok.encode_batch(SENTENCES * 32, add_special_tokens=False) + os.environ["TOKENIZERS_PARALLELISM"] = "true" + try: + parallel = tok.encode_batch(SENTENCES * 32, add_special_tokens=False) + finally: + del os.environ["TOKENIZERS_PARALLELISM"] + for got, want in zip(parallel, serial): + assert np.array_equal(got, want) + + +def test_concurrent_mutation_and_encode_is_safe(): + tok = train_word_tokenizer() + + def encode_some(_): + return tok.encode_batch(SENTENCES, add_special_tokens=False) + + def add_some(i): + return tok.add_tokens([f""]) + + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool: + encoded = pool.map(encode_some, range(8)) + added = pool.map(add_some, range(8)) + list(encoded) + assert sum(added) == 8 diff --git a/bindings/python/tests/test_tokenizer.py b/bindings/python/tests/test_tokenizer.py new file mode 100644 index 000000000..ed3bbd286 --- /dev/null +++ b/bindings/python/tests/test_tokenizer.py @@ -0,0 +1,117 @@ +import pickle + +import numpy as np +import pytest + +from tokenizers import AddedToken, Tokenizer, models + +from .conftest import SENTENCES, train_word_tokenizer + + +def test_encode_returns_uint32_array(word_tokenizer): + ids = word_tokenizer.encode(SENTENCES[0], add_special_tokens=False) + assert isinstance(ids, np.ndarray) + assert ids.dtype == np.uint32 + words = [word_tokenizer.id_to_token(int(i)) for i in ids] + assert words == SENTENCES[0].split() + + +def test_encode_batch_matches_encode(word_tokenizer): + batch = word_tokenizer.encode_batch(SENTENCES, add_special_tokens=False) + assert len(batch) == len(SENTENCES) + for line, ids in zip(SENTENCES, batch): + single = word_tokenizer.encode(line, add_special_tokens=False) + assert np.array_equal(ids, single) + + +def test_unknown_words_map_to_unk(word_tokenizer): + ids = word_tokenizer.encode("supercalifragilistic", add_special_tokens=False) + assert [word_tokenizer.id_to_token(int(i)) for i in ids] == ["[UNK]"] + + +def test_vocab_and_lookups(word_tokenizer): + vocab = word_tokenizer.get_vocab() + assert len(vocab) == word_tokenizer.get_vocab_size() + token, id_ = next(iter(vocab.items())) + assert word_tokenizer.token_to_id(token) == id_ + assert word_tokenizer.id_to_token(id_) == token + assert word_tokenizer.token_to_id("definitely-not-in-vocab") is None + + +def test_add_tokens_and_encode_them(): + tok = train_word_tokenizer() + assert tok.add_tokens(["procrastination"]) == 1 + assert tok.add_tokens(["procrastination"]) == 0 + ids = tok.encode("the procrastination", add_special_tokens=False) + assert [tok.id_to_token(int(i)) for i in ids] == ["the", "procrastination"] + + +def test_add_special_tokens_marks_special(): + tok = train_word_tokenizer() + assert tok.add_special_tokens([""]) == 1 + ids = tok.encode("the ", add_special_tokens=False) + assert [tok.id_to_token(int(i)) for i in ids] == ["the", ""] + + +def test_add_tokens_accepts_added_token(): + tok = train_word_tokenizer() + assert tok.add_tokens([AddedToken("", single_word=True)]) == 1 + assert tok.token_to_id("") is not None + + +def test_save_and_from_file_round_trip(word_tokenizer, tmp_path): + path = tmp_path / "tokenizer.json" + word_tokenizer.save(path) + reloaded = Tokenizer.from_file(path) + for line in SENTENCES[:4]: + assert np.array_equal( + reloaded.encode(line, add_special_tokens=False), + word_tokenizer.encode(line, add_special_tokens=False), + ) + + +def test_to_str_and_from_buffer_round_trip(word_tokenizer): + reloaded = Tokenizer.from_buffer(word_tokenizer.to_str().encode()) + assert np.array_equal( + reloaded.encode(SENTENCES[0], add_special_tokens=False), + word_tokenizer.encode(SENTENCES[0], add_special_tokens=False), + ) + + +def test_pickle_round_trip(word_tokenizer): + reloaded = pickle.loads(pickle.dumps(word_tokenizer)) + assert np.array_equal( + reloaded.encode(SENTENCES[0], add_special_tokens=False), + word_tokenizer.encode(SENTENCES[0], add_special_tokens=False), + ) + + +def test_repr_names_model(word_tokenizer): + assert "WordLevel" in repr(word_tokenizer) + + +def test_decode_raises(): + tok = Tokenizer(models.BPE()) + with pytest.raises(NotImplementedError): + tok.decode([1, 2, 3]) + + +def test_model_getter_returns_typed_copy(word_tokenizer): + assert isinstance(word_tokenizer.model, models.WordLevel) + + +def test_train_iterator_error_propagates(): + tok = Tokenizer(models.WordLevel(unk_token="[UNK]")) + + def broken(): + yield "fine" + raise ValueError("boom") + + with pytest.raises(ValueError, match="boom"): + tok.train_from_iterator(broken()) + + +def test_train_iterator_rejects_non_str(): + tok = Tokenizer(models.WordLevel(unk_token="[UNK]")) + with pytest.raises(TypeError): + tok.train_from_iterator([b"bytes are not str"]) diff --git a/bindings/python/tests/test_trainers.py b/bindings/python/tests/test_trainers.py new file mode 100644 index 000000000..1626d4986 --- /dev/null +++ b/bindings/python/tests/test_trainers.py @@ -0,0 +1,68 @@ +import pytest + +from tokenizers import Tokenizer, models, pre_tokenizers, trainers + +from .conftest import SENTENCES + + +def fresh(model): + tok = Tokenizer(model) + tok.pre_tokenizer = pre_tokenizers.Whitespace() + return tok + + +@pytest.mark.parametrize( + ("model", "trainer"), + [ + (models.BPE(), trainers.BpeTrainer(vocab_size=120, special_tokens=[""], show_progress=False)), + ( + models.WordPiece(unk_token="[UNK]"), + trainers.WordPieceTrainer(vocab_size=120, special_tokens=["[UNK]"], show_progress=False), + ), + ( + models.WordLevel(unk_token="[UNK]"), + trainers.WordLevelTrainer(special_tokens=["[UNK]"], show_progress=False), + ), + ( + models.Unigram(), + trainers.UnigramTrainer(vocab_size=64, special_tokens=[""], unk_token="", show_progress=False), + ), + ], + ids=["bpe", "wordpiece", "wordlevel", "unigram"], +) +def test_each_trainer_trains(model, trainer): + tok = fresh(model) + tok.train_from_iterator(SENTENCES, trainer=trainer) + assert tok.get_vocab_size() > 0 + ids = tok.encode(SENTENCES[0], add_special_tokens=False) + assert len(ids) > 0 + + +def test_special_tokens_get_first_ids(): + tok = fresh(models.BPE()) + tok.train_from_iterator( + SENTENCES, + trainer=trainers.BpeTrainer(vocab_size=120, special_tokens=["", "", ""], show_progress=False), + ) + assert tok.token_to_id("") == 0 + assert tok.token_to_id("") == 1 + assert tok.token_to_id("") == 2 + + +def test_initial_alphabet_is_forced_in(): + tok = fresh(models.BPE()) + tok.train_from_iterator( + SENTENCES, + trainer=trainers.BpeTrainer(vocab_size=120, initial_alphabet=["£"], show_progress=False), + ) + assert tok.token_to_id("£") is not None + + +def test_default_trainer_used_when_none_given(): + tok = fresh(models.WordLevel(unk_token="[UNK]")) + tok.train_from_iterator(SENTENCES) + assert tok.get_vocab_size() > 0 + + +def test_trainer_repr(): + assert "BpeTrainer" in repr(trainers.BpeTrainer()) From 87a560820b87448439d76e54190550f728922f14 Mon Sep 17 00:00:00 2001 From: SBrandeis <33657802+SBrandeis@users.noreply.github.com> Date: Wed, 22 Jul 2026 14:33:40 +0200 Subject: [PATCH 09/19] macos linker config --- bindings/python/.cargo/config.toml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 bindings/python/.cargo/config.toml diff --git a/bindings/python/.cargo/config.toml b/bindings/python/.cargo/config.toml new file mode 100644 index 000000000..fbfc5de16 --- /dev/null +++ b/bindings/python/.cargo/config.toml @@ -0,0 +1,12 @@ +# Required flags on MacOS to defer resolution of the CPython symbols +[target.x86_64-apple-darwin] +rustflags = [ + "-C", "link-arg=-undefined", + "-C", "link-arg=dynamic_lookup", +] + +[target.aarch64-apple-darwin] +rustflags = [ + "-C", "link-arg=-undefined", + "-C", "link-arg=dynamic_lookup", +] From e88445b664cda93e11d61112a370650d74f4782a Mon Sep 17 00:00:00 2001 From: SBrandeis <33657802+SBrandeis@users.noreply.github.com> Date: Wed, 22 Jul 2026 15:05:58 +0200 Subject: [PATCH 10/19] async + free-threaded python --- .github/workflows/python-release.yml | 34 ++++++++++++--- .github/workflows/python.yml | 11 ++--- bindings/python/.gitignore | 1 + bindings/python/CHANGELOG.md | 7 +++- bindings/python/Cargo.toml | 8 +++- bindings/python/README.md | 9 ++-- .../python/py_src/tokenizers/__init__.pyi | 15 +++++++ bindings/python/src/tokenizer.rs | 41 +++++++++++++++++++ bindings/python/tests/test_async.py | 41 +++++++++++++++++++ bindings/python/tools/stub-gen/src/main.rs | 6 +++ 10 files changed, 155 insertions(+), 18 deletions(-) create mode 100644 bindings/python/tests/test_async.py diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml index c857ecda9..e03ffb696 100644 --- a/.github/workflows/python-release.yml +++ b/.github/workflows/python-release.yml @@ -1,8 +1,8 @@ name: Python Release -# All wheels are abi3-py310: one binary per platform covers CPython -# 3.10–3.14. No free-threaded (3.14t) wheels — the 1.x bindings are -# abi3-only. +# Default wheels are abi3-py310: one binary per platform covers CPython +# 3.10–3.14. Free-threaded 3.14t gets its own interpreter-specific wheels +# (flavor: ft), built without the default `abi3` cargo feature. on: push: tags: @@ -23,7 +23,7 @@ jobs: working-directory: ./bindings/python build: - name: build on ${{ matrix.platform || matrix.os }} (${{ matrix.target }} - ${{ matrix.manylinux || 'auto' }} - ${{ matrix.interpreter || '3.14' }}) + name: build on ${{ matrix.platform || matrix.os }} (${{ matrix.target }} - ${{ matrix.manylinux || 'auto' }} - ${{ matrix.flavor == 'ft' && '3.14t' || matrix.interpreter || '3.14' }}) # only run on push to main and on release needs: [lock_exists] if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'Full Build') @@ -33,6 +33,10 @@ jobs: os: [ubuntu, macos, windows] target: [x86_64, aarch64] manylinux: [auto] + # `flavor` discriminates abi3 wheels from free-threaded (3.14t, + # non-abi3) wheels so the ft include: entries below create new + # matrix cells instead of merging with the abi3 combos. + flavor: [abi3] include: - os: ubuntu platform: linux @@ -92,6 +96,20 @@ jobs: platform: linux target: s390x interpreter: "3.14" + + # --- Free-threaded Python 3.14t wheels ------------------------- + # `flavor: ft` drops the default `abi3` cargo feature (see the + # maturin args below), producing interpreter-specific cp314t + # wheels. linux container builds get 3.14t from the docker image, + # so `python-install` is unset there; macOS/windows host builds + # need it so setup-python actually installs 3.14t. + - { os: ubuntu, platform: linux, target: x86_64, manylinux: auto, flavor: ft } + - { os: ubuntu, platform: linux, target: aarch64, manylinux: auto, flavor: ft } + - { os: ubuntu, platform: linux, target: x86_64, manylinux: musllinux_1_1, flavor: ft } + - { os: ubuntu, platform: linux, target: aarch64, manylinux: musllinux_1_1, flavor: ft } + - { os: macos, target: x86_64, manylinux: auto, flavor: ft, python-install: "3.14t" } + - { os: macos, target: aarch64, manylinux: auto, flavor: ft, python-install: "3.14t" } + - { os: windows, ls: dir, target: x86_64, manylinux: auto, python-architecture: x64, python-install: "3.14t", flavor: ft } exclude: - os: windows target: aarch64 @@ -126,9 +144,13 @@ jobs: working-directory: ./bindings/python manylinux: ${{ matrix.manylinux || 'auto' }} container: ${{ matrix.container }} + # `flavor=ft` drops the default `abi3` cargo feature so the wheel + # is non-abi3 (free-threaded Python can't load limited-API + # extensions). `flavor=abi3` builds use defaults. args: >- --release --out dist - --interpreter ${{ matrix.interpreter || '3.14' }} + --interpreter ${{ matrix.flavor == 'ft' && '3.14t' || matrix.interpreter || '3.14' }} + ${{ matrix.flavor == 'ft' && '--no-default-features' || '' }} rust-toolchain: stable sccache: false docker-options: -e CI @@ -141,7 +163,7 @@ jobs: - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: pypi_files-${{ matrix.os }}-${{ matrix.target }}-${{ matrix.manylinux }} + name: pypi_files-${{ matrix.os }}-${{ matrix.target }}-${{ matrix.manylinux }}-${{ matrix.flavor || 'abi3' }} path: ./bindings/python/dist build-sdist: name: build sdist diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 1076d61be..102a7ec6e 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -18,10 +18,10 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest] - # The cdylib is built with `abi3-py310`, so a single binary covers - # 3.10–3.14; test the floor and the newest. Free-threaded 3.14t is - # not supported (abi3 extensions cannot load there). - python: ["3.10", "3.14"] + # The default build is abi3-py310 (one binary covers 3.10–3.14): + # test the floor and the newest. Free-threaded 3.14t can't load + # abi3 extensions, so it builds without the default `abi3` feature. + python: ["3.10", "3.14", "3.14t"] steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -44,7 +44,8 @@ jobs: run: | uv venv .venv --python ${{ matrix.python }} uv pip install --python .venv/bin/python maturin numpy pytest datasets - source .venv/bin/activate && maturin develop --release + source .venv/bin/activate && maturin develop --release \ + ${{ matrix.python == '3.14t' && '--no-default-features' || '' }} # The examples read real tokenizer.json files + a text corpus. Cached # under a key of this workflow's own (not shared with pipeline-bench, diff --git a/bindings/python/.gitignore b/bindings/python/.gitignore index 9cbe9d5c8..80473e575 100644 --- a/bindings/python/.gitignore +++ b/bindings/python/.gitignore @@ -1,4 +1,5 @@ .venv/ +.venv-ft/ .release/ python_bench.json python_bench.md diff --git a/bindings/python/CHANGELOG.md b/bindings/python/CHANGELOG.md index ce8a79b2b..2047103c7 100644 --- a/bindings/python/CHANGELOG.md +++ b/bindings/python/CHANGELOG.md @@ -22,8 +22,11 @@ Breaking changes: `Metaspace` pre-tokenizer. - Custom Python components are not supported; components are plain values. - `decoders`, `processors`, and the `implementations` helpers are gone. -- Wheels are abi3-py310 only; free-threaded interpreters (3.14t) are not - supported. + +Kept from 0.x: `async_encode`/`async_encode_batch` (now a thin +`asyncio.to_thread` wrapper — no tokio runtime), parity-aware BPE training +(`trainers.ParityBpeTrainer`), and free-threaded Python support (default +wheels are abi3-py310; 3.14t ships non-abi3 `cp314t` wheels). ## [0.13.2] diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index eb9552fc3..b38039c83 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -10,7 +10,7 @@ name = "_native" crate-type = ["cdylib", "rlib"] [dependencies] -pyo3 = { version = "=0.29", features = ["abi3-py310", "experimental-inspect"] } +pyo3 = { version = "=0.29", features = ["experimental-inspect"] } numpy = "0.29" rayon = "1.10" serde = "1.0" @@ -21,6 +21,10 @@ tk-train = { path = "../../tokenizers/tk-train", features = ["parity-aware-bpe"] [target.'cfg(unix)'.dependencies] libc = "0.2" +# abi3 is on by default: one wheel per platform covers CPython 3.10–3.14. +# Free-threaded interpreters (3.14t) cannot load limited-API extensions, so +# their wheels build with `--no-default-features` (interpreter-specific tag). [features] -default = [] +default = ["abi3"] +abi3 = ["pyo3/abi3-py310"] ext-module = ["pyo3/extension-module"] diff --git a/bindings/python/README.md b/bindings/python/README.md index efa25afe1..74e8433f1 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -39,9 +39,12 @@ tok.save("tokenizer.json") you subclass. - `decoders`, `processors`, and the `implementations` helpers (`BertWordPieceTokenizer`, …) are gone. -- The wheel is abi3 (one binary for CPython 3.10–3.14); free-threaded - interpreters (3.14t) are not supported yet. Multi-threaded encode does not - need them — the GIL is released. + +Still here: `async_encode`/`async_encode_batch` (awaitable, run in a worker +thread — encode releases the GIL, so a tokio runtime is no longer needed), +parity-aware BPE training, and free-threaded Python — default wheels are +abi3 (one binary for CPython 3.10–3.14), and 3.14t gets its own non-abi3 +wheels (`maturin build --no-default-features`). ## Build and use locally diff --git a/bindings/python/py_src/tokenizers/__init__.pyi b/bindings/python/py_src/tokenizers/__init__.pyi index 71fd16c91..5498172af 100644 --- a/bindings/python/py_src/tokenizers/__init__.pyi +++ b/bindings/python/py_src/tokenizers/__init__.pyi @@ -1,3 +1,6 @@ +from collections.abc import Coroutine +from typing import Any + import numpy as np import numpy.typing as npt @@ -70,6 +73,18 @@ class Tokenizer: on. Plain strings match with default options; pass `AddedToken` to control matching. Returns how many were actually new. """ + def async_encode(self, /, text: Any, *, add_special_tokens: bool = True) -> "Coroutine[Any, Any, npt.NDArray[np.uint32]]": + """ + Awaitable `encode`: same arguments and result, run in a worker thread + (`asyncio.to_thread`) so the event loop stays free. The thread releases + the interpreter lock while Rust encodes, so encodes genuinely overlap. + """ + def async_encode_batch(self, /, texts: Any, *, add_special_tokens: bool = True) -> "Coroutine[Any, Any, list[npt.NDArray[np.uint32]]]": + """ + Awaitable `encode_batch`: same arguments and result, run in a worker + thread (`asyncio.to_thread`) so the event loop stays free while the + batch encodes on Rust threads. + """ def decode(self, /, ids: Sequence[int], *, skip_special_tokens: bool = True) -> str: """ Not implemented yet: decoding is not part of the encode pipeline. diff --git a/bindings/python/src/tokenizer.rs b/bindings/python/src/tokenizer.rs index e9f502970..838d82a70 100644 --- a/bindings/python/src/tokenizer.rs +++ b/bindings/python/src/tokenizer.rs @@ -169,6 +169,23 @@ impl PyTokenizer { } } +/// Wrap a bound method call in `asyncio.to_thread`, returning the coroutine. +/// The whole legacy async surface (a tokio runtime + pyo3-async-runtimes) is +/// unnecessary here because encode already releases the interpreter lock — +/// a worker thread is all it takes to keep the event loop responsive. +fn to_thread<'py>( + slf: &Bound<'py, PyTokenizer>, + method: &str, + input: &Bound<'py, PyAny>, + add_special_tokens: bool, +) -> PyResult> { + use pyo3::types::IntoPyDict; + let py = slf.py(); + let kwargs = [("add_special_tokens", add_special_tokens)].into_py_dict(py)?; + py.import("asyncio")? + .call_method("to_thread", (slf.getattr(method)?, input), Some(&kwargs)) +} + /// Normalize and pre-tokenize one sequence into word strings — the same /// splitting `Tokenizer.train` applies before counting words. fn pretokenize( @@ -389,6 +406,30 @@ impl PyTokenizer { Ok(list) } + /// Awaitable `encode`: same arguments and result, run in a worker thread + /// (`asyncio.to_thread`) so the event loop stays free. The thread releases + /// the interpreter lock while Rust encodes, so encodes genuinely overlap. + #[pyo3(signature = (text, *, add_special_tokens = true) -> "Coroutine[Any, Any, npt.NDArray[np.uint32]]")] + fn async_encode<'py>( + slf: &Bound<'py, Self>, + text: &Bound<'py, PyAny>, + add_special_tokens: bool, + ) -> PyResult> { + to_thread(slf, "encode", text, add_special_tokens) + } + + /// Awaitable `encode_batch`: same arguments and result, run in a worker + /// thread (`asyncio.to_thread`) so the event loop stays free while the + /// batch encodes on Rust threads. + #[pyo3(signature = (texts, *, add_special_tokens = true) -> "Coroutine[Any, Any, list[npt.NDArray[np.uint32]]]")] + fn async_encode_batch<'py>( + slf: &Bound<'py, Self>, + texts: &Bound<'py, PyAny>, + add_special_tokens: bool, + ) -> PyResult> { + to_thread(slf, "encode_batch", texts, add_special_tokens) + } + /// Not implemented yet: decoding is not part of the encode pipeline. #[pyo3(signature = (ids, *, skip_special_tokens = true))] #[allow(unused_variables)] diff --git a/bindings/python/tests/test_async.py b/bindings/python/tests/test_async.py new file mode 100644 index 000000000..af1e54538 --- /dev/null +++ b/bindings/python/tests/test_async.py @@ -0,0 +1,41 @@ +import asyncio + +import numpy as np +import pytest + +from .conftest import SENTENCES, train_word_tokenizer + + +def test_async_encode_matches_sync(): + tok = train_word_tokenizer() + + async def go(): + single = await tok.async_encode(SENTENCES[0], add_special_tokens=False) + batch = await tok.async_encode_batch(SENTENCES, add_special_tokens=False) + return single, batch + + single, batch = asyncio.run(go()) + assert np.array_equal(single, tok.encode(SENTENCES[0], add_special_tokens=False)) + for got, want in zip(batch, tok.encode_batch(SENTENCES, add_special_tokens=False)): + assert np.array_equal(got, want) + + +def test_async_encodes_overlap(): + tok = train_word_tokenizer() + + async def go(): + return await asyncio.gather(*(tok.async_encode(s, add_special_tokens=False) for s in SENTENCES)) + + results = asyncio.run(go()) + for got, line in zip(results, SENTENCES): + assert np.array_equal(got, tok.encode(line, add_special_tokens=False)) + + +def test_async_error_surfaces_at_await(): + tok = train_word_tokenizer() + + async def go(): + await tok.async_encode(123, add_special_tokens=False) + + with pytest.raises(TypeError): + asyncio.run(go()) diff --git a/bindings/python/tools/stub-gen/src/main.rs b/bindings/python/tools/stub-gen/src/main.rs index 012aeec7f..ebb2c96ad 100644 --- a/bindings/python/tools/stub-gen/src/main.rs +++ b/bindings/python/tools/stub-gen/src/main.rs @@ -83,6 +83,12 @@ fn postprocess(contents: &str) -> String { "import numpy as np\nimport numpy.typing as npt\n\n{contents}" ); } + // The async_* annotations reference Coroutine/Any. + if contents.contains("Coroutine[") { + contents = format!( + "from collections.abc import Coroutine\nfrom typing import Any\n\n{contents}" + ); + } contents } From d484bdde1c9192a10e4e1a146cf094b3518dbee5 Mon Sep 17 00:00:00 2001 From: SBrandeis <33657802+SBrandeis@users.noreply.github.com> Date: Wed, 22 Jul 2026 15:14:22 +0200 Subject: [PATCH 11/19] comments --- bindings/python/README.md | 10 +++++----- bindings/python/examples/01_train_and_encode.py | 2 +- bindings/python/examples/04_train_bert_wordpiece.py | 5 +++-- bindings/python/examples/05_train_bytelevel_bpe.py | 8 ++++---- bindings/python/py_src/tokenizers/models/__init__.pyi | 3 ++- bindings/python/src/added_token.rs | 4 ++-- bindings/python/src/lib.rs | 4 ++-- bindings/python/src/models.rs | 3 ++- bindings/python/src/tokenizer.rs | 7 +++---- 9 files changed, 24 insertions(+), 22 deletions(-) diff --git a/bindings/python/README.md b/bindings/python/README.md index 74e8433f1..70132f6ac 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -40,11 +40,11 @@ tok.save("tokenizer.json") - `decoders`, `processors`, and the `implementations` helpers (`BertWordPieceTokenizer`, …) are gone. -Still here: `async_encode`/`async_encode_batch` (awaitable, run in a worker -thread — encode releases the GIL, so a tokio runtime is no longer needed), -parity-aware BPE training, and free-threaded Python — default wheels are -abi3 (one binary for CPython 3.10–3.14), and 3.14t gets its own non-abi3 -wheels (`maturin build --no-default-features`). +Unchanged from 0.x: `async_encode`/`async_encode_batch` (awaitable; encode +releases the GIL, so they run in a plain worker thread), parity-aware BPE +training (`trainers.ParityBpeTrainer`), and free-threaded Python — default +wheels are abi3 (one binary for CPython 3.10–3.14), and 3.14t gets its own +non-abi3 wheels (`maturin build --no-default-features`). ## Build and use locally diff --git a/bindings/python/examples/01_train_and_encode.py b/bindings/python/examples/01_train_and_encode.py index 674c36d63..93092da6b 100644 --- a/bindings/python/examples/01_train_and_encode.py +++ b/bindings/python/examples/01_train_and_encode.py @@ -71,7 +71,7 @@ def corpus(): assert np.array_equal(tok.encode(text), unpickled.encode(text)) print("pickle round-trip: identical ids") -# 8. decode is a stub for now +# 8. decode is not implemented yet — it raises instead of guessing try: tok.decode(ids) raise AssertionError("decode should not be implemented yet") diff --git a/bindings/python/examples/04_train_bert_wordpiece.py b/bindings/python/examples/04_train_bert_wordpiece.py index 20a30bb32..8bb0ec843 100644 --- a/bindings/python/examples/04_train_bert_wordpiece.py +++ b/bindings/python/examples/04_train_bert_wordpiece.py @@ -1,5 +1,6 @@ -"""Train a BERT-style WordPiece tokenizer from text files — the 1.x version of -the old `BertWordPieceTokenizer` helper, built from explicit components.""" +"""Train a BERT-style WordPiece tokenizer from text files: BertNormalizer + +BertPreTokenizer + WordPieceTrainer (the recipe `BertWordPieceTokenizer` +bundled in tokenizers 0.x).""" import argparse import glob diff --git a/bindings/python/examples/05_train_bytelevel_bpe.py b/bindings/python/examples/05_train_bytelevel_bpe.py index 0e0eec252..f16e46977 100644 --- a/bindings/python/examples/05_train_bytelevel_bpe.py +++ b/bindings/python/examples/05_train_bytelevel_bpe.py @@ -1,7 +1,7 @@ -"""Train a byte-level BPE tokenizer (GPT-2 style) from text files — the 1.x -version of the old `ByteLevelBPETokenizer` helper, built from explicit -components. One difference: the encode pipeline does not support -`add_prefix_space`, so it is always off.""" +"""Train a byte-level BPE tokenizer (GPT-2 style) from text files: ByteLevel +pre-tokenization + BpeTrainer seeded with the byte alphabet (the recipe +`ByteLevelBPETokenizer` bundled in tokenizers 0.x). The encode pipeline does +not support `add_prefix_space`, so it is always off.""" import argparse import glob diff --git a/bindings/python/py_src/tokenizers/models/__init__.pyi b/bindings/python/py_src/tokenizers/models/__init__.pyi index ef01439ae..264f52f51 100644 --- a/bindings/python/py_src/tokenizers/models/__init__.pyi +++ b/bindings/python/py_src/tokenizers/models/__init__.pyi @@ -17,7 +17,8 @@ class BPE(Model): @staticmethod def from_file(vocab: str, merges: str, *, unk_token: str |None = None) -> "BPE": """ - Load a BPE from the legacy vocab.json + merges.txt format. + Load a BPE from split vocab.json + merges.txt files (the format that + predates single-file tokenizer.json). """ class Model: diff --git a/bindings/python/src/added_token.rs b/bindings/python/src/added_token.rs index 06c23d5c8..5665c30f6 100644 --- a/bindings/python/src/added_token.rs +++ b/bindings/python/src/added_token.rs @@ -86,8 +86,8 @@ pub enum TokenInput { Token(PyAddedToken), } -/// Plain strings become tokens with `special=special_default` (and -/// `normalized=!special_default`, matching v1). +/// Plain strings become tokens with `special=special_default` and +/// `normalized=!special_default` (special tokens match raw text). pub fn parse_tokens(items: Vec, special_default: bool) -> Vec { items .into_iter() diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs index 6bcc17ebe..49e31e98f 100644 --- a/bindings/python/src/lib.rs +++ b/bindings/python/src/lib.rs @@ -18,8 +18,8 @@ pub fn component_repr(component: &T) -> String { } // Forked children of a process that used our rayon threads would inherit a -// poisoned thread pool; disable parallelism there unless the user configured -// it explicitly (same behavior as the v1 bindings). +// poisoned thread pool; disable parallelism there unless the user opted in +// explicitly through TOKENIZERS_PARALLELISM. #[cfg(target_family = "unix")] extern "C" fn child_after_fork() { use std::sync::atomic::Ordering; diff --git a/bindings/python/src/models.rs b/bindings/python/src/models.rs index 578a78123..a202e232a 100644 --- a/bindings/python/src/models.rs +++ b/bindings/python/src/models.rs @@ -71,7 +71,8 @@ impl PyBPE { Ok(PyClassInitializer::from(PyModel { inner: bpe.into() }).add_subclass(PyBPE)) } - /// Load a BPE from the legacy vocab.json + merges.txt format. + /// Load a BPE from split vocab.json + merges.txt files (the format that + /// predates single-file tokenizer.json). #[staticmethod] #[pyo3(signature = (vocab, merges, *, unk_token = None) -> "BPE")] fn from_file( diff --git a/bindings/python/src/tokenizer.rs b/bindings/python/src/tokenizer.rs index 838d82a70..ac0299891 100644 --- a/bindings/python/src/tokenizer.rs +++ b/bindings/python/src/tokenizer.rs @@ -28,7 +28,7 @@ use crate::trainers::PyTrainer; /// Set when the bindings actually run a rayon-parallel section, so the /// pthread_atfork handler only disables parallelism in children of processes -/// that really used it (mirrors the v1 bindings' semantics). +/// that really used it — forking before any parallel work stays quiet. pub static USED_PARALLELISM: AtomicBool = AtomicBool::new(false); /// The compiled encode path plus the facts about the spec the encode calls @@ -170,9 +170,8 @@ impl PyTokenizer { } /// Wrap a bound method call in `asyncio.to_thread`, returning the coroutine. -/// The whole legacy async surface (a tokio runtime + pyo3-async-runtimes) is -/// unnecessary here because encode already releases the interpreter lock — -/// a worker thread is all it takes to keep the event loop responsive. +/// No async runtime is involved: encode releases the interpreter lock, so a +/// plain worker thread is enough to keep the event loop responsive. fn to_thread<'py>( slf: &Bound<'py, PyTokenizer>, method: &str, From 259c52a9987505ae565c8f9007a48cc3701b8718 Mon Sep 17 00:00:00 2001 From: SBrandeis <33657802+SBrandeis@users.noreply.github.com> Date: Wed, 22 Jul 2026 15:25:40 +0200 Subject: [PATCH 12/19] don't test thread scaling on macos --- .github/workflows/python.yml | 6 ++- bindings/python/examples/03_threading.py | 54 +++++++++++++++--------- 2 files changed, 39 insertions(+), 21 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 102a7ec6e..6db3687fe 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -71,11 +71,15 @@ jobs: run: .venv/bin/python -m pytest -q -m "not network" # Every example runs here so they can never rot. 06 pulls wikitext-2 - # from the Hub (cached above). + # from the Hub (cached above). The threading example's scaling asserts + # are disabled on macOS: those runners' noisy shared hosts make parallel + # speedup unmeasurable (observed 0.4-0.5x on starved VMs). A held GIL + # would be platform-independent, so the strict ubuntu jobs still catch it. - name: Run the examples working-directory: ./bindings/python env: HF_TOKEN: ${{ secrets.HF_TOKEN }} + TOKENIZERS_SCALING_ASSERTS: ${{ runner.os == 'macOS' && '0' || '1' }} run: | .venv/bin/python examples/01_train_and_encode.py .venv/bin/python examples/02_pretrained.py diff --git a/bindings/python/examples/03_threading.py b/bindings/python/examples/03_threading.py index b18211f9b..130cb920c 100644 --- a/bindings/python/examples/03_threading.py +++ b/bindings/python/examples/03_threading.py @@ -1,5 +1,10 @@ """Demonstrates that encode runs without the GIL: Python threads calling -encode() scale, and encode_batch parallelizes in Rust via rayon.""" +encode() scale, and encode_batch parallelizes in Rust via rayon. + +Scaling is asserted unless TOKENIZERS_SCALING_ASSERTS=0 (CI sets it on shared +macOS runners, whose noisy hosts make parallel speedup unmeasurable there); +result-correctness asserts always apply. +""" import os import time @@ -9,12 +14,30 @@ from tokenizers import Tokenizer DATA = Path(__file__).resolve().parents[3] / "tokenizers" / "data" -N_THREADS = 4 +N_THREADS = min(4, os.cpu_count() or 1) +STRICT = os.environ.get("TOKENIZERS_SCALING_ASSERTS", "1") != "0" + + +def check_scaling(cond, msg): + if STRICT: + assert cond, msg + elif not cond: + print(f"WARNING (not asserted here): {msg}") + + +def best_of(n, fn): + times = [] + for _ in range(n): + start = time.perf_counter() + fn() + times.append(time.perf_counter() - start) + return min(times) + tok = Tokenizer.from_file(DATA / "llama-3-tokenizer.json") with open(DATA / "big.txt", encoding="utf-8") as f: text = f.read(2_000_000) -lines = [line for line in text.splitlines() if line.strip()] +lines = [line for line in text.splitlines() if line.strip()] * 4 tok.encode(text, add_special_tokens=False) # warmup + compile @@ -24,16 +47,10 @@ def encode_once(): # 1. Python threads: with the GIL held during encode this could not scale -start = time.perf_counter() -for _ in range(N_THREADS): - encode_once() -sequential = time.perf_counter() - start - with ThreadPoolExecutor(N_THREADS) as pool: # warmup thread pool list(pool.map(lambda _: None, range(N_THREADS))) - start = time.perf_counter() - results = list(pool.map(lambda _: encode_once(), range(N_THREADS))) - threaded = time.perf_counter() - start + sequential = best_of(3, lambda: [encode_once() for _ in range(N_THREADS)]) + threaded = best_of(3, lambda: list(pool.map(lambda _: encode_once(), range(N_THREADS)))) speedup = sequential / threaded print( @@ -41,26 +58,23 @@ def encode_once(): f"sequential {sequential:.2f}s, {N_THREADS} threads {threaded:.2f}s " f"({speedup:.1f}x)" ) -assert speedup > 1.5, f"threads did not scale ({speedup:.2f}x): is the GIL held?" +check_scaling(speedup > 1.5, f"threads did not scale ({speedup:.2f}x): is the GIL held?") # 2. encode_batch: rayon parallelism inside one call, toggled by env var os.environ["TOKENIZERS_PARALLELISM"] = "false" -start = time.perf_counter() serial_ids = tok.encode_batch(lines, add_special_tokens=False) -serial = time.perf_counter() - start +serial = best_of(3, lambda: tok.encode_batch(lines, add_special_tokens=False)) os.environ["TOKENIZERS_PARALLELISM"] = "true" -tok.encode_batch(lines[:100], add_special_tokens=False) # spin up the pool -start = time.perf_counter() -parallel_ids = tok.encode_batch(lines, add_special_tokens=False) -parallel = time.perf_counter() - start +parallel_ids = tok.encode_batch(lines, add_special_tokens=False) # warmup: spins up the pool +parallel = best_of(3, lambda: tok.encode_batch(lines, add_special_tokens=False)) assert all(a.tolist() == b.tolist() for a, b in zip(serial_ids, parallel_ids, strict=True)) -mbps = len(text) / parallel / 1e6 +mbps = sum(len(line) for line in lines) / parallel / 1e6 print( f"encode_batch {len(lines)} lines: serial {serial:.2f}s, " f"rayon {parallel:.2f}s ({serial / parallel:.1f}x, {mbps:.0f} MB/s)" ) -assert serial / parallel > 1.5, "rayon batch did not scale" +check_scaling(serial / parallel > 1.5, f"rayon batch did not scale ({serial / parallel:.2f}x)") print("OK") From 94eb3b6a8f587b1832aed36edb92b11fbaac225f Mon Sep 17 00:00:00 2001 From: SBrandeis <33657802+SBrandeis@users.noreply.github.com> Date: Wed, 22 Jul 2026 16:04:55 +0200 Subject: [PATCH 13/19] iteration --- .github/workflows/build_pr_documentation.yml | 14 +- .github/workflows/python-release.yml | 103 ++++++--- .github/workflows/python.yml | 53 +++-- README.md | 33 +-- bindings/python/.gitignore | 1 + bindings/python/CHANGELOG.md | 68 ++++-- bindings/python/Cargo.toml | 2 +- bindings/python/LICENSE | 201 ++++++++++++++++++ bindings/python/Makefile | 7 +- bindings/python/README.md | 97 +++++++-- .../{02_pretrained.py => 01_pretrained.py} | 17 +- ...n_and_encode.py => 02_train_and_encode.py} | 9 +- bindings/python/examples/03_threading.py | 5 +- .../examples/04_train_bert_wordpiece.py | 2 +- .../python/examples/05_train_bytelevel_bpe.py | 2 +- .../python/examples/06_train_with_datasets.py | 2 +- bindings/python/py_src/tokenizers/__init__.py | 2 +- .../python/py_src/tokenizers/__init__.pyi | 18 +- .../py_src/tokenizers/models/__init__.pyi | 4 +- .../tokenizers/normalizers/__init__.pyi | 2 + .../tokenizers/pre_tokenizers/__init__.pyi | 5 +- .../py_src/tokenizers/trainers/__init__.pyi | 2 + bindings/python/pyproject.toml | 29 ++- bindings/python/src/lib.rs | 3 +- bindings/python/src/models.rs | 2 +- bindings/python/src/pre_tokenizers.rs | 3 +- bindings/python/src/tokenizer.rs | 1 + bindings/python/stubtest_allowlist.txt | 12 ++ bindings/python/tools/stub-gen/src/main.rs | 74 ++++++- 29 files changed, 647 insertions(+), 126 deletions(-) create mode 100644 bindings/python/LICENSE rename bindings/python/examples/{02_pretrained.py => 01_pretrained.py} (71%) rename bindings/python/examples/{01_train_and_encode.py => 02_train_and_encode.py} (88%) create mode 100644 bindings/python/stubtest_allowlist.txt diff --git a/.github/workflows/build_pr_documentation.yml b/.github/workflows/build_pr_documentation.yml index 188e86499..34d25dae0 100644 --- a/.github/workflows/build_pr_documentation.yml +++ b/.github/workflows/build_pr_documentation.yml @@ -2,9 +2,17 @@ name: Build PR Documentation # docs/source-doc-builder describes the 0.x Python API. The bindings were # rewritten for 1.0 (PipelineTokenizer); until the docs are rewritten to -# match, PR doc previews only run on demand. +# match, PR doc previews only run on demand. On dispatch, pass the PR's head +# sha and number by hand (they have no value outside a pull_request event). on: workflow_dispatch: + inputs: + commit_sha: + description: "Head commit sha of the PR to build docs for" + required: true + pr_number: + description: "PR number the preview is posted to" + required: true concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -14,8 +22,8 @@ jobs: build: uses: huggingface/doc-builder/.github/workflows/build_pr_documentation.yml@e60a538eea9817ab312196d0d233604b01697265 # main with: - commit_sha: ${{ github.event.pull_request.head.sha }} - pr_number: ${{ github.event.number }} + commit_sha: ${{ inputs.commit_sha }} + pr_number: ${{ inputs.pr_number }} package: tokenizers path_to_docs: tokenizers/docs/source-doc-builder/ package_path: tokenizers/bindings/python/ diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml index e03ffb696..c257e330f 100644 --- a/.github/workflows/python-release.yml +++ b/.github/workflows/python-release.yml @@ -1,12 +1,15 @@ name: Python Release # Default wheels are abi3-py310: one binary per platform covers CPython -# 3.10–3.14. Free-threaded 3.14t gets its own interpreter-specific wheels -# (flavor: ft), built without the default `abi3` cargo feature. +# 3.10–3.14. Free-threaded interpreters (3.13t/3.14t) get their own +# interpreter-specific wheels (flavor: ft), built without the default `abi3` +# cargo feature. `workflow_dispatch` runs the full matrix without releasing — +# use it to exercise the exotic targets before tagging. on: push: tags: - v* + workflow_dispatch: env: AWS_DEFAULT_REGION: us-east-1 @@ -23,10 +26,9 @@ jobs: working-directory: ./bindings/python build: - name: build on ${{ matrix.platform || matrix.os }} (${{ matrix.target }} - ${{ matrix.manylinux || 'auto' }} - ${{ matrix.flavor == 'ft' && '3.14t' || matrix.interpreter || '3.14' }}) - # only run on push to main and on release + name: build on ${{ matrix.platform || matrix.os }} (${{ matrix.target }} - ${{ matrix.manylinux || 'auto' }} - ${{ matrix.interpreter || '3.14' }}) needs: [lock_exists] - if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'Full Build') + if: startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' strategy: fail-fast: false matrix: @@ -61,6 +63,7 @@ jobs: python-architecture: arm64 python-install: "3.14" interpreter: "3.14" + smoke: true # - os: windows # ls: dir # target: aarch64 @@ -97,19 +100,34 @@ jobs: target: s390x interpreter: "3.14" - # --- Free-threaded Python 3.14t wheels ------------------------- + # --- Free-threaded Python wheels (3.13t and 3.14t) -------------- # `flavor: ft` drops the default `abi3` cargo feature (see the - # maturin args below), producing interpreter-specific cp314t - # wheels. linux container builds get 3.14t from the docker image, - # so `python-install` is unset there; macOS/windows host builds - # need it so setup-python actually installs 3.14t. - - { os: ubuntu, platform: linux, target: x86_64, manylinux: auto, flavor: ft } - - { os: ubuntu, platform: linux, target: aarch64, manylinux: auto, flavor: ft } - - { os: ubuntu, platform: linux, target: x86_64, manylinux: musllinux_1_1, flavor: ft } - - { os: ubuntu, platform: linux, target: aarch64, manylinux: musllinux_1_1, flavor: ft } - - { os: macos, target: x86_64, manylinux: auto, flavor: ft, python-install: "3.14t" } - - { os: macos, target: aarch64, manylinux: auto, flavor: ft, python-install: "3.14t" } - - { os: windows, ls: dir, target: x86_64, manylinux: auto, python-architecture: x64, python-install: "3.14t", flavor: ft } + # maturin args below), producing interpreter-specific cp313t/cp314t + # wheels. linux container builds get the ft interpreters from the + # docker image, so `python-install` is unset there; macOS/windows + # host builds need it so setup-python actually installs them. + - { os: ubuntu, platform: linux, target: x86_64, manylinux: auto, flavor: ft, interpreter: "3.13t" } + - { os: ubuntu, platform: linux, target: x86_64, manylinux: auto, flavor: ft, interpreter: "3.14t" } + - { os: ubuntu, platform: linux, target: aarch64, manylinux: auto, flavor: ft, interpreter: "3.13t" } + - { os: ubuntu, platform: linux, target: aarch64, manylinux: auto, flavor: ft, interpreter: "3.14t" } + - { os: ubuntu, platform: linux, target: x86_64, manylinux: musllinux_1_1, flavor: ft, interpreter: "3.13t" } + - { os: ubuntu, platform: linux, target: x86_64, manylinux: musllinux_1_1, flavor: ft, interpreter: "3.14t" } + - { os: ubuntu, platform: linux, target: aarch64, manylinux: musllinux_1_1, flavor: ft, interpreter: "3.13t" } + - { os: ubuntu, platform: linux, target: aarch64, manylinux: musllinux_1_1, flavor: ft, interpreter: "3.14t" } + - { os: macos, target: x86_64, manylinux: auto, flavor: ft, interpreter: "3.13t", python-install: "3.13t" } + - { os: macos, target: x86_64, manylinux: auto, flavor: ft, interpreter: "3.14t", python-install: "3.14t" } + - { os: macos, target: aarch64, manylinux: auto, flavor: ft, interpreter: "3.13t", python-install: "3.13t", smoke: true } + - { os: macos, target: aarch64, manylinux: auto, flavor: ft, interpreter: "3.14t", python-install: "3.14t", smoke: true } + - { os: windows, ls: dir, target: x86_64, manylinux: auto, python-architecture: x64, python-install: "3.13t", interpreter: "3.13t", flavor: ft, smoke: true } + - { os: windows, ls: dir, target: x86_64, manylinux: auto, python-architecture: x64, python-install: "3.14t", interpreter: "3.14t", flavor: ft, smoke: true } + + # --- Smoke-test flags -------------------------------------------- + # `smoke: true` marks the cells whose wheel the runner itself can + # import (matching OS/arch/libc). Cross-compiled and musl wheels + # can't run here; twine check still covers their metadata. + - { os: ubuntu, target: x86_64, manylinux: auto, flavor: abi3, smoke: true } + - { os: macos, target: aarch64, flavor: abi3, smoke: true } + - { os: windows, target: x86_64, flavor: abi3, smoke: true } exclude: - os: windows target: aarch64 @@ -149,7 +167,7 @@ jobs: # extensions). `flavor=abi3` builds use defaults. args: >- --release --out dist - --interpreter ${{ matrix.flavor == 'ft' && '3.14t' || matrix.interpreter || '3.14' }} + --interpreter ${{ matrix.interpreter || '3.14' }} ${{ matrix.flavor == 'ft' && '--no-default-features' || '' }} rust-toolchain: stable sccache: false @@ -161,9 +179,37 @@ jobs: - run: twine check --strict dist/* working-directory: ./bindings/python + - name: Install uv + if: matrix.smoke + uses: astral-sh/setup-uv@v6 + + # abi3 wheels are deliberately smoked on 3.12 — not the 3.14 they were + # built with — to prove the stable-ABI claim. ft wheels run on their + # exact interpreter and must keep the GIL disabled. + - name: Smoke test the wheel + if: matrix.smoke + working-directory: ./bindings/python + shell: bash + run: | + uv venv .smoke --python ${{ matrix.flavor == 'ft' && matrix.interpreter || '3.12' }} + source .smoke/bin/activate 2>/dev/null || source .smoke/Scripts/activate + uv pip install dist/*.whl + python -c " + import sys + import tokenizers + print('tokenizers', tokenizers.__version__, 'on', sys.version) + tok = tokenizers.Tokenizer(tokenizers.models.BPE()) + tok.pre_tokenizer = tokenizers.pre_tokenizers.Whitespace() + tok.train_from_iterator(['smoke test'] * 4, trainer=tokenizers.trainers.BpeTrainer(show_progress=False)) + assert tok.encode('smoke', add_special_tokens=False).dtype == 'uint32' + gil = getattr(sys, '_is_gil_enabled', lambda: True)() + assert not ('${{ matrix.flavor }}' == 'ft' and gil), 'free-threaded wheel re-enabled the GIL' + print('smoke OK (gil enabled:', gil, ')') + " + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: pypi_files-${{ matrix.os }}-${{ matrix.target }}-${{ matrix.manylinux }}-${{ matrix.flavor || 'abi3' }} + name: pypi_files-${{ matrix.os }}-${{ matrix.target }}-${{ matrix.manylinux }}-${{ matrix.flavor || 'abi3' }}-${{ matrix.interpreter || 'default' }} path: ./bindings/python/dist build-sdist: name: build sdist @@ -179,7 +225,7 @@ jobs: rust-toolchain: stable - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: pypi_files-srt + name: pypi_files-sdist path: ./bindings/python/dist @@ -187,6 +233,8 @@ jobs: name: Upload package to PyPi runs-on: ubuntu-latest needs: [build, build-sdist] + # Tags only: a workflow_dispatch run is a build dry-run, never a release. + if: startsWith(github.ref, 'refs/tags/') env: PYPI_TOKEN: ${{ secrets.PYPI_TOKEN_DIST }} @@ -203,9 +251,12 @@ jobs: with: path: ./bindings/python/dist merge-multiple: true - # Temporary deactivation while testing abi3 CI - # - name: Upload to PyPi - # working-directory: ./bindings/python - # run: | - # pip install twine - # twine upload dist/* -u __token__ -p "$PYPI_TOKEN" + + # TODO(release): re-enable before shipping 1.0 — deactivated while the + # abi3 CI rework is validated. Until then, tagging builds and checks + # everything but publishes nothing. + # - name: Upload to PyPi + # working-directory: ./bindings/python + # run: | + # pip install twine + # twine upload dist/* -u __token__ -p "$PYPI_TOKEN" diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 6db3687fe..c2813bc98 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -43,7 +43,7 @@ jobs: working-directory: ./bindings/python run: | uv venv .venv --python ${{ matrix.python }} - uv pip install --python .venv/bin/python maturin numpy pytest datasets + uv pip install --python .venv/bin/python maturin numpy pytest datasets mypy source .venv/bin/activate && maturin develop --release \ ${{ matrix.python == '3.14t' && '--no-default-features' || '' }} @@ -81,23 +81,37 @@ jobs: HF_TOKEN: ${{ secrets.HF_TOKEN }} TOKENIZERS_SCALING_ASSERTS: ${{ runner.os == 'macOS' && '0' || '1' }} run: | - .venv/bin/python examples/01_train_and_encode.py - .venv/bin/python examples/02_pretrained.py + .venv/bin/python examples/01_pretrained.py + .venv/bin/python examples/02_train_and_encode.py .venv/bin/python examples/03_threading.py .venv/bin/python examples/04_train_bert_wordpiece.py .venv/bin/python examples/05_train_bytelevel_bpe.py .venv/bin/python examples/06_train_with_datasets.py - # The .pyi stubs are generated from the built extension; a diff here - # means someone changed the Rust API without running `make stubs`. + # The .pyi stubs are generated from the built extension; a difference + # here means someone changed the Rust API without running `make stubs`. + # `git status --porcelain` (not `git diff`) so brand-new, not-yet-tracked + # stub files are caught too. - name: Check the committed stubs are current working-directory: ./bindings/python run: | cargo run --manifest-path tools/stub-gen/Cargo.toml - git diff --exit-code -- py_src + if [ -n "$(git status --porcelain -- py_src)" ]; then + git status --porcelain -- py_src + git diff -- py_src + echo "::error::committed .pyi stubs are stale — run 'make stubs' and commit the result" + exit 1 + fi + + # Freshness (above) proves the stubs match the Rust API; stubtest + # proves they match the *runtime* — every stubbed name exists with the + # right shape, and nothing importable is missing from the stubs. + - name: Check the stubs against the runtime (stubtest) + working-directory: ./bindings/python + run: .venv/bin/python -m mypy.stubtest tokenizers --allowlist stubtest_allowlist.txt build_windows: - name: Check it builds on Windows + name: Build and test on Windows runs-on: windows-latest steps: - name: Checkout repository @@ -106,13 +120,28 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - - name: Install Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - name: Cache cargo registry / git / target + uses: Swatinem/rust-cache@23869a5bd66c73db3c0ac40331f3206eb23791dc # v2.9.1 with: - python-version: "3.14" + workspaces: bindings/python + shared-key: python + cache-bin: false - - name: Build - run: cargo build --release --manifest-path ./bindings/python/Cargo.toml + - name: Install uv + uses: astral-sh/setup-uv@v6 + + # No test data on Windows (the fetch targets need `make`): the + # data-dependent tests skip themselves, the rest still exercise the + # wheel end-to-end (train, encode, pickle, threading). + - name: Build and test + working-directory: ./bindings/python + shell: bash + run: | + uv venv .venv --python 3.14 + source .venv/Scripts/activate + uv pip install maturin numpy pytest + maturin develop --release + python -m pytest -q -m "not network" quality: name: Lint & format diff --git a/README.md b/README.md index 6b561b662..bd735d36f 100644 --- a/README.md +++ b/README.md @@ -23,9 +23,11 @@ versatility. less than 20 seconds to tokenize a GB of text on a server's CPU. - Easy to use, but also extremely versatile. - Designed for research and production. - - Normalization comes with alignments tracking. It's always possible to get the part of the - original sentence that corresponds to a given token. - - Does all the pre-processing: Truncate, Pad, add the special tokens your model needs. + - The Rust library additionally tracks alignments (which part of the original + sentence a token comes from) and does the full pre-processing: truncate, + pad, add the special tokens your model needs. The 1.x Python bindings + focus on fast encoding and do not expose these yet — see the + [breaking changes](bindings/python/README.md#breaking-changes-vs-0x). ## Performances Performances can vary depending on hardware. The Python bindings ship a @@ -46,17 +48,14 @@ We provide bindings to the following languages (more to come!): ## Installation -You can install from source using: +`pip install tokenizers` installs the released **0.x** version. The example +below uses the **1.x** rewrite of the Python bindings, which is not released +yet — install it from source (needs a Rust toolchain): + ```bash pip install git+https://github.com/huggingface/tokenizers.git#subdirectory=bindings/python ``` -or install the released versions with - -```bash -pip install tokenizers -``` - ## Quick example using Python: Choose your model between Byte-Pair Encoding, WordPiece or Unigram and instantiate a tokenizer: @@ -65,7 +64,7 @@ Choose your model between Byte-Pair Encoding, WordPiece or Unigram and instantia from tokenizers import Tokenizer from tokenizers.models import BPE -tokenizer = Tokenizer(BPE()) +tokenizer = Tokenizer(BPE(unk_token="[UNK]")) ``` You can customize how pre-tokenization (e.g., splitting into words) is done: @@ -93,7 +92,11 @@ print([tokenizer.id_to_token(i) for i in ids]) ``` `encode` returns the token ids as a `numpy.uint32` array — ready to hand to -your model with no further conversion. - -Check the [documentation](https://huggingface.co/docs/tokenizers/index) -or the [quicktour](https://huggingface.co/docs/tokenizers/quicktour) to learn more! +your model with no further conversion. The emoji comes out as `[UNK]`: it +never appeared in the training files, so it is not in the vocabulary, and BPE +falls back to the `unk_token` we configured above. + +More in [bindings/python](bindings/python) — its README and `examples/` +cover loading pretrained tokenizers, threading and async, and training. (The +[hosted documentation](https://huggingface.co/docs/tokenizers/index) still +describes the released 0.x API.) diff --git a/bindings/python/.gitignore b/bindings/python/.gitignore index 80473e575..dfeb86359 100644 --- a/bindings/python/.gitignore +++ b/bindings/python/.gitignore @@ -3,3 +3,4 @@ .release/ python_bench.json python_bench.md +.smoke/ diff --git a/bindings/python/CHANGELOG.md b/bindings/python/CHANGELOG.md index 2047103c7..5fbca3bbc 100644 --- a/bindings/python/CHANGELOG.md +++ b/bindings/python/CHANGELOG.md @@ -12,21 +12,63 @@ Python: encode never holds the GIL, batches run multi-threaded in Rust, inputs are borrowed instead of copied, and ids come back as `numpy.uint32` arrays without a copy. -Breaking changes: - -- `encode` returns a numpy array of ids, not an `Encoding` object. Offsets, - type ids, attention masks, truncation and padding are gone from the encode - path. +Breaking changes — encoding: + +- `encode` returns a numpy array of ids, not an `Encoding` object. Everything + the `Encoding` carried is gone from the encode path: tokens, offsets, type + ids, attention masks, special-tokens masks, word ids, overflowing/stride, + and the char/word/token mapping helpers. Truncation and padding + (`enable_truncation`/`enable_padding` and their getters) are gone too. +- `encode` takes a single text: the `pair=` argument and the + `is_pretokenized=` mode no longer exist (same for `encode_batch`). - Not implemented yet (loud errors, never wrong ids): `decode`, post-processor templates (pass `add_special_tokens=False`), and the - `Metaspace` pre-tokenizer. -- Custom Python components are not supported; components are plain values. -- `decoders`, `processors`, and the `implementations` helpers are gone. - -Kept from 0.x: `async_encode`/`async_encode_batch` (now a thin -`asyncio.to_thread` wrapper — no tokio runtime), parity-aware BPE training -(`trainers.ParityBpeTrainer`), and free-threaded Python support (default -wheels are abi3-py310; 3.14t ships non-abi3 `cp314t` wheels). + `Metaspace` pre-tokenizer. `decode_batch`, `DecodeStream`, + `encode_batch_fast`, and `Tokenizer.post_process` are removed. +- **`transformers`' `PreTrainedTokenizerFast` cannot run on 1.0 yet** — it + needs the `Encoding` fields, padding/truncation, and pair inputs listed + above. Pin `tokenizers<1.0` for `transformers` until it targets 1.x. + +Breaking changes — components and introspection: + +- Components (models, normalizers, pre-tokenizers, trainers) are immutable + values you construct and assign. All attribute getters/setters, + `__getstate__`/`__setstate__`, and the helper methods + (`Normalizer.normalize_str`, `PreTokenizer.pre_tokenize_str`, + `Model.tokenize`, `Model.save`, `Model.get_trainer`, …) are gone; `repr()` + shows a component's full `tokenizer.json` serialization instead. +- Custom Python components (`Normalizer.custom`, `PreTokenizer.custom`, + `Decoder.custom`) are not supported, and the supporting types + (`NormalizedString`, `PreTokenizedString`, `Regex`, `Token`) are removed. + `Replace`/`Split` take `regex=True` instead of a `Regex` object. +- Models no longer accept in-memory vocabs: `BPE(vocab, merges)`, + `WordPiece(vocab)`, `WordLevel(vocab)`, `Unigram(vocab)` and the + `read_file`/`from_file` helpers are gone (only `BPE.from_file` remains). + Build models by training or by loading a `tokenizer.json`. +- `decoders`, `processors`, the `implementations` helpers + (`BertWordPieceTokenizer`, …), and `tools` (`EncodingVisualizer`) are gone. +- `pre_tokenizers.ByteLevel` no longer takes `add_prefix_space` (files using + it fail at encode); `normalizers.Precompiled`/`Nmt`/`ByteLevel` cannot be + constructed from Python (loaded ones still run). + +Breaking changes — `Tokenizer` API and packaging: + +- Removed: `Tokenizer.from_str` (use `from_buffer`), `get_added_tokens_decoder`, + `num_special_tokens_to_add`, the `encode_special_tokens` property, and the + `length=` argument of `train_from_iterator`. +- Most arguments are now keyword-only: `get_vocab(True)` becomes + `get_vocab(with_added_tokens=True)`, and so on. +- Errors raise the new `tokenizers.TokenizersError` (a `RuntimeError` was + raised before in most places). +- `from_pretrained` now calls the `huggingface_hub` Python package, which + moved from a required dependency to the `hub` extra: + `pip install 'tokenizers[hub]'`. `numpy>=1.24` is a new required dependency. + +Kept: `async_encode`/`async_encode_batch` (now a thin `asyncio.to_thread` +wrapper — no tokio runtime) and free-threaded Python support (default wheels +are abi3-py310; free-threaded interpreters get their own non-abi3 wheels). +New in 1.0: parity-aware BPE training (`trainers.ParityBpeTrainer`), which +never shipped in a 0.x release. ## [0.13.2] diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index b38039c83..b08410bf2 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -2,7 +2,7 @@ name = "tokenizers-python" version = "1.0.0-dev.0" edition = "2024" -description = "Fast Python bindings for 🤗 tokenizers, built on the PipelineTokenizer encode path" +description = "Fast, thread-friendly Python bindings for 🤗 tokenizers" license = "Apache-2.0" [lib] diff --git a/bindings/python/LICENSE b/bindings/python/LICENSE new file mode 100644 index 000000000..261eeb9e9 --- /dev/null +++ b/bindings/python/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/bindings/python/Makefile b/bindings/python/Makefile index 896436d05..7aef0366e 100644 --- a/bindings/python/Makefile +++ b/bindings/python/Makefile @@ -7,7 +7,7 @@ dev: .venv .venv: uv venv .venv - uv pip install --python $(PYTHON) maturin numpy pytest ruff + uv pip install --python $(PYTHON) maturin numpy pytest ruff mypy # Regenerate the .pyi stubs from the built extension. Run after `make dev`. .PHONY: stubs @@ -17,14 +17,15 @@ stubs: .PHONY: test test: dev $(PYTHON) -m pytest -q + $(PYTHON) -m mypy.stubtest tokenizers --allowlist stubtest_allowlist.txt # Run the end-to-end examples. `datasets` (for 06) is installed on demand so # `make dev` stays lean; 06 downloads wikitext-2 (~12 MB) on first run. .PHONY: examples examples: dev uv pip install --python $(PYTHON) datasets - $(PYTHON) examples/01_train_and_encode.py - $(PYTHON) examples/02_pretrained.py + $(PYTHON) examples/01_pretrained.py + $(PYTHON) examples/02_train_and_encode.py $(PYTHON) examples/03_threading.py $(PYTHON) examples/04_train_bert_wordpiece.py $(PYTHON) examples/05_train_bytelevel_bpe.py diff --git a/bindings/python/README.md b/bindings/python/README.md index 70132f6ac..41a5ce676 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -1,20 +1,46 @@ # tokenizers (Python bindings) -Python bindings for 🤗 tokenizers, built on the `PipelineTokenizer` encode -path. This is the 1.x rewrite of the bindings: same `tokenizer.json` files, -same ids as 0.x, and much faster through Python — encode never holds the GIL, -batches run multi-threaded in Rust, inputs are borrowed instead of copied, and -ids come back as `numpy.uint32` arrays without a copy. +Tokenizers turn text into the sequences of integer ids that language models +consume. This package is the Python interface to Hugging Face's Rust +[tokenizers](https://github.com/huggingface/tokenizers) library. + +This is the 1.x rewrite of the bindings: it loads the same `tokenizer.json` +files as 0.x and produces the same ids, but is much faster through Python — +encoding runs in Rust threads without blocking your Python program, and ids +come back as ready-to-use `numpy` arrays. ```python import tokenizers as tk tok = tk.Tokenizer.from_file("tokenizer.json") -ids = tok.encode("Hello world", add_special_tokens=False) # np.ndarray[uint32] -batch = tok.encode_batch(lines, add_special_tokens=False) # list of arrays + +# Returns a numpy.uint32 array of token ids. +# add_special_tokens=False skips template tokens like [CLS]/[SEP]; inserting +# them is not implemented yet in 1.0, so leaving it True raises a loud +# NotImplementedError on tokenizers that use such templates (BERT, Llama, …). +ids = tok.encode("Hello world", add_special_tokens=False) + +# A list of arrays, encoded in parallel across Rust threads. +batch = tok.encode_batch(["Hello world", "How are you?"], add_special_tokens=False) +``` + +To load a tokenizer straight from the [Hugging Face Hub](https://huggingface.co), +install the `hub` extra (`pip install 'tokenizers[hub]'`): + +```python +tok = tk.Tokenizer.from_pretrained("openai-community/gpt2") ``` -Training and in-place modification work too: +## Training your own tokenizer + +A tokenizer has three parts, applied in order: + +- a **normalizer** cleans the text (lowercasing, Unicode fix-ups), +- a **pre-tokenizer** cuts it into pieces (usually words), +- the **model** turns each piece into ids, using a vocabulary learned during + training (BPE, WordPiece, Unigram, or WordLevel). + +You pick the parts, then train the model's vocabulary on your own text: ```python tok = tk.Tokenizer(tk.models.BPE()) @@ -24,12 +50,36 @@ tok.train_from_iterator(lines, trainer=tk.trainers.BpeTrainer(vocab_size=30000)) tok.save("tokenizer.json") ``` +The `examples/` directory walks through all of this, starting with the +simplest case (load a pretrained file and encode). + +## Threading and async + +Encoding releases the Python interpreter lock (the GIL), so this package +plays well with threads and event loops: + +- Calling `encode` from several Python threads scales — the threads really + run in parallel, on every interpreter (free-threaded or not). +- `encode_batch` parallelizes one batch across Rust threads. Set the + `TOKENIZERS_PARALLELISM` environment variable to `false`/`true` to + disable/force this. +- In `asyncio` code, `await tok.async_encode(text)` / + `await tok.async_encode_batch(texts)` keep the event loop free while Rust + encodes in a worker thread. + +```python +ids = await tok.async_encode("Hello world", add_special_tokens=False) +``` + ## Breaking changes vs 0.x 1.x is a ground-up rewrite with a smaller, faster API. The headline changes: - `encode` returns a `numpy.uint32` array of ids, not an `Encoding` object. - Offsets, type ids, and attention masks are gone from the encode path. + Tokens, offsets, type ids, attention masks, and word ids are gone from the + encode path, and so are truncation and padding + (`enable_truncation`/`enable_padding`). +- `encode` takes a single text: no `pair=` argument, no `is_pretokenized=`. - Not implemented yet (loud errors, never wrong ids): `decode`, post-processor templates (`[CLS]`/`` insertion — pass `add_special_tokens=False`), and the `Metaspace` pre-tokenizer @@ -39,12 +89,15 @@ tok.save("tokenizer.json") you subclass. - `decoders`, `processors`, and the `implementations` helpers (`BertWordPieceTokenizer`, …) are gone. +- **`transformers` cannot use 1.0 as its backend yet** — it needs several of + the removed pieces. Pin `tokenizers<1.0` for `transformers`. -Unchanged from 0.x: `async_encode`/`async_encode_batch` (awaitable; encode -releases the GIL, so they run in a plain worker thread), parity-aware BPE -training (`trainers.ParityBpeTrainer`), and free-threaded Python — default -wheels are abi3 (one binary for CPython 3.10–3.14), and 3.14t gets its own -non-abi3 wheels (`maturin build --no-default-features`). +The full list, including smaller removals and renames, is in the 1.0.0 entry +of [CHANGELOG.md](CHANGELOG.md). + +Kept from 0.x: `async_encode`/`async_encode_batch` and free-threaded Python +support. New in 1.0: parity-aware BPE training across several languages +(`trainers.ParityBpeTrainer`). ## Build and use locally @@ -62,7 +115,10 @@ Rebuild after changing Rust code with `make dev` again (or `maturin develop 10-100× slower and any timing you take from it is meaningless. To build a distributable wheel instead: `maturin build --release` (find it in -`target/wheels/`). +`target/wheels/`). Default wheels use the stable Python ABI (abi3): one +binary per platform covers CPython 3.10–3.14. Free-threaded interpreters +(3.13t/3.14t) cannot load abi3 extensions, so their wheels are built +per-version with `maturin build --no-default-features`. Other targets: @@ -87,7 +143,7 @@ signatures come from the Rust sources; return types that introspection cannot see (numpy arrays, `Self`) are declared with `#[pyo3(signature = (...) -> "Type")]` annotations in the Rust code. -## How it works +## How it works (internals) A `Tokenizer` holds two things behind one lock: @@ -100,7 +156,8 @@ A `Tokenizer` holds two things behind one lock: Every method releases the GIL before touching the lock — enforced at compile time by `DetachedRwLock` (see `src/detached_lock.rs`), with a clippy ban on -`Python::attach` as the backstop. +`Python::attach` as the backstop. Input strings are borrowed, not copied, and +the output arrays take ownership of the Rust buffers, also copy-free. ## Benchmark @@ -113,6 +170,12 @@ before the run counts. Because the released wheel and this build share the package name, the release is installed into `.release/` (`make bench` does this) and benched in a subprocess with `PYTHONPATH` pointing there. +One caveat when quoting numbers: the 0.x side is timed on its fastest API +(`encode_batch_fast`), but it still builds `Encoding` objects, while 1.x +returns bare id arrays — part of the speedup is the new API doing strictly +less output work. That is what a user pays end-to-end, but it is not a +model-algorithm-only comparison. + CI runs it in the `python-bindings-bench` job of the Pipeline Benchmark workflow, posts the table to the run's step summary, and the report job renders it as a chart (`.github/scripts/render_python_bench.py`) appended to diff --git a/bindings/python/examples/02_pretrained.py b/bindings/python/examples/01_pretrained.py similarity index 71% rename from bindings/python/examples/02_pretrained.py rename to bindings/python/examples/01_pretrained.py index 78a57cbe2..5d3ad6951 100644 --- a/bindings/python/examples/02_pretrained.py +++ b/bindings/python/examples/01_pretrained.py @@ -1,8 +1,9 @@ -"""Load real tokenizer.json files and encode a real corpus. Also demonstrates -the two loud failure modes: unsupported pre-tokenizers and unwired -post-processing. (Id parity against the released wheel is checked by -benches/bench_vs_release.py — the released package shares our name, so the -comparison needs two processes.)""" +"""The simplest starting point: load real tokenizer.json files and encode a +real corpus. Also demonstrates the two loud failure modes: pre-tokenizers the +pipeline does not support yet, and post-processing (special-token insertion), +which is not implemented yet. (Id parity against the released wheel is +checked by benches/bench_vs_release.py — the released package shares our +name, so the comparison needs two processes.)""" from pathlib import Path @@ -26,9 +27,9 @@ batch = tok.encode_batch(LINES, add_special_tokens=False) assert all(ids.dtype == np.uint32 for ids in batch) total = sum(len(ids) for ids in batch) - round_trip = tok.id_to_token(int(batch[0][0])) - assert round_trip is not None - print(f"{name}: {total} tokens, first token {round_trip!r}") + first_token = tok.id_to_token(batch[0][0]) + assert first_token is not None + print(f"{name}: {total} tokens, first token {first_token!r}") # Expected failure 1: post-processor would add special tokens -> loud error, # not silently wrong ids diff --git a/bindings/python/examples/01_train_and_encode.py b/bindings/python/examples/02_train_and_encode.py similarity index 88% rename from bindings/python/examples/01_train_and_encode.py rename to bindings/python/examples/02_train_and_encode.py index 93092da6b..26be42d9d 100644 --- a/bindings/python/examples/01_train_and_encode.py +++ b/bindings/python/examples/02_train_and_encode.py @@ -1,5 +1,5 @@ -"""End-to-end: build a tokenizer from scratch, train it, mutate its components -in place, encode, serialize, pickle, and hit the decode stub.""" +"""End-to-end: build a tokenizer from scratch, train it, swap its components, +encode, serialize, pickle, and see that decode raises for now.""" import pickle import tempfile @@ -25,7 +25,8 @@ def corpus(): tok.normalizer = normalizers.Sequence([normalizers.NFKC(), normalizers.Lowercase()]) tok.pre_tokenizer = pre_tokenizers.Whitespace() -# 2. Train from a Python iterator (GIL only taken to refill 256-line buffers) +# 2. Train from a Python iterator. Training runs in Rust threads; the +# interpreter lock is only taken briefly to pull lines from the iterator. trainer = trainers.BpeTrainer( vocab_size=1000, special_tokens=["", "", AddedToken("", special=True)], @@ -41,7 +42,7 @@ def corpus(): print(f"ids: {ids.dtype} {ids}") assert isinstance(ids, np.ndarray) and ids.dtype == np.uint32 assert tok.token_to_id("") in ids -assert all(tok.id_to_token(int(i)) is not None for i in ids) +assert all(tok.id_to_token(i) is not None for i in ids) # 4. Mutate a component in place: dropping the lowercasing normalizer changes ids tok_ids_lower = tok.encode("HELLO WORLD") diff --git a/bindings/python/examples/03_threading.py b/bindings/python/examples/03_threading.py index 130cb920c..0c6864c5b 100644 --- a/bindings/python/examples/03_threading.py +++ b/bindings/python/examples/03_threading.py @@ -1,5 +1,6 @@ -"""Demonstrates that encode runs without the GIL: Python threads calling -encode() scale, and encode_batch parallelizes in Rust via rayon. +"""Demonstrates that encode runs without the interpreter lock (the GIL): +Python threads calling encode() scale, and encode_batch spreads one batch +across Rust's thread pool (rayon). Scaling is asserted unless TOKENIZERS_SCALING_ASSERTS=0 (CI sets it on shared macOS runners, whose noisy hosts make parallel speedup unmeasurable there); diff --git a/bindings/python/examples/04_train_bert_wordpiece.py b/bindings/python/examples/04_train_bert_wordpiece.py index 8bb0ec843..9d9bad690 100644 --- a/bindings/python/examples/04_train_bert_wordpiece.py +++ b/bindings/python/examples/04_train_bert_wordpiece.py @@ -48,4 +48,4 @@ reloaded = Tokenizer.from_file(out) ids = reloaded.encode("Training a WordPiece tokenizer is very easy", add_special_tokens=False) -print([reloaded.id_to_token(int(i)) for i in ids]) +print([reloaded.id_to_token(i) for i in ids]) diff --git a/bindings/python/examples/05_train_bytelevel_bpe.py b/bindings/python/examples/05_train_bytelevel_bpe.py index f16e46977..b1d8ceb57 100644 --- a/bindings/python/examples/05_train_bytelevel_bpe.py +++ b/bindings/python/examples/05_train_bytelevel_bpe.py @@ -45,4 +45,4 @@ reloaded = Tokenizer.from_file(out) ids = reloaded.encode("Training ByteLevel BPE is very easy", add_special_tokens=False) -print([reloaded.id_to_token(int(i)) for i in ids]) +print([reloaded.id_to_token(i) for i in ids]) diff --git a/bindings/python/examples/06_train_with_datasets.py b/bindings/python/examples/06_train_with_datasets.py index a8eaeea03..57b68aa78 100644 --- a/bindings/python/examples/06_train_with_datasets.py +++ b/bindings/python/examples/06_train_with_datasets.py @@ -24,4 +24,4 @@ def texts(batch_size=1000): print(f"trained: {tokenizer.get_vocab_size()} tokens") ids = tokenizer.encode("the quick brown fox", add_special_tokens=False) -print([tokenizer.id_to_token(int(i)) for i in ids]) +print([tokenizer.id_to_token(i) for i in ids]) diff --git a/bindings/python/py_src/tokenizers/__init__.py b/bindings/python/py_src/tokenizers/__init__.py index 003333d64..226208dcd 100644 --- a/bindings/python/py_src/tokenizers/__init__.py +++ b/bindings/python/py_src/tokenizers/__init__.py @@ -1,4 +1,4 @@ -"""Fast tokenizers built on the pipeline encode path. Start with `Tokenizer`.""" +"""Fast tokenizers: turn text into the token ids models consume. Start with `Tokenizer`.""" from ._native import ( AddedToken, diff --git a/bindings/python/py_src/tokenizers/__init__.pyi b/bindings/python/py_src/tokenizers/__init__.pyi index 5498172af..65c6610f5 100644 --- a/bindings/python/py_src/tokenizers/__init__.pyi +++ b/bindings/python/py_src/tokenizers/__init__.pyi @@ -1,18 +1,18 @@ +""" +Fast tokenizers: turn text into the token ids models consume. +Start with `Tokenizer`. +""" + from collections.abc import Coroutine from typing import Any import numpy as np import numpy.typing as npt -""" -Fast tokenizers built on the pipeline encode path. Start with `Tokenizer`. -""" - from tokenizers.models import Model from tokenizers.normalizers import Normalizer from tokenizers.pre_tokenizers import PreTokenizer from tokenizers.trainers import Trainer -from _typeshed import Incomplete from collections.abc import Sequence from os import PathLike from typing import Any, Final, final @@ -183,6 +183,12 @@ class Tokenizer: Rust with the lock released. """ -def __getattr__(name: str) -> Incomplete: ... class TokenizersError(Exception): ... + +from tokenizers import models as models +from tokenizers import normalizers as normalizers +from tokenizers import pre_tokenizers as pre_tokenizers +from tokenizers import trainers as trainers + +__all__ = ["AddedToken", "Tokenizer", "TokenizersError", "__version__", "models", "normalizers", "pre_tokenizers", "trainers"] diff --git a/bindings/python/py_src/tokenizers/models/__init__.pyi b/bindings/python/py_src/tokenizers/models/__init__.pyi index 264f52f51..fe7d1997e 100644 --- a/bindings/python/py_src/tokenizers/models/__init__.pyi +++ b/bindings/python/py_src/tokenizers/models/__init__.pyi @@ -36,7 +36,7 @@ class Unigram(Model): """ The SentencePiece Unigram model: picks the most probable segmentation under a learned piece vocabulary. Starts empty — train it, or load a - tokenizer.json. + trained one with `Tokenizer.from_file`. """ def __new__(cls, /) -> Unigram: ... @@ -56,3 +56,5 @@ class WordPiece(Model): `max_input_chars_per_word` becomes `unk_token` outright. """ def __new__(cls, /, *, unk_token: str = ..., continuing_subword_prefix: str = ..., max_input_chars_per_word: int = 100) -> WordPiece: ... + +__all__ = ["BPE", "Model", "Unigram", "WordLevel", "WordPiece"] diff --git a/bindings/python/py_src/tokenizers/normalizers/__init__.pyi b/bindings/python/py_src/tokenizers/normalizers/__init__.pyi index 04ebfd371..1463e8c4b 100644 --- a/bindings/python/py_src/tokenizers/normalizers/__init__.pyi +++ b/bindings/python/py_src/tokenizers/normalizers/__init__.pyi @@ -95,3 +95,5 @@ class StripAccents(Normalizer): Removes accents (é becomes e). Only works on decomposed text: put NFD before it. """ def __new__(cls, /) -> StripAccents: ... + +__all__ = ["BertNormalizer", "Lowercase", "NFC", "NFD", "NFKC", "NFKD", "Normalizer", "Prepend", "Replace", "Sequence", "Strip", "StripAccents"] diff --git a/bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi b/bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi index d42e35fe9..37a4caa23 100644 --- a/bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi +++ b/bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi @@ -66,7 +66,8 @@ class PreTokenizer: class Punctuation(PreTokenizer): """ Splits on punctuation. `behavior` says what happens to the punctuation - itself — see `Split` for the options. + itself — see `Split` for the options; the default, "isolated", keeps each + punctuation character as its own piece. """ def __new__(cls, /, behavior: str = ...) -> Punctuation: ... @@ -109,3 +110,5 @@ class WhitespaceSplit(PreTokenizer): Splits on whitespace only. """ def __new__(cls, /) -> WhitespaceSplit: ... + +__all__ = ["BertPreTokenizer", "ByteLevel", "CharDelimiterSplit", "Digits", "FixedLength", "PreTokenizer", "Punctuation", "Sequence", "Split", "UnicodeScripts", "Whitespace", "WhitespaceSplit"] diff --git a/bindings/python/py_src/tokenizers/trainers/__init__.pyi b/bindings/python/py_src/tokenizers/trainers/__init__.pyi index 182e68883..38d784af8 100644 --- a/bindings/python/py_src/tokenizers/trainers/__init__.pyi +++ b/bindings/python/py_src/tokenizers/trainers/__init__.pyi @@ -75,3 +75,5 @@ class WordPieceTrainer(Trainer): continuation prefix ("##" by default). """ def __new__(cls, /, *, vocab_size: int = 30000, min_frequency: int = 0, special_tokens: Sequence[str |AddedToken] = ..., limit_alphabet: int |None = None, initial_alphabet: Sequence[str] = ..., continuing_subword_prefix: str = ..., end_of_word_suffix: str |None = None, show_progress: bool = True) -> WordPieceTrainer: ... + +__all__ = ["BpeTrainer", "ParityBpeTrainer", "Trainer", "UnigramTrainer", "WordLevelTrainer", "WordPieceTrainer"] diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml index 7cb58e069..a2fd65908 100644 --- a/bindings/python/pyproject.toml +++ b/bindings/python/pyproject.toml @@ -1,12 +1,37 @@ [build-system] -requires = ["maturin>=1.5,<2.0"] +# 1.8 is the floor for PEP 639 license metadata (license as an SPDX string). +requires = ["maturin>=1.8,<2.0"] build-backend = "maturin" [project] name = "tokenizers" -description = "Fast Python bindings for 🤗 tokenizers, built on the PipelineTokenizer encode path" +description = "Fast, thread-friendly Python bindings for 🤗 tokenizers" readme = "README.md" +license = "Apache-2.0" +license-files = ["LICENSE"] requires-python = ">=3.10" +authors = [ + { name = "Anthony MOI", email = "m.anthony.moi@gmail.com" }, + { name = "Nicolas Patry", email = "patry.nicolas@protonmail.com" }, + { name = "Arthur Zucker", email = "arthur@huggingface.co" }, + { name = "Luc Georges", email = "luc@huggingface.co" }, + { name = "Simon Brandeis", email = "simon@huggingface.co" }, +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: Education", + "Intended Audience :: Science/Research", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3 :: Only", + "Topic :: Scientific/Engineering :: Artificial Intelligence", +] keywords = ["NLP", "tokenizer", "BPE", "transformer", "deep learning"] dependencies = ["numpy>=1.24"] dynamic = ["version"] diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs index 49e31e98f..3f2128313 100644 --- a/bindings/python/src/lib.rs +++ b/bindings/python/src/lib.rs @@ -29,7 +29,8 @@ extern "C" fn child_after_fork() { } } -/// Fast tokenizers built on the pipeline encode path. Start with `Tokenizer`. +/// Fast tokenizers: turn text into the token ids models consume. +/// Start with `Tokenizer`. #[pymodule(gil_used = false)] pub mod _native { use super::*; diff --git a/bindings/python/src/models.rs b/bindings/python/src/models.rs index a202e232a..946d7e90c 100644 --- a/bindings/python/src/models.rs +++ b/bindings/python/src/models.rs @@ -135,7 +135,7 @@ impl PyWordLevel { /// The SentencePiece Unigram model: picks the most probable segmentation /// under a learned piece vocabulary. Starts empty — train it, or load a -/// tokenizer.json. +/// trained one with `Tokenizer.from_file`. #[pyclass(frozen, extends = PyModel, name = "Unigram", module = "tokenizers.models")] pub struct PyUnigram; diff --git a/bindings/python/src/pre_tokenizers.rs b/bindings/python/src/pre_tokenizers.rs index 01cebf531..9277733f5 100644 --- a/bindings/python/src/pre_tokenizers.rs +++ b/bindings/python/src/pre_tokenizers.rs @@ -214,7 +214,8 @@ impl PyFixedLength { } /// Splits on punctuation. `behavior` says what happens to the punctuation -/// itself — see `Split` for the options. +/// itself — see `Split` for the options; the default, "isolated", keeps each +/// punctuation character as its own piece. #[pyclass(frozen, extends = PyPreTokenizer, name = "Punctuation", module = "tokenizers.pre_tokenizers")] pub struct PyPunctuation; diff --git a/bindings/python/src/tokenizer.rs b/bindings/python/src/tokenizer.rs index ac0299891..358563e2e 100644 --- a/bindings/python/src/tokenizer.rs +++ b/bindings/python/src/tokenizer.rs @@ -449,6 +449,7 @@ impl PyTokenizer { ) -> PyResult<()> { let explicit = trainer.map(|t| t.inner.clone()); self.inner.with(py, |lock| { + USED_PARALLELISM.store(true, Ordering::SeqCst); let mut guard = lock.write().map_err(poisoned)?; let mut trainer = explicit.unwrap_or_else(|| guard.spec.get_model().get_trainer()); guard diff --git a/bindings/python/stubtest_allowlist.txt b/bindings/python/stubtest_allowlist.txt new file mode 100644 index 000000000..8e85580e5 --- /dev/null +++ b/bindings/python/stubtest_allowlist.txt @@ -0,0 +1,12 @@ +# Differences between the generated stubs and the runtime that stubtest may +# ignore (`python -m mypy.stubtest tokenizers --allowlist stubtest_allowlist.txt`, +# run by CI and `make test`). Keep this list tiny; explain every entry. +# +# These four are "disjoint bases" (PEP 800): pyo3 classes have their own +# object layout, so no Python class can inherit from two of them. Stub +# introspection cannot emit `@disjoint_base` yet, and the stubs are generated, +# never hand-edited. Type checkers only lose a narrow narrowing refinement. +tokenizers.models.Model +tokenizers.normalizers.Normalizer +tokenizers.pre_tokenizers.PreTokenizer +tokenizers.trainers.Trainer diff --git a/bindings/python/tools/stub-gen/src/main.rs b/bindings/python/tools/stub-gen/src/main.rs index ebb2c96ad..23295bd52 100644 --- a/bindings/python/tools/stub-gen/src/main.rs +++ b/bindings/python/tools/stub-gen/src/main.rs @@ -46,10 +46,18 @@ fn main() -> Result<(), Box> { std::fs::create_dir_all(parent)?; } let mut contents = postprocess(&contents); - if rel_path == Path::new("__init__.pyi") { + let is_root = rel_path == Path::new("__init__.pyi"); + if is_root { // `create_exception!` types carry no introspection metadata. contents.push_str("\nclass TokenizersError(Exception): ...\n"); + // The runtime package re-exports the submodules; mirror that so + // `tokenizers.models` resolves on the stub too. + contents.push('\n'); + for sub in &module.modules { + contents.push_str(&format!("from {MODULE} import {0} as {0}\n", sub.name)); + } } + contents.push_str(&render_all(&module, &rel_path, is_root)); std::fs::write(&out_path, &contents)?; println!("generated {}", out_path.display()); } @@ -77,21 +85,77 @@ fn postprocess(contents: &str) -> String { let mut contents = contents .replace("from . import", &format!("from {MODULE} import")) .replace("from .", &format!("from {MODULE}.")); + // Introspection emits a `__getattr__ -> Incomplete` catch-all, which makes + // the stub non-exhaustive: any removed or misspelled attribute would still + // type-check. Our exports are fully introspected, so drop the escape hatch + // and let type checkers reject unknown names. + contents = contents + .replace("def __getattr__(name: str) -> Incomplete: ...\n", "") + .replace("from _typeshed import Incomplete\n", ""); // Annotated numpy return types need their imports. if contents.contains("npt.") || contents.contains("np.") { - contents = format!( - "import numpy as np\nimport numpy.typing as npt\n\n{contents}" + contents = insert_imports( + &contents, + "import numpy as np\nimport numpy.typing as npt\n", ); } // The async_* annotations reference Coroutine/Any. if contents.contains("Coroutine[") { - contents = format!( - "from collections.abc import Coroutine\nfrom typing import Any\n\n{contents}" + contents = insert_imports( + &contents, + "from collections.abc import Coroutine\nfrom typing import Any\n", ); } contents } +/// Add `imports` to a stub, after the module docstring if there is one — +/// prepending would demote the docstring to a stray string literal. +fn insert_imports(contents: &str, imports: &str) -> String { + const QUOTES: &str = "\"\"\""; + if let Some(body) = contents.strip_prefix(QUOTES) + && let Some(body_len) = body.find(QUOTES) + { + let closing_line_end = QUOTES.len() + body_len + QUOTES.len(); + let doc_end = contents[closing_line_end..] + .find('\n') + .map_or(contents.len(), |nl| closing_line_end + nl + 1); + let (docstring, rest) = contents.split_at(doc_end); + return format!("{docstring}\n{imports}{rest}"); + } + format!("{imports}\n{contents}") +} + +/// `__all__` for one stub, from the introspected module contents — it must +/// match the runtime `__all__` of the shim `__init__.py`, and `stubtest` +/// checks that it does. +fn render_all(root: &pyo3_introspection::model::Module, rel_path: &Path, is_root: bool) -> String { + let module = if is_root { + root + } else { + let stem = rel_path + .file_stem() + .and_then(|s| s.to_str()) + .expect("stub paths are utf-8 files"); + root.modules + .iter() + .find(|m| m.name == stem) + .expect("every submodule stub has an introspected module") + }; + let mut names: Vec<&str> = Vec::new(); + names.extend(module.classes.iter().map(|c| c.name.as_str())); + names.extend(module.functions.iter().map(|f| f.name.as_str())); + names.extend(module.attributes.iter().map(|a| a.name.as_str())); + if is_root { + names.push("TokenizersError"); + names.extend(module.modules.iter().map(|m| m.name.as_str())); + } + names.sort_unstable(); + names.dedup(); + let quoted: Vec = names.iter().map(|n| format!("\"{n}\"")).collect(); + format!("\n__all__ = [{}]\n", quoted.join(", ")) +} + /// Fail loudly if introspection came back without docstrings — that means the /// cdylib was built without `experimental-inspect` (or the feature broke) and /// the stubs would silently lose all documentation. From 68d6dc07967b0c94944866d92d03f4bdc5b8235e Mon Sep 17 00:00:00 2001 From: SBrandeis <33657802+SBrandeis@users.noreply.github.com> Date: Wed, 22 Jul 2026 16:11:44 +0200 Subject: [PATCH 14/19] iteration --- .github/scripts/render_python_bench.py | 4 ++-- .github/workflows/docs-check.yml | 2 +- .github/workflows/python-release.yml | 16 +++------------- bindings/python/.cargo/config.toml | 4 +++- bindings/python/README.md | 4 ++++ bindings/python/benches/bench_vs_release.py | 9 +++++++-- 6 files changed, 20 insertions(+), 19 deletions(-) diff --git a/.github/scripts/render_python_bench.py b/.github/scripts/render_python_bench.py index 347775d2c..237efb10d 100644 --- a/.github/scripts/render_python_bench.py +++ b/.github/scripts/render_python_bench.py @@ -19,8 +19,8 @@ import render_pipeline_bench as rpb -# Same-system series colors: pipeline blue for single-thread, the catalog teal -# for multi-thread (CVD ΔE 16.8 protan, both >= 3:1 on the chart surface). +# Pipeline blue for single-thread, catalog teal for multi-thread — picked to +# stay distinguishable for color-blind readers and keep ≥3:1 contrast. PY_SINK = {"st": "#2a78d6", "mt": "#2a9d8f"} SLUG = "pybindings" diff --git a/.github/workflows/docs-check.yml b/.github/workflows/docs-check.yml index 9e4aa1f9d..ccb829878 100644 --- a/.github/workflows/docs-check.yml +++ b/.github/workflows/docs-check.yml @@ -19,7 +19,7 @@ jobs: python-version: 3.12 - name: Install dependencies - run: pip install sphinx sphinx_rtd_theme setuptools-rust + run: pip install sphinx sphinx_rtd_theme - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml index c257e330f..bbb6feffc 100644 --- a/.github/workflows/python-release.yml +++ b/.github/workflows/python-release.yml @@ -21,7 +21,7 @@ jobs: name: Cargo.lock steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Cargo.lock lock exists + - name: Cargo.lock exists run: cat Cargo.lock working-directory: ./bindings/python @@ -64,10 +64,6 @@ jobs: python-install: "3.14" interpreter: "3.14" smoke: true - # - os: windows - # ls: dir - # target: aarch64 - # interpreter: 3.11 3.12 - os: macos target: aarch64 interpreter: "3.14" @@ -129,16 +125,10 @@ jobs: - { os: macos, target: aarch64, flavor: abi3, smoke: true } - { os: windows, target: x86_64, flavor: abi3, smoke: true } exclude: + # Built by the windows-11-arm include cell instead (needs an arm + # runner; the plain windows runner is x64). - os: windows target: aarch64 - # # Optimized PGO builds for x86_64 manylinux and windows follow a different matrix, - # # maybe in future maturin-action can support this automatically - # - os: ubuntu - # target: x86_64 - # manylinux: auto - # - os: windows - # target: x86_64 - # Windows on arm64 only supports Python 3.11+ runs-on: ${{ matrix.os == 'windows-11-arm' && matrix.os || diff --git a/bindings/python/.cargo/config.toml b/bindings/python/.cargo/config.toml index fbfc5de16..90cbdbee4 100644 --- a/bindings/python/.cargo/config.toml +++ b/bindings/python/.cargo/config.toml @@ -1,4 +1,6 @@ -# Required flags on MacOS to defer resolution of the CPython symbols +# macOS: leave CPython symbols unresolved at link time — they come from the +# Python process that loads the extension. maturin passes these flags itself; +# this file covers plain `cargo build` / `cargo clippy`. [target.x86_64-apple-darwin] rustflags = [ "-C", "link-arg=-undefined", diff --git a/bindings/python/README.md b/bindings/python/README.md index 41a5ce676..6025f18b5 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -143,6 +143,10 @@ signatures come from the Rust sources; return types that introspection cannot see (numpy arrays, `Self`) are declared with `#[pyo3(signature = (...) -> "Type")]` annotations in the Rust code. +CI enforces this twice: the committed stubs must match what stub-gen +produces, and `mypy.stubtest` checks them against the actual runtime +(accepted differences are listed and explained in `stubtest_allowlist.txt`). + ## How it works (internals) A `Tokenizer` holds two things behind one lock: diff --git a/bindings/python/benches/bench_vs_release.py b/bindings/python/benches/bench_vs_release.py index 577d9c8e2..3fa70cb14 100644 --- a/bindings/python/benches/bench_vs_release.py +++ b/bindings/python/benches/bench_vs_release.py @@ -60,7 +60,7 @@ def make_chunks(text: str) -> list[str]: for line in text.splitlines(): if not line.strip(): continue - cur_bytes += len(line.encode()) + bool(cur) + cur_bytes += len(line.encode()) + bool(cur) # +1 for the joining "\n" cur.append(line) if cur_bytes >= CHUNK_BYTES: chunks.append("\n".join(cur)) @@ -227,7 +227,12 @@ def main() -> int: models = json.load(open(args.manifest)) if args.manifest else DEFAULT_MODELS for model in models: model["path"] = str(args.data_dir / model.get("file", model["name"] + ".json")) - models = [m for m in models if Path(m["path"]).is_file()] or sys.exit("no model files found") + missing = [m["name"] for m in models if not Path(m["path"]).is_file()] + if missing: + print(f"skipping models with no tokenizer file: {', '.join(missing)}", file=sys.stderr) + models = [m for m in models if m["name"] not in missing] + if not models: + sys.exit("no model files found") fixtures = load_fixtures(args.data_dir) From d888aa261f557dc2481850674cefc67a77d675be Mon Sep 17 00:00:00 2001 From: SBrandeis <33657802+SBrandeis@users.noreply.github.com> Date: Wed, 22 Jul 2026 17:12:24 +0200 Subject: [PATCH 15/19] rename encode -> encode_ids --- README.md | 4 +- bindings/python/CHANGELOG.md | 19 ++++++---- bindings/python/README.md | 37 +++++++++++-------- bindings/python/benches/bench_vs_release.py | 8 ++-- bindings/python/examples/01_pretrained.py | 6 +-- .../python/examples/02_train_and_encode.py | 14 +++---- bindings/python/examples/03_threading.py | 18 ++++----- .../examples/04_train_bert_wordpiece.py | 2 +- .../python/examples/05_train_bytelevel_bpe.py | 2 +- .../python/examples/06_train_with_datasets.py | 2 +- .../python/py_src/tokenizers/__init__.pyi | 34 +++++++++-------- bindings/python/src/tokenizer.rs | 28 +++++++------- bindings/python/tests/test_async.py | 14 +++---- bindings/python/tests/test_components.py | 10 ++--- bindings/python/tests/test_parity_trainer.py | 2 +- bindings/python/tests/test_pretrained.py | 10 ++--- bindings/python/tests/test_threading.py | 10 ++--- bindings/python/tests/test_tokenizer.py | 26 ++++++------- bindings/python/tests/test_trainers.py | 2 +- 19 files changed, 131 insertions(+), 117 deletions(-) diff --git a/README.md b/README.md index bd735d36f..2115235dd 100644 --- a/README.md +++ b/README.md @@ -86,12 +86,12 @@ tokenizer.train(files=["wiki.train.raw", "wiki.valid.raw", "wiki.test.raw"], tra Once your tokenizer is trained, encode any text with just one line: ```python -ids = tokenizer.encode("Hello, y'all! How are you 😁 ?") +ids = tokenizer.encode_ids("Hello, y'all! How are you 😁 ?") print([tokenizer.id_to_token(i) for i in ids]) # ["Hello", ",", "y", "'", "all", "!", "How", "are", "you", "[UNK]", "?"] ``` -`encode` returns the token ids as a `numpy.uint32` array — ready to hand to +`encode_ids` returns the token ids as a `numpy.uint32` array — ready to hand to your model with no further conversion. The emoji comes out as `[UNK]`: it never appeared in the training files, so it is not in the vocabulary, and BPE falls back to the `unk_token` we configured above. diff --git a/bindings/python/CHANGELOG.md b/bindings/python/CHANGELOG.md index 5fbca3bbc..a1434337a 100644 --- a/bindings/python/CHANGELOG.md +++ b/bindings/python/CHANGELOG.md @@ -14,13 +14,16 @@ arrays without a copy. Breaking changes — encoding: -- `encode` returns a numpy array of ids, not an `Encoding` object. Everything - the `Encoding` carried is gone from the encode path: tokens, offsets, type - ids, attention masks, special-tokens masks, word ids, overflowing/stride, - and the char/word/token mapping helpers. Truncation and padding +- Encoding returns a numpy array of ids, not an `Encoding` object, and the + methods are accordingly named `encode_ids`/`encode_batch_ids` — the + `encode`/`encode_batch` names are reserved for a planned + `Encoding`-returning API. Everything the `Encoding` carried is gone from + the encode path: tokens, offsets, type ids, attention masks, + special-tokens masks, word ids, overflowing/stride, and the + char/word/token mapping helpers. Truncation and padding (`enable_truncation`/`enable_padding` and their getters) are gone too. -- `encode` takes a single text: the `pair=` argument and the - `is_pretokenized=` mode no longer exist (same for `encode_batch`). +- `encode_ids` takes a single text: the `pair=` argument and the + `is_pretokenized=` mode no longer exist (same for `encode_batch_ids`). - Not implemented yet (loud errors, never wrong ids): `decode`, post-processor templates (pass `add_special_tokens=False`), and the `Metaspace` pre-tokenizer. `decode_batch`, `DecodeStream`, @@ -64,8 +67,8 @@ Breaking changes — `Tokenizer` API and packaging: moved from a required dependency to the `hub` extra: `pip install 'tokenizers[hub]'`. `numpy>=1.24` is a new required dependency. -Kept: `async_encode`/`async_encode_batch` (now a thin `asyncio.to_thread` -wrapper — no tokio runtime) and free-threaded Python support (default wheels +Kept: awaitable encodes, as `async_encode_ids`/`async_encode_batch_ids` +(now a thin `asyncio.to_thread` wrapper — no tokio runtime) and free-threaded Python support (default wheels are abi3-py310; free-threaded interpreters get their own non-abi3 wheels). New in 1.0: parity-aware BPE training (`trainers.ParityBpeTrainer`), which never shipped in a 0.x release. diff --git a/bindings/python/README.md b/bindings/python/README.md index 6025f18b5..6d2a37ce4 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -18,12 +18,17 @@ tok = tk.Tokenizer.from_file("tokenizer.json") # add_special_tokens=False skips template tokens like [CLS]/[SEP]; inserting # them is not implemented yet in 1.0, so leaving it True raises a loud # NotImplementedError on tokenizers that use such templates (BERT, Llama, …). -ids = tok.encode("Hello world", add_special_tokens=False) +ids = tok.encode_ids("Hello world", add_special_tokens=False) # A list of arrays, encoded in parallel across Rust threads. -batch = tok.encode_batch(["Hello world", "How are you?"], add_special_tokens=False) +batch = tok.encode_batch_ids(["Hello world", "How are you?"], add_special_tokens=False) ``` +The methods are named `encode_ids`/`encode_batch_ids` because they return +bare ids: the `encode`/`encode_batch` names are reserved for a planned +`Encoding`-returning API (masks, word ids, truncation/padding — the interface +`transformers` consumes). + To load a tokenizer straight from the [Hugging Face Hub](https://huggingface.co), install the `hub` extra (`pip install 'tokenizers[hub]'`): @@ -58,28 +63,30 @@ simplest case (load a pretrained file and encode). Encoding releases the Python interpreter lock (the GIL), so this package plays well with threads and event loops: -- Calling `encode` from several Python threads scales — the threads really - run in parallel, on every interpreter (free-threaded or not). -- `encode_batch` parallelizes one batch across Rust threads. Set the +- Calling `encode_ids` from several Python threads scales — the threads + really run in parallel, on every interpreter (free-threaded or not). +- `encode_batch_ids` parallelizes one batch across Rust threads. Set the `TOKENIZERS_PARALLELISM` environment variable to `false`/`true` to disable/force this. -- In `asyncio` code, `await tok.async_encode(text)` / - `await tok.async_encode_batch(texts)` keep the event loop free while Rust - encodes in a worker thread. +- In `asyncio` code, `await tok.async_encode_ids(text)` / + `await tok.async_encode_batch_ids(texts)` keep the event loop free while + Rust encodes in a worker thread. ```python -ids = await tok.async_encode("Hello world", add_special_tokens=False) +ids = await tok.async_encode_ids("Hello world", add_special_tokens=False) ``` ## Breaking changes vs 0.x 1.x is a ground-up rewrite with a smaller, faster API. The headline changes: -- `encode` returns a `numpy.uint32` array of ids, not an `Encoding` object. +- Encoding returns a `numpy.uint32` array of ids, not an `Encoding` object, + and the methods are accordingly named `encode_ids`/`encode_batch_ids`. Tokens, offsets, type ids, attention masks, and word ids are gone from the encode path, and so are truncation and padding (`enable_truncation`/`enable_padding`). -- `encode` takes a single text: no `pair=` argument, no `is_pretokenized=`. +- `encode_ids` takes a single text: no `pair=` argument, no + `is_pretokenized=`. - Not implemented yet (loud errors, never wrong ids): `decode`, post-processor templates (`[CLS]`/`` insertion — pass `add_special_tokens=False`), and the `Metaspace` pre-tokenizer @@ -95,9 +102,9 @@ ids = await tok.async_encode("Hello world", add_special_tokens=False) The full list, including smaller removals and renames, is in the 1.0.0 entry of [CHANGELOG.md](CHANGELOG.md). -Kept from 0.x: `async_encode`/`async_encode_batch` and free-threaded Python -support. New in 1.0: parity-aware BPE training across several languages -(`trainers.ParityBpeTrainer`). +Kept from 0.x: awaitable encodes (`async_encode_ids`/`async_encode_batch_ids`) +and free-threaded Python support. New in 1.0: parity-aware BPE training across +several languages (`trainers.ParityBpeTrainer`). ## Build and use locally @@ -165,7 +172,7 @@ the output arrays take ownership of the Rust buffers, also copy-free. ## Benchmark -`benches/bench_vs_release.py` times `encode_batch` end-to-end through Python +`benches/bench_vs_release.py` times batch encoding end-to-end through Python against the latest released `tokenizers` wheel, on the same corpora and ~10 KiB chunking as the Rust benchmark (`tk-encode/examples/fixture_bench.rs`): every fixture under `data/fixtures/{lang,modalities}`, warmed up, median of N runs, diff --git a/bindings/python/benches/bench_vs_release.py b/bindings/python/benches/bench_vs_release.py index 3fa70cb14..d974b15be 100644 --- a/bindings/python/benches/bench_vs_release.py +++ b/bindings/python/benches/bench_vs_release.py @@ -135,8 +135,8 @@ def bench_local_side(tok, fixtures: list[dict], release_row: dict, iters: int) - os.environ["TOKENIZERS_PARALLELISM"] = "false" for fixture, rel in zip(fixtures, release_row["fixtures"], strict=True): chunks = fixture["chunks"] - encoded = tok.encode_batch(chunks, add_special_tokens=False) - t = timed(lambda: tok.encode_batch(chunks, add_special_tokens=False), iters) + encoded = tok.encode_batch_ids(chunks, add_special_tokens=False) + t = timed(lambda: tok.encode_batch_ids(chunks, add_special_tokens=False), iters) mbps = fixture["bytes"] / t / 1e6 row["fixtures"].append( { @@ -154,7 +154,7 @@ def bench_local_side(tok, fixtures: list[dict], release_row: dict, iters: int) - all_chunks = [c for f in fixtures for c in f["chunks"]] nbytes = sum(f["bytes"] for f in fixtures) os.environ["TOKENIZERS_PARALLELISM"] = "true" - t = timed(lambda: tok.encode_batch(all_chunks, add_special_tokens=False), iters) + t = timed(lambda: tok.encode_batch_ids(all_chunks, add_special_tokens=False), iters) mbps = nbytes / t / 1e6 row["multi_thread"] = { "bytes": nbytes, @@ -241,7 +241,7 @@ def main() -> int: for m in models: try: tok = tokenizers.Tokenizer.from_file(m["path"]) - tok.encode("warmup", add_special_tokens=False) + tok.encode_ids("warmup", add_special_tokens=False) compiled[m["name"]] = tok except (tokenizers.TokenizersError, NotImplementedError) as e: skipped[m["name"]] = str(e) diff --git a/bindings/python/examples/01_pretrained.py b/bindings/python/examples/01_pretrained.py index 5d3ad6951..ddf667919 100644 --- a/bindings/python/examples/01_pretrained.py +++ b/bindings/python/examples/01_pretrained.py @@ -24,7 +24,7 @@ ("bert-base-uncased", "bert-base-uncased.json"), ]: tok = Tokenizer.from_file(DATA / file) - batch = tok.encode_batch(LINES, add_special_tokens=False) + batch = tok.encode_batch_ids(LINES, add_special_tokens=False) assert all(ids.dtype == np.uint32 for ids in batch) total = sum(len(ids) for ids in batch) first_token = tok.id_to_token(batch[0][0]) @@ -35,7 +35,7 @@ # not silently wrong ids bert = Tokenizer.from_file(DATA / "bert-base-uncased.json") try: - bert.encode("hello") + bert.encode_ids("hello") raise AssertionError("should have raised") except NotImplementedError as e: print(f"bert with add_special_tokens=True: NotImplementedError({e})") @@ -44,7 +44,7 @@ # at compile time, with the reason t5 = Tokenizer.from_file(DATA / "t5-base.json") try: - t5.encode("hello", add_special_tokens=False) + t5.encode_ids("hello", add_special_tokens=False) raise AssertionError("should have raised") except TokenizersError as e: print(f"t5-base (Metaspace): TokenizersError({e})") diff --git a/bindings/python/examples/02_train_and_encode.py b/bindings/python/examples/02_train_and_encode.py index 26be42d9d..4a18da50d 100644 --- a/bindings/python/examples/02_train_and_encode.py +++ b/bindings/python/examples/02_train_and_encode.py @@ -38,25 +38,25 @@ def corpus(): assert tok.token_to_id("") == 0 # 3. Encode -> numpy uint32 array; special tokens are matched in the text -ids = tok.encode("The quick brown fox jumps over the lazy dog") +ids = tok.encode_ids("The quick brown fox jumps over the lazy dog") print(f"ids: {ids.dtype} {ids}") assert isinstance(ids, np.ndarray) and ids.dtype == np.uint32 assert tok.token_to_id("") in ids assert all(tok.id_to_token(i) is not None for i in ids) # 4. Mutate a component in place: dropping the lowercasing normalizer changes ids -tok_ids_lower = tok.encode("HELLO WORLD") +tok_ids_lower = tok.encode_ids("HELLO WORLD") tok.normalizer = normalizers.NFKC() -tok_ids_upper = tok.encode("HELLO WORLD") +tok_ids_upper = tok.encode_ids("HELLO WORLD") assert not np.array_equal(tok_ids_lower, tok_ids_upper), "normalizer change must affect ids" tok.normalizer = normalizers.Sequence([normalizers.NFKC(), normalizers.Lowercase()]) -assert np.array_equal(tok.encode("HELLO WORLD"), tok_ids_lower) +assert np.array_equal(tok.encode_ids("HELLO WORLD"), tok_ids_lower) print(f"component swap: {tok.normalizer!r}") # 5. Post-hoc vocabulary extension added = tok.add_special_tokens([""]) assert added == 1 and tok.token_to_id("") is not None -assert tok.token_to_id("") in tok.encode("a b") +assert tok.token_to_id("") in tok.encode_ids("a b") # 6. Serialize / reload round-trip with tempfile.TemporaryDirectory() as tmp: @@ -64,12 +64,12 @@ def corpus(): tok.save(path) reloaded = Tokenizer.from_file(path) text = "Round-trip: 42 tokens?" -assert np.array_equal(tok.encode(text), reloaded.encode(text)) +assert np.array_equal(tok.encode_ids(text), reloaded.encode_ids(text)) print("save/load round-trip: identical ids") # 7. Pickle round-trip (multiprocessing readiness) unpickled = pickle.loads(pickle.dumps(tok)) -assert np.array_equal(tok.encode(text), unpickled.encode(text)) +assert np.array_equal(tok.encode_ids(text), unpickled.encode_ids(text)) print("pickle round-trip: identical ids") # 8. decode is not implemented yet — it raises instead of guessing diff --git a/bindings/python/examples/03_threading.py b/bindings/python/examples/03_threading.py index 0c6864c5b..1bfa9d01f 100644 --- a/bindings/python/examples/03_threading.py +++ b/bindings/python/examples/03_threading.py @@ -1,5 +1,5 @@ """Demonstrates that encode runs without the interpreter lock (the GIL): -Python threads calling encode() scale, and encode_batch spreads one batch +Python threads calling encode_ids() scale, and encode_batch_ids spreads one batch across Rust's thread pool (rayon). Scaling is asserted unless TOKENIZERS_SCALING_ASSERTS=0 (CI sets it on shared @@ -40,11 +40,11 @@ def best_of(n, fn): text = f.read(2_000_000) lines = [line for line in text.splitlines() if line.strip()] * 4 -tok.encode(text, add_special_tokens=False) # warmup + compile +tok.encode_ids(text, add_special_tokens=False) # warmup + compile def encode_once(): - return tok.encode(text, add_special_tokens=False) + return tok.encode_ids(text, add_special_tokens=False) # 1. Python threads: with the GIL held during encode this could not scale @@ -61,19 +61,19 @@ def encode_once(): ) check_scaling(speedup > 1.5, f"threads did not scale ({speedup:.2f}x): is the GIL held?") -# 2. encode_batch: rayon parallelism inside one call, toggled by env var +# 2. encode_batch_ids: rayon parallelism inside one call, toggled by env var os.environ["TOKENIZERS_PARALLELISM"] = "false" -serial_ids = tok.encode_batch(lines, add_special_tokens=False) -serial = best_of(3, lambda: tok.encode_batch(lines, add_special_tokens=False)) +serial_ids = tok.encode_batch_ids(lines, add_special_tokens=False) +serial = best_of(3, lambda: tok.encode_batch_ids(lines, add_special_tokens=False)) os.environ["TOKENIZERS_PARALLELISM"] = "true" -parallel_ids = tok.encode_batch(lines, add_special_tokens=False) # warmup: spins up the pool -parallel = best_of(3, lambda: tok.encode_batch(lines, add_special_tokens=False)) +parallel_ids = tok.encode_batch_ids(lines, add_special_tokens=False) # warmup: spins up the pool +parallel = best_of(3, lambda: tok.encode_batch_ids(lines, add_special_tokens=False)) assert all(a.tolist() == b.tolist() for a, b in zip(serial_ids, parallel_ids, strict=True)) mbps = sum(len(line) for line in lines) / parallel / 1e6 print( - f"encode_batch {len(lines)} lines: serial {serial:.2f}s, " + f"encode_batch_ids {len(lines)} lines: serial {serial:.2f}s, " f"rayon {parallel:.2f}s ({serial / parallel:.1f}x, {mbps:.0f} MB/s)" ) check_scaling(serial / parallel > 1.5, f"rayon batch did not scale ({serial / parallel:.2f}x)") diff --git a/bindings/python/examples/04_train_bert_wordpiece.py b/bindings/python/examples/04_train_bert_wordpiece.py index 9d9bad690..cf3e29966 100644 --- a/bindings/python/examples/04_train_bert_wordpiece.py +++ b/bindings/python/examples/04_train_bert_wordpiece.py @@ -47,5 +47,5 @@ print(f"saved {out} ({tokenizer.get_vocab_size()} tokens)") reloaded = Tokenizer.from_file(out) -ids = reloaded.encode("Training a WordPiece tokenizer is very easy", add_special_tokens=False) +ids = reloaded.encode_ids("Training a WordPiece tokenizer is very easy", add_special_tokens=False) print([reloaded.id_to_token(i) for i in ids]) diff --git a/bindings/python/examples/05_train_bytelevel_bpe.py b/bindings/python/examples/05_train_bytelevel_bpe.py index b1d8ceb57..8bdabd98e 100644 --- a/bindings/python/examples/05_train_bytelevel_bpe.py +++ b/bindings/python/examples/05_train_bytelevel_bpe.py @@ -44,5 +44,5 @@ print(f"saved {out} ({tokenizer.get_vocab_size()} tokens)") reloaded = Tokenizer.from_file(out) -ids = reloaded.encode("Training ByteLevel BPE is very easy", add_special_tokens=False) +ids = reloaded.encode_ids("Training ByteLevel BPE is very easy", add_special_tokens=False) print([reloaded.id_to_token(i) for i in ids]) diff --git a/bindings/python/examples/06_train_with_datasets.py b/bindings/python/examples/06_train_with_datasets.py index 57b68aa78..cc2ad3e0d 100644 --- a/bindings/python/examples/06_train_with_datasets.py +++ b/bindings/python/examples/06_train_with_datasets.py @@ -23,5 +23,5 @@ def texts(batch_size=1000): tokenizer.train_from_iterator(texts()) print(f"trained: {tokenizer.get_vocab_size()} tokens") -ids = tokenizer.encode("the quick brown fox", add_special_tokens=False) +ids = tokenizer.encode_ids("the quick brown fox", add_special_tokens=False) print([tokenizer.id_to_token(i) for i in ids]) diff --git a/bindings/python/py_src/tokenizers/__init__.pyi b/bindings/python/py_src/tokenizers/__init__.pyi index 65c6610f5..0a1f6c00c 100644 --- a/bindings/python/py_src/tokenizers/__init__.pyi +++ b/bindings/python/py_src/tokenizers/__init__.pyi @@ -73,36 +73,38 @@ class Tokenizer: on. Plain strings match with default options; pass `AddedToken` to control matching. Returns how many were actually new. """ - def async_encode(self, /, text: Any, *, add_special_tokens: bool = True) -> "Coroutine[Any, Any, npt.NDArray[np.uint32]]": + def async_encode_batch_ids(self, /, texts: Any, *, add_special_tokens: bool = True) -> "Coroutine[Any, Any, list[npt.NDArray[np.uint32]]]": """ - Awaitable `encode`: same arguments and result, run in a worker thread - (`asyncio.to_thread`) so the event loop stays free. The thread releases - the interpreter lock while Rust encodes, so encodes genuinely overlap. + Awaitable `encode_batch_ids`: same arguments and result, run in a + worker thread (`asyncio.to_thread`) so the event loop stays free while + the batch encodes on Rust threads. """ - def async_encode_batch(self, /, texts: Any, *, add_special_tokens: bool = True) -> "Coroutine[Any, Any, list[npt.NDArray[np.uint32]]]": + def async_encode_ids(self, /, text: Any, *, add_special_tokens: bool = True) -> "Coroutine[Any, Any, npt.NDArray[np.uint32]]": """ - Awaitable `encode_batch`: same arguments and result, run in a worker - thread (`asyncio.to_thread`) so the event loop stays free while the - batch encodes on Rust threads. + Awaitable `encode_ids`: same arguments and result, run in a worker + thread (`asyncio.to_thread`) so the event loop stays free. The thread + releases the interpreter lock while Rust encodes, so encodes genuinely + overlap. """ def decode(self, /, ids: Sequence[int], *, skip_special_tokens: bool = True) -> str: """ Not implemented yet: decoding is not part of the encode pipeline. """ - def encode(self, /, text: str, *, add_special_tokens: bool = True) -> "npt.NDArray[np.uint32]": - """ - Encode `text` into token ids. - - Runs entirely outside the interpreter lock and returns a `numpy.uint32` - array backed by the Rust output buffer (no copy). - """ - def encode_batch(self, /, texts: Sequence[str], *, add_special_tokens: bool = True) -> "list[npt.NDArray[np.uint32]]": + def encode_batch_ids(self, /, texts: Sequence[str], *, add_special_tokens: bool = True) -> "list[npt.NDArray[np.uint32]]": """ Encode a batch of texts, in parallel across Rust threads (respects `TOKENIZERS_PARALLELISM`), without holding the interpreter lock. Input strings are borrowed, not copied; each output is a `numpy.uint32` array backed by its Rust buffer. """ + def encode_ids(self, /, text: str, *, add_special_tokens: bool = True) -> "npt.NDArray[np.uint32]": + """ + Encode `text` into token ids. + + Runs entirely outside the interpreter lock and returns a `numpy.uint32` + array backed by the Rust output buffer (no copy). The `encode` name is + reserved for the upcoming `Encoding`-returning API. + """ @staticmethod def from_buffer(buffer: Sequence[int]) -> "Tokenizer": """ diff --git a/bindings/python/src/tokenizer.rs b/bindings/python/src/tokenizer.rs index 358563e2e..7554f1a15 100644 --- a/bindings/python/src/tokenizer.rs +++ b/bindings/python/src/tokenizer.rs @@ -346,9 +346,10 @@ impl PyTokenizer { /// Encode `text` into token ids. /// /// Runs entirely outside the interpreter lock and returns a `numpy.uint32` - /// array backed by the Rust output buffer (no copy). + /// array backed by the Rust output buffer (no copy). The `encode` name is + /// reserved for the upcoming `Encoding`-returning API. #[pyo3(signature = (text, *, add_special_tokens = true) -> "npt.NDArray[np.uint32]")] - fn encode<'py>( + fn encode_ids<'py>( &self, py: Python<'py>, text: &str, @@ -369,7 +370,7 @@ impl PyTokenizer { /// Input strings are borrowed, not copied; each output is a `numpy.uint32` /// array backed by its Rust buffer. #[pyo3(signature = (texts, *, add_special_tokens = true) -> "list[npt.NDArray[np.uint32]]")] - fn encode_batch<'py>( + fn encode_batch_ids<'py>( &self, py: Python<'py>, texts: Vec, @@ -405,28 +406,29 @@ impl PyTokenizer { Ok(list) } - /// Awaitable `encode`: same arguments and result, run in a worker thread - /// (`asyncio.to_thread`) so the event loop stays free. The thread releases - /// the interpreter lock while Rust encodes, so encodes genuinely overlap. + /// Awaitable `encode_ids`: same arguments and result, run in a worker + /// thread (`asyncio.to_thread`) so the event loop stays free. The thread + /// releases the interpreter lock while Rust encodes, so encodes genuinely + /// overlap. #[pyo3(signature = (text, *, add_special_tokens = true) -> "Coroutine[Any, Any, npt.NDArray[np.uint32]]")] - fn async_encode<'py>( + fn async_encode_ids<'py>( slf: &Bound<'py, Self>, text: &Bound<'py, PyAny>, add_special_tokens: bool, ) -> PyResult> { - to_thread(slf, "encode", text, add_special_tokens) + to_thread(slf, "encode_ids", text, add_special_tokens) } - /// Awaitable `encode_batch`: same arguments and result, run in a worker - /// thread (`asyncio.to_thread`) so the event loop stays free while the - /// batch encodes on Rust threads. + /// Awaitable `encode_batch_ids`: same arguments and result, run in a + /// worker thread (`asyncio.to_thread`) so the event loop stays free while + /// the batch encodes on Rust threads. #[pyo3(signature = (texts, *, add_special_tokens = true) -> "Coroutine[Any, Any, list[npt.NDArray[np.uint32]]]")] - fn async_encode_batch<'py>( + fn async_encode_batch_ids<'py>( slf: &Bound<'py, Self>, texts: &Bound<'py, PyAny>, add_special_tokens: bool, ) -> PyResult> { - to_thread(slf, "encode_batch", texts, add_special_tokens) + to_thread(slf, "encode_batch_ids", texts, add_special_tokens) } /// Not implemented yet: decoding is not part of the encode pipeline. diff --git a/bindings/python/tests/test_async.py b/bindings/python/tests/test_async.py index af1e54538..bc8a11d19 100644 --- a/bindings/python/tests/test_async.py +++ b/bindings/python/tests/test_async.py @@ -10,13 +10,13 @@ def test_async_encode_matches_sync(): tok = train_word_tokenizer() async def go(): - single = await tok.async_encode(SENTENCES[0], add_special_tokens=False) - batch = await tok.async_encode_batch(SENTENCES, add_special_tokens=False) + single = await tok.async_encode_ids(SENTENCES[0], add_special_tokens=False) + batch = await tok.async_encode_batch_ids(SENTENCES, add_special_tokens=False) return single, batch single, batch = asyncio.run(go()) - assert np.array_equal(single, tok.encode(SENTENCES[0], add_special_tokens=False)) - for got, want in zip(batch, tok.encode_batch(SENTENCES, add_special_tokens=False)): + assert np.array_equal(single, tok.encode_ids(SENTENCES[0], add_special_tokens=False)) + for got, want in zip(batch, tok.encode_batch_ids(SENTENCES, add_special_tokens=False)): assert np.array_equal(got, want) @@ -24,18 +24,18 @@ def test_async_encodes_overlap(): tok = train_word_tokenizer() async def go(): - return await asyncio.gather(*(tok.async_encode(s, add_special_tokens=False) for s in SENTENCES)) + return await asyncio.gather(*(tok.async_encode_ids(s, add_special_tokens=False) for s in SENTENCES)) results = asyncio.run(go()) for got, line in zip(results, SENTENCES): - assert np.array_equal(got, tok.encode(line, add_special_tokens=False)) + assert np.array_equal(got, tok.encode_ids(line, add_special_tokens=False)) def test_async_error_surfaces_at_await(): tok = train_word_tokenizer() async def go(): - await tok.async_encode(123, add_special_tokens=False) + await tok.async_encode_ids(123, add_special_tokens=False) with pytest.raises(TypeError): asyncio.run(go()) diff --git a/bindings/python/tests/test_components.py b/bindings/python/tests/test_components.py index 01a6e137e..aa723c2d7 100644 --- a/bindings/python/tests/test_components.py +++ b/bindings/python/tests/test_components.py @@ -67,11 +67,11 @@ def test_byte_level_alphabet(): def test_lowercase_normalizer_changes_ids(): tok = train_word_tokenizer() assert tok.token_to_id("THE") is None - before = tok.encode("THE", add_special_tokens=False) + before = tok.encode_ids("THE", add_special_tokens=False) assert [tok.id_to_token(int(i)) for i in before] == ["[UNK]"] tok.normalizer = normalizers.Lowercase() - after = tok.encode("THE", add_special_tokens=False) + after = tok.encode_ids("THE", add_special_tokens=False) assert [tok.id_to_token(int(i)) for i in after] == ["the"] @@ -79,15 +79,15 @@ def test_char_delimiter_split_effect(): tok = Tokenizer(models.WordLevel(unk_token="[UNK]")) tok.pre_tokenizer = pre_tokenizers.CharDelimiterSplit(",") tok.train_from_iterator(["a,b", "b,c"], trainer=trainers.WordLevelTrainer(special_tokens=["[UNK]"])) - ids = tok.encode("a,c", add_special_tokens=False) + ids = tok.encode_ids("a,c", add_special_tokens=False) assert [tok.id_to_token(int(i)) for i in ids] == ["a", "c"] def test_component_assignment_invalidates_pipeline(): tok = train_word_tokenizer() - the_id = tok.encode("the", add_special_tokens=False) + the_id = tok.encode_ids("the", add_special_tokens=False) tok.pre_tokenizer = pre_tokenizers.FixedLength(length=1) - per_char = tok.encode("the", add_special_tokens=False) + per_char = tok.encode_ids("the", add_special_tokens=False) assert len(per_char) == 3 assert not np.array_equal(the_id, per_char) diff --git a/bindings/python/tests/test_parity_trainer.py b/bindings/python/tests/test_parity_trainer.py index 25717e68e..ba612b741 100644 --- a/bindings/python/tests/test_parity_trainer.py +++ b/bindings/python/tests/test_parity_trainer.py @@ -18,7 +18,7 @@ def test_trains_with_dev_iterators(): trainer.train_from_iterator(tok, [iter(EN), iter(DE)], dev_iterators=[iter(EN[:6]), iter(DE[:6])]) assert tok.token_to_id("") is not None for line in (EN[0], DE[0]): - ids = tok.encode(line, add_special_tokens=False) + ids = tok.encode_ids(line, add_special_tokens=False) assert len(ids) > 0 assert all(tok.id_to_token(int(i)) is not None for i in ids) diff --git a/bindings/python/tests/test_pretrained.py b/bindings/python/tests/test_pretrained.py index ebf0d4ac6..65a505e74 100644 --- a/bindings/python/tests/test_pretrained.py +++ b/bindings/python/tests/test_pretrained.py @@ -6,7 +6,7 @@ def test_gpt2_encodes_corpus(gpt2_file, corpus): tok = Tokenizer.from_file(gpt2_file) - batch = tok.encode_batch(corpus, add_special_tokens=False) + batch = tok.encode_batch_ids(corpus, add_special_tokens=False) assert sum(len(ids) for ids in batch) > 0 assert all(ids.dtype == np.uint32 for ids in batch) @@ -14,19 +14,19 @@ def test_gpt2_encodes_corpus(gpt2_file, corpus): def test_bert_special_tokens_gate(bert_file): tok = Tokenizer.from_file(bert_file) with pytest.raises(NotImplementedError, match="post-process"): - tok.encode("hello") - ids = tok.encode("hello", add_special_tokens=False) + tok.encode_ids("hello") + ids = tok.encode_ids("hello", add_special_tokens=False) assert len(ids) > 0 def test_metaspace_fails_loudly_at_compile(t5_file): tok = Tokenizer.from_file(t5_file) with pytest.raises(TokenizersError, match="Metaspace"): - tok.encode("hello", add_special_tokens=False) + tok.encode_ids("hello", add_special_tokens=False) @pytest.mark.network def test_from_pretrained(): tok = Tokenizer.from_pretrained("bert-base-uncased") - ids = tok.encode("hello world", add_special_tokens=False) + ids = tok.encode_ids("hello world", add_special_tokens=False) assert [tok.id_to_token(int(i)) for i in ids] == ["hello", "world"] diff --git a/bindings/python/tests/test_threading.py b/bindings/python/tests/test_threading.py index ed0a1d100..7575ff753 100644 --- a/bindings/python/tests/test_threading.py +++ b/bindings/python/tests/test_threading.py @@ -8,9 +8,9 @@ def test_concurrent_encode_matches_serial(): tok = train_word_tokenizer() - expected = [tok.encode(line, add_special_tokens=False) for line in SENTENCES] + expected = [tok.encode_ids(line, add_special_tokens=False) for line in SENTENCES] with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: - results = list(pool.map(lambda s: tok.encode(s, add_special_tokens=False), SENTENCES)) + results = list(pool.map(lambda s: tok.encode_ids(s, add_special_tokens=False), SENTENCES)) for got, want in zip(results, expected): assert np.array_equal(got, want) @@ -18,10 +18,10 @@ def test_concurrent_encode_matches_serial(): def test_parallel_encode_batch_matches_serial(): tok = train_word_tokenizer() os.environ["TOKENIZERS_PARALLELISM"] = "false" - serial = tok.encode_batch(SENTENCES * 32, add_special_tokens=False) + serial = tok.encode_batch_ids(SENTENCES * 32, add_special_tokens=False) os.environ["TOKENIZERS_PARALLELISM"] = "true" try: - parallel = tok.encode_batch(SENTENCES * 32, add_special_tokens=False) + parallel = tok.encode_batch_ids(SENTENCES * 32, add_special_tokens=False) finally: del os.environ["TOKENIZERS_PARALLELISM"] for got, want in zip(parallel, serial): @@ -32,7 +32,7 @@ def test_concurrent_mutation_and_encode_is_safe(): tok = train_word_tokenizer() def encode_some(_): - return tok.encode_batch(SENTENCES, add_special_tokens=False) + return tok.encode_batch_ids(SENTENCES, add_special_tokens=False) def add_some(i): return tok.add_tokens([f""]) diff --git a/bindings/python/tests/test_tokenizer.py b/bindings/python/tests/test_tokenizer.py index ed3bbd286..0b2f9586e 100644 --- a/bindings/python/tests/test_tokenizer.py +++ b/bindings/python/tests/test_tokenizer.py @@ -9,23 +9,23 @@ def test_encode_returns_uint32_array(word_tokenizer): - ids = word_tokenizer.encode(SENTENCES[0], add_special_tokens=False) + ids = word_tokenizer.encode_ids(SENTENCES[0], add_special_tokens=False) assert isinstance(ids, np.ndarray) assert ids.dtype == np.uint32 words = [word_tokenizer.id_to_token(int(i)) for i in ids] assert words == SENTENCES[0].split() -def test_encode_batch_matches_encode(word_tokenizer): - batch = word_tokenizer.encode_batch(SENTENCES, add_special_tokens=False) +def test_encode_batch_ids_matches_encode_ids(word_tokenizer): + batch = word_tokenizer.encode_batch_ids(SENTENCES, add_special_tokens=False) assert len(batch) == len(SENTENCES) for line, ids in zip(SENTENCES, batch): - single = word_tokenizer.encode(line, add_special_tokens=False) + single = word_tokenizer.encode_ids(line, add_special_tokens=False) assert np.array_equal(ids, single) def test_unknown_words_map_to_unk(word_tokenizer): - ids = word_tokenizer.encode("supercalifragilistic", add_special_tokens=False) + ids = word_tokenizer.encode_ids("supercalifragilistic", add_special_tokens=False) assert [word_tokenizer.id_to_token(int(i)) for i in ids] == ["[UNK]"] @@ -42,14 +42,14 @@ def test_add_tokens_and_encode_them(): tok = train_word_tokenizer() assert tok.add_tokens(["procrastination"]) == 1 assert tok.add_tokens(["procrastination"]) == 0 - ids = tok.encode("the procrastination", add_special_tokens=False) + ids = tok.encode_ids("the procrastination", add_special_tokens=False) assert [tok.id_to_token(int(i)) for i in ids] == ["the", "procrastination"] def test_add_special_tokens_marks_special(): tok = train_word_tokenizer() assert tok.add_special_tokens([""]) == 1 - ids = tok.encode("the ", add_special_tokens=False) + ids = tok.encode_ids("the ", add_special_tokens=False) assert [tok.id_to_token(int(i)) for i in ids] == ["the", ""] @@ -65,24 +65,24 @@ def test_save_and_from_file_round_trip(word_tokenizer, tmp_path): reloaded = Tokenizer.from_file(path) for line in SENTENCES[:4]: assert np.array_equal( - reloaded.encode(line, add_special_tokens=False), - word_tokenizer.encode(line, add_special_tokens=False), + reloaded.encode_ids(line, add_special_tokens=False), + word_tokenizer.encode_ids(line, add_special_tokens=False), ) def test_to_str_and_from_buffer_round_trip(word_tokenizer): reloaded = Tokenizer.from_buffer(word_tokenizer.to_str().encode()) assert np.array_equal( - reloaded.encode(SENTENCES[0], add_special_tokens=False), - word_tokenizer.encode(SENTENCES[0], add_special_tokens=False), + reloaded.encode_ids(SENTENCES[0], add_special_tokens=False), + word_tokenizer.encode_ids(SENTENCES[0], add_special_tokens=False), ) def test_pickle_round_trip(word_tokenizer): reloaded = pickle.loads(pickle.dumps(word_tokenizer)) assert np.array_equal( - reloaded.encode(SENTENCES[0], add_special_tokens=False), - word_tokenizer.encode(SENTENCES[0], add_special_tokens=False), + reloaded.encode_ids(SENTENCES[0], add_special_tokens=False), + word_tokenizer.encode_ids(SENTENCES[0], add_special_tokens=False), ) diff --git a/bindings/python/tests/test_trainers.py b/bindings/python/tests/test_trainers.py index 1626d4986..2c123cfb7 100644 --- a/bindings/python/tests/test_trainers.py +++ b/bindings/python/tests/test_trainers.py @@ -34,7 +34,7 @@ def test_each_trainer_trains(model, trainer): tok = fresh(model) tok.train_from_iterator(SENTENCES, trainer=trainer) assert tok.get_vocab_size() > 0 - ids = tok.encode(SENTENCES[0], add_special_tokens=False) + ids = tok.encode_ids(SENTENCES[0], add_special_tokens=False) assert len(ids) > 0 From 4a17bff94a80ed06fe1d551b6a048b3d5abc4073 Mon Sep 17 00:00:00 2001 From: SBrandeis <33657802+SBrandeis@users.noreply.github.com> Date: Wed, 22 Jul 2026 17:44:32 +0200 Subject: [PATCH 16/19] Encoding class --- README.md | 13 +- bindings/python/CHANGELOG.md | 36 ++-- bindings/python/README.md | 58 ++--- bindings/python/examples/01_pretrained.py | 13 +- .../python/examples/02_train_and_encode.py | 29 ++- bindings/python/py_src/tokenizers/__init__.py | 4 + .../python/py_src/tokenizers/__init__.pyi | 119 ++++++++++- bindings/python/src/encoding.rs | 199 ++++++++++++++++++ bindings/python/src/lib.rs | 3 + bindings/python/src/tokenizer.rs | 120 +++++++++-- bindings/python/stubtest_allowlist.txt | 4 + bindings/python/tests/test_async.py | 19 ++ bindings/python/tests/test_encoding.py | 116 ++++++++++ 13 files changed, 650 insertions(+), 83 deletions(-) create mode 100644 bindings/python/src/encoding.rs create mode 100644 bindings/python/tests/test_encoding.py diff --git a/README.md b/README.md index 2115235dd..ddde2b894 100644 --- a/README.md +++ b/README.md @@ -86,15 +86,16 @@ tokenizer.train(files=["wiki.train.raw", "wiki.valid.raw", "wiki.test.raw"], tra Once your tokenizer is trained, encode any text with just one line: ```python -ids = tokenizer.encode_ids("Hello, y'all! How are you 😁 ?") -print([tokenizer.id_to_token(i) for i in ids]) +encoding = tokenizer.encode("Hello, y'all! How are you 😁 ?") +print(encoding.tokens) # ["Hello", ",", "y", "'", "all", "!", "How", "are", "you", "[UNK]", "?"] ``` -`encode_ids` returns the token ids as a `numpy.uint32` array — ready to hand to -your model with no further conversion. The emoji comes out as `[UNK]`: it -never appeared in the training files, so it is not in the vocabulary, and BPE -falls back to the `unk_token` we configured above. +`encode` returns an `Encoding` — ids, tokens, and the masks a model consumes. +The emoji comes out as `[UNK]`: it never appeared in the training files, so it +is not in the vocabulary, and BPE falls back to the `unk_token` we configured +above. When you only need the ids, `encode_ids` returns them as a +`numpy.uint32` array with no copy. More in [bindings/python](bindings/python) — its README and `examples/` cover loading pretrained tokenizers, threading and async, and training. (The diff --git a/bindings/python/CHANGELOG.md b/bindings/python/CHANGELOG.md index a1434337a..bff62e2fa 100644 --- a/bindings/python/CHANGELOG.md +++ b/bindings/python/CHANGELOG.md @@ -14,23 +14,24 @@ arrays without a copy. Breaking changes — encoding: -- Encoding returns a numpy array of ids, not an `Encoding` object, and the - methods are accordingly named `encode_ids`/`encode_batch_ids` — the - `encode`/`encode_batch` names are reserved for a planned - `Encoding`-returning API. Everything the `Encoding` carried is gone from - the encode path: tokens, offsets, type ids, attention masks, - special-tokens masks, word ids, overflowing/stride, and the - char/word/token mapping helpers. Truncation and padding - (`enable_truncation`/`enable_padding` and their getters) are gone too. -- `encode_ids` takes a single text: the `pair=` argument and the - `is_pretokenized=` mode no longer exist (same for `encode_batch_ids`). +- `encode`/`encode_batch` return an `Encoding`/`EncodingBatch` carrying ids, + tokens, type ids, attention and special-tokens masks, and sequence ids + (`ids` is a `list`; `ids_array()` gives a numpy array). `encode_ids`/ + `encode_batch_ids` are the new names for the bare-`numpy.uint32` path — same + encode work, no `Encoding` wrapper. Not carried yet, and raising rather than + returning a guess: word ids and character offsets (and the char/word/token + mapping helpers built on them). Overflowing/stride, truncation, and padding + (`enable_truncation`/`enable_padding` and their getters) are gone. +- `encode` takes a single text: the `pair=` argument and the + `is_pretokenized=` mode no longer exist (same for `encode_batch`). - Not implemented yet (loud errors, never wrong ids): `decode`, post-processor templates (pass `add_special_tokens=False`), and the `Metaspace` pre-tokenizer. `decode_batch`, `DecodeStream`, `encode_batch_fast`, and `Tokenizer.post_process` are removed. - **`transformers`' `PreTrainedTokenizerFast` cannot run on 1.0 yet** — it - needs the `Encoding` fields, padding/truncation, and pair inputs listed - above. Pin `tokenizers<1.0` for `transformers` until it targets 1.x. + needs the not-yet-implemented `Encoding` fields (offsets), post-processing, + padding/truncation, and pair inputs. Pin `tokenizers<1.0` for `transformers` + until it targets 1.x. Breaking changes — components and introspection: @@ -67,11 +68,12 @@ Breaking changes — `Tokenizer` API and packaging: moved from a required dependency to the `hub` extra: `pip install 'tokenizers[hub]'`. `numpy>=1.24` is a new required dependency. -Kept: awaitable encodes, as `async_encode_ids`/`async_encode_batch_ids` -(now a thin `asyncio.to_thread` wrapper — no tokio runtime) and free-threaded Python support (default wheels -are abi3-py310; free-threaded interpreters get their own non-abi3 wheels). -New in 1.0: parity-aware BPE training (`trainers.ParityBpeTrainer`), which -never shipped in a 0.x release. +Kept: awaitable encodes — `async_encode`/`async_encode_batch` (returning an +`Encoding`/`EncodingBatch`) and `async_encode_ids`/`async_encode_batch_ids`, +now a thin `asyncio.to_thread` wrapper (no tokio runtime) — and free-threaded +Python support (default wheels are abi3-py310; free-threaded interpreters get +their own non-abi3 wheels). New in 1.0: parity-aware BPE training +(`trainers.ParityBpeTrainer`), which never shipped in a 0.x release. ## [0.13.2] diff --git a/bindings/python/README.md b/bindings/python/README.md index 6d2a37ce4..467db067d 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -14,20 +14,28 @@ import tokenizers as tk tok = tk.Tokenizer.from_file("tokenizer.json") -# Returns a numpy.uint32 array of token ids. +# encode returns an Encoding: ids plus the masks and metadata a model consumes. # add_special_tokens=False skips template tokens like [CLS]/[SEP]; inserting # them is not implemented yet in 1.0, so leaving it True raises a loud # NotImplementedError on tokenizers that use such templates (BERT, Llama, …). -ids = tok.encode_ids("Hello world", add_special_tokens=False) +enc = tok.encode("Hello world", add_special_tokens=False) +enc.ids # list[int] +enc.tokens # list[str] +enc.attention_mask # list[int] + +batch = tok.encode_batch(["Hello world", "How are you?"], add_special_tokens=False) +batch[0].ids # a batch is a sequence of Encodings -# A list of arrays, encoded in parallel across Rust threads. -batch = tok.encode_batch_ids(["Hello world", "How are you?"], add_special_tokens=False) +# When you only want the ids, encode_ids skips the Encoding and returns them as +# a numpy.uint32 array with no copy (encode_batch_ids returns a list of arrays). +ids = tok.encode_ids("Hello world", add_special_tokens=False) ``` -The methods are named `encode_ids`/`encode_batch_ids` because they return -bare ids: the `encode`/`encode_batch` names are reserved for a planned -`Encoding`-returning API (masks, word ids, truncation/padding — the interface -`transformers` consumes). +`encode` and `encode_ids` run exactly the same work; `encode` wraps the ids in +an `Encoding` and derives its fields on access, so you never pay for what you +don't read. Fields that need per-token provenance the pipeline does not compute +yet — `word_ids` and character `offsets` — raise `NotImplementedError` rather +than return a guess. To load a tokenizer straight from the [Hugging Face Hub](https://huggingface.co), install the `hub` extra (`pip install 'tokenizers[hub]'`): @@ -63,30 +71,29 @@ simplest case (load a pretrained file and encode). Encoding releases the Python interpreter lock (the GIL), so this package plays well with threads and event loops: -- Calling `encode_ids` from several Python threads scales — the threads - really run in parallel, on every interpreter (free-threaded or not). -- `encode_batch_ids` parallelizes one batch across Rust threads. Set the - `TOKENIZERS_PARALLELISM` environment variable to `false`/`true` to +- Calling `encode`/`encode_ids` from several Python threads scales — the + threads really run in parallel, on every interpreter (free-threaded or not). +- `encode_batch`/`encode_batch_ids` parallelize one batch across Rust threads. + Set the `TOKENIZERS_PARALLELISM` environment variable to `false`/`true` to disable/force this. -- In `asyncio` code, `await tok.async_encode_ids(text)` / - `await tok.async_encode_batch_ids(texts)` keep the event loop free while - Rust encodes in a worker thread. +- In `asyncio` code, `await tok.async_encode(text)` (and the `_ids` / + `_batch` variants) keep the event loop free while Rust encodes in a worker + thread. ```python -ids = await tok.async_encode_ids("Hello world", add_special_tokens=False) +enc = await tok.async_encode("Hello world", add_special_tokens=False) ``` ## Breaking changes vs 0.x 1.x is a ground-up rewrite with a smaller, faster API. The headline changes: -- Encoding returns a `numpy.uint32` array of ids, not an `Encoding` object, - and the methods are accordingly named `encode_ids`/`encode_batch_ids`. - Tokens, offsets, type ids, attention masks, and word ids are gone from the - encode path, and so are truncation and padding - (`enable_truncation`/`enable_padding`). -- `encode_ids` takes a single text: no `pair=` argument, no - `is_pretokenized=`. +- `encode` returns an `Encoding` carrying ids, tokens, type ids, attention + and special-tokens masks, and sequence ids. Word ids and character offsets + are not computed yet and raise; truncation and padding + (`enable_truncation`/`enable_padding`) are gone. `encode_ids` is the new + name for a bare `numpy.uint32` id array. +- `encode` takes a single text: no `pair=` argument, no `is_pretokenized=`. - Not implemented yet (loud errors, never wrong ids): `decode`, post-processor templates (`[CLS]`/`` insertion — pass `add_special_tokens=False`), and the `Metaspace` pre-tokenizer @@ -96,8 +103,9 @@ ids = await tok.async_encode_ids("Hello world", add_special_tokens=False) you subclass. - `decoders`, `processors`, and the `implementations` helpers (`BertWordPieceTokenizer`, …) are gone. -- **`transformers` cannot use 1.0 as its backend yet** — it needs several of - the removed pieces. Pin `tokenizers<1.0` for `transformers`. +- **`transformers` cannot use 1.0 as its backend yet** — it needs the + not-yet-implemented pieces above (post-processing, offsets, decode). Pin + `tokenizers<1.0` for `transformers`. The full list, including smaller removals and renames, is in the 1.0.0 entry of [CHANGELOG.md](CHANGELOG.md). diff --git a/bindings/python/examples/01_pretrained.py b/bindings/python/examples/01_pretrained.py index ddf667919..b4a1e5723 100644 --- a/bindings/python/examples/01_pretrained.py +++ b/bindings/python/examples/01_pretrained.py @@ -24,12 +24,13 @@ ("bert-base-uncased", "bert-base-uncased.json"), ]: tok = Tokenizer.from_file(DATA / file) - batch = tok.encode_batch_ids(LINES, add_special_tokens=False) - assert all(ids.dtype == np.uint32 for ids in batch) - total = sum(len(ids) for ids in batch) - first_token = tok.id_to_token(batch[0][0]) - assert first_token is not None - print(f"{name}: {total} tokens, first token {first_token!r}") + batch = tok.encode_batch(LINES, add_special_tokens=False) + assert len(batch) == len(LINES) + total = sum(len(enc) for enc in batch) + first = batch[0] + assert first.ids_array().dtype == np.uint32 + assert first.attention_mask == [1] * len(first) + print(f"{name}: {total} tokens, first token {first.tokens[0]!r}") # Expected failure 1: post-processor would add special tokens -> loud error, # not silently wrong ids diff --git a/bindings/python/examples/02_train_and_encode.py b/bindings/python/examples/02_train_and_encode.py index 4a18da50d..a61af6f2a 100644 --- a/bindings/python/examples/02_train_and_encode.py +++ b/bindings/python/examples/02_train_and_encode.py @@ -1,5 +1,6 @@ """End-to-end: build a tokenizer from scratch, train it, swap its components, -encode, serialize, pickle, and see that decode raises for now.""" +encode to an `Encoding` (and to a bare id array), serialize, pickle, and see +that decode raises for now.""" import pickle import tempfile @@ -37,12 +38,30 @@ def corpus(): assert tok.get_vocab_size() == 1000, tok.get_vocab_size() assert tok.token_to_id("") == 0 -# 3. Encode -> numpy uint32 array; special tokens are matched in the text +# 3. Encode -> Encoding: ids plus the masks and metadata a model consumes. +enc = tok.encode("The quick brown fox jumps over the lazy dog") +print(f"encoding: {enc!r}") +print(f" tokens: {enc.tokens}") +print(f" type_ids / attention_mask: {enc.type_ids} / {enc.attention_mask}") +assert enc.ids == tok.encode_ids("The quick brown fox jumps over the lazy dog").tolist() +assert tok.token_to_id("") in enc.ids +assert enc.type_ids == [0] * len(enc) and enc.attention_mask == [1] * len(enc) +# .ids is a list (models pad with `ids + [pad_id] * n`); .ids_array is numpy. +assert isinstance(enc.ids, list) +assert enc.ids_array().dtype == np.uint32 + +# Word ids and character offsets are not emitted by the encode pipeline yet; +# they raise rather than returning a plausible-looking guess. +for feature in (lambda: enc.word_ids, lambda: enc.offsets, lambda: enc.char_to_token(0)): + try: + feature() + raise AssertionError("unavailable feature should raise") + except NotImplementedError: + pass + +# encode_ids skips the Encoding and returns the ids array directly (no copy). ids = tok.encode_ids("The quick brown fox jumps over the lazy dog") -print(f"ids: {ids.dtype} {ids}") assert isinstance(ids, np.ndarray) and ids.dtype == np.uint32 -assert tok.token_to_id("") in ids -assert all(tok.id_to_token(i) is not None for i in ids) # 4. Mutate a component in place: dropping the lowercasing normalizer changes ids tok_ids_lower = tok.encode_ids("HELLO WORLD") diff --git a/bindings/python/py_src/tokenizers/__init__.py b/bindings/python/py_src/tokenizers/__init__.py index 226208dcd..e5812534b 100644 --- a/bindings/python/py_src/tokenizers/__init__.py +++ b/bindings/python/py_src/tokenizers/__init__.py @@ -2,6 +2,8 @@ from ._native import ( AddedToken, + Encoding, + EncodingBatch, Tokenizer, TokenizersError, __version__, @@ -10,6 +12,8 @@ __all__ = [ "AddedToken", + "Encoding", + "EncodingBatch", "Tokenizer", "TokenizersError", "__version__", diff --git a/bindings/python/py_src/tokenizers/__init__.pyi b/bindings/python/py_src/tokenizers/__init__.pyi index 0a1f6c00c..d33cc9b41 100644 --- a/bindings/python/py_src/tokenizers/__init__.pyi +++ b/bindings/python/py_src/tokenizers/__init__.pyi @@ -43,6 +43,97 @@ class AddedToken: @property def special(self, /) -> bool: ... +@final +class Encoding: + """ + The result of encoding one sequence: token ids plus the masks and metadata + a model consumes. The fields are derived from the ids on access, so an + `Encoding` costs the same to produce as a bare id array — `Tokenizer.encode` + runs exactly the work `encode_ids` does. + + `encode` only produces an `Encoding` for a single sequence with no + post-processor-inserted special tokens (it raises otherwise), so the + segment, attention, special-token and sequence values are constant: one + sequence numbered 0, nothing padded, nothing special. Anything that would + need per-token provenance the pipeline does not compute — word ids and + character offsets — raises rather than returning a plausible-looking guess. + """ + def __len__(self, /) -> int: ... + def __repr__(self, /) -> str: ... + @property + def attention_mask(self, /) -> list[int]: + """ + Attention mask, one entry per token: all 1 (nothing padded). + """ + def char_to_token(self, /, char_pos: int, sequence_index: int = 0) -> int |None: ... + def char_to_word(self, /, char_pos: int, sequence_index: int = 0) -> int |None: ... + @property + def ids(self, /) -> list[int]: + """ + The token ids, as a list. + """ + def ids_array(self, /) -> "npt.NDArray[np.uint32]": + """ + The token ids as a `numpy.uint32` array. This copies; for the copy-free + array use `Tokenizer.encode_ids`, which hands ownership of the buffer + straight to numpy. + """ + @property + def n_sequences(self, /) -> int: + """ + Number of sequences in this encoding: always 1. + """ + @property + def offsets(self, /) -> list[tuple[int, int]]: + """ + Character span per token — not available: the pipeline does not track + offsets yet. + """ + @property + def sequence_ids(self, /) -> list[int |None]: + """ + The sequence each token belongs to: all 0 (single sequence). + """ + @property + def special_tokens_mask(self, /) -> list[int]: + """ + Special-tokens mask, one entry per token: all 0 (no post-processing). + """ + def token_to_chars(self, /, token_index: int) -> tuple[int, int] |None: ... + def token_to_sequence(self, /, token_index: int) -> int |None: + """ + The sequence a token belongs to (0), or None for an out-of-range index. + """ + def token_to_word(self, /, token_index: int) -> int |None: ... + @property + def tokens(self, /) -> list[str]: + """ + The token strings behind the ids. + """ + @property + def type_ids(self, /) -> list[int]: + """ + Segment id per token: all 0 (single sequence). + """ + @property + def word_ids(self, /) -> list[int |None]: + """ + Word id per token — not available: the pipeline does not emit word + boundaries yet. + """ + def word_to_chars(self, /, word_index: int, sequence_index: int = 0) -> tuple[int, int] |None: ... + def word_to_tokens(self, /, word_index: int, sequence_index: int = 0) -> tuple[int, int] |None: ... + +@final +class EncodingBatch: + """ + The result of encoding a batch: a sequence of `Encoding`s. Index it + (`batch[0]`) or iterate it. + """ + def __getitem__(self, /, index: int) -> Encoding: ... + def __len__(self, /) -> int: ... + def __repr__(self, /) -> str: ... + @final class Tokenizer: """ @@ -73,6 +164,15 @@ class Tokenizer: on. Plain strings match with default options; pass `AddedToken` to control matching. Returns how many were actually new. """ + def async_encode(self, /, text: Any, *, add_special_tokens: bool = True) -> "Coroutine[Any, Any, Encoding]": + """ + Awaitable `encode`: same arguments and result, run in a worker thread. + """ + def async_encode_batch(self, /, texts: Any, *, add_special_tokens: bool = True) -> "Coroutine[Any, Any, EncodingBatch]": + """ + Awaitable `encode_batch`: same arguments and result, run in a worker + thread while the batch encodes on Rust threads. + """ def async_encode_batch_ids(self, /, texts: Any, *, add_special_tokens: bool = True) -> "Coroutine[Any, Any, list[npt.NDArray[np.uint32]]]": """ Awaitable `encode_batch_ids`: same arguments and result, run in a @@ -90,6 +190,18 @@ class Tokenizer: """ Not implemented yet: decoding is not part of the encode pipeline. """ + def encode(self, /, text: str, *, add_special_tokens: bool = True) -> "Encoding": + """ + Encode `text` into an `Encoding`: token ids plus the masks and metadata + a model consumes. Same encode work as `encode_ids` (GIL released, no + copies); the `Encoding` wraps the ids and derives its fields on access. + """ + def encode_batch(self, /, texts: Sequence[str], *, add_special_tokens: bool = True) -> "EncodingBatch": + """ + Encode a batch of texts into an `EncodingBatch`, in parallel across Rust + threads (respects `TOKENIZERS_PARALLELISM`), without holding the + interpreter lock. The batch version of `encode`. + """ def encode_batch_ids(self, /, texts: Sequence[str], *, add_special_tokens: bool = True) -> "list[npt.NDArray[np.uint32]]": """ Encode a batch of texts, in parallel across Rust threads (respects @@ -102,8 +214,9 @@ class Tokenizer: Encode `text` into token ids. Runs entirely outside the interpreter lock and returns a `numpy.uint32` - array backed by the Rust output buffer (no copy). The `encode` name is - reserved for the upcoming `Encoding`-returning API. + array backed by the Rust output buffer (no copy). For ids plus the + masks and metadata models consume, use `encode`, which returns an + `Encoding` from the same work. """ @staticmethod def from_buffer(buffer: Sequence[int]) -> "Tokenizer": @@ -193,4 +306,4 @@ from tokenizers import normalizers as normalizers from tokenizers import pre_tokenizers as pre_tokenizers from tokenizers import trainers as trainers -__all__ = ["AddedToken", "Tokenizer", "TokenizersError", "__version__", "models", "normalizers", "pre_tokenizers", "trainers"] +__all__ = ["AddedToken", "Encoding", "EncodingBatch", "Tokenizer", "TokenizersError", "__version__", "models", "normalizers", "pre_tokenizers", "trainers"] diff --git a/bindings/python/src/encoding.rs b/bindings/python/src/encoding.rs new file mode 100644 index 000000000..3c8cafe68 --- /dev/null +++ b/bindings/python/src/encoding.rs @@ -0,0 +1,199 @@ +use std::sync::Arc; + +use numpy::{IntoPyArray, PyArray1}; +use pyo3::exceptions::{PyIndexError, PyNotImplementedError}; +use pyo3::prelude::*; + +use crate::tokenizer::PyTokenizer; + +const NO_OFFSETS: &str = "character offsets are not tracked by the encode pipeline"; +const NO_WORD_IDS: &str = "word ids are not emitted by the encode pipeline"; + +fn deferred(what: &str, why: &str) -> PyErr { + PyNotImplementedError::new_err(format!("{what} is not available yet: {why}")) +} + +/// The result of encoding one sequence: token ids plus the masks and metadata +/// a model consumes. The fields are derived from the ids on access, so an +/// `Encoding` costs the same to produce as a bare id array — `Tokenizer.encode` +/// runs exactly the work `encode_ids` does. +/// +/// `encode` only produces an `Encoding` for a single sequence with no +/// post-processor-inserted special tokens (it raises otherwise), so the +/// segment, attention, special-token and sequence values are constant: one +/// sequence numbered 0, nothing padded, nothing special. Anything that would +/// need per-token provenance the pipeline does not compute — word ids and +/// character offsets — raises rather than returning a plausible-looking guess. +#[pyclass(frozen, name = "Encoding", module = "tokenizers")] +pub struct PyEncoding { + ids: Arc<[u32]>, + tokenizer: Py, +} + +impl PyEncoding { + pub(crate) fn new(ids: Arc<[u32]>, tokenizer: Py) -> Self { + Self { ids, tokenizer } + } +} + +#[pymethods] +impl PyEncoding { + fn __len__(&self) -> usize { + self.ids.len() + } + + fn __repr__(&self) -> String { + format!("Encoding(length={})", self.ids.len()) + } + + /// The token ids, as a list. + #[getter] + fn ids(&self) -> Vec { + self.ids.to_vec() + } + + /// The token ids as a `numpy.uint32` array. This copies; for the copy-free + /// array use `Tokenizer.encode_ids`, which hands ownership of the buffer + /// straight to numpy. + #[pyo3(signature = () -> "npt.NDArray[np.uint32]")] + fn ids_array<'py>(&self, py: Python<'py>) -> Bound<'py, PyArray1> { + self.ids.to_vec().into_pyarray(py) + } + + /// The token strings behind the ids. + #[getter] + fn tokens(&self, py: Python<'_>) -> PyResult> { + self.tokenizer.bind(py).get().ids_to_tokens(py, &self.ids) + } + + /// Segment id per token: all 0 (single sequence). + #[getter] + fn type_ids(&self) -> Vec { + vec![0; self.ids.len()] + } + + /// Attention mask, one entry per token: all 1 (nothing padded). + #[getter] + fn attention_mask(&self) -> Vec { + vec![1; self.ids.len()] + } + + /// Special-tokens mask, one entry per token: all 0 (no post-processing). + #[getter] + fn special_tokens_mask(&self) -> Vec { + vec![0; self.ids.len()] + } + + /// The sequence each token belongs to: all 0 (single sequence). + #[getter] + fn sequence_ids(&self) -> Vec> { + vec![Some(0); self.ids.len()] + } + + /// Number of sequences in this encoding: always 1. + #[getter] + fn n_sequences(&self) -> usize { + 1 + } + + /// The sequence a token belongs to (0), or None for an out-of-range index. + #[pyo3(signature = (token_index))] + fn token_to_sequence(&self, token_index: usize) -> Option { + (token_index < self.ids.len()).then_some(0) + } + + /// Word id per token — not available: the pipeline does not emit word + /// boundaries yet. + #[getter] + fn word_ids(&self) -> PyResult>> { + Err(deferred("word_ids", NO_WORD_IDS)) + } + + /// Character span per token — not available: the pipeline does not track + /// offsets yet. + #[getter] + fn offsets(&self) -> PyResult> { + Err(deferred("offsets", NO_OFFSETS)) + } + + #[pyo3(signature = (token_index))] + #[allow(unused_variables)] + fn token_to_word(&self, token_index: usize) -> PyResult> { + Err(deferred("token_to_word", NO_WORD_IDS)) + } + + #[pyo3(signature = (word_index, sequence_index = 0))] + #[allow(unused_variables)] + fn word_to_tokens( + &self, + word_index: u32, + sequence_index: usize, + ) -> PyResult> { + Err(deferred("word_to_tokens", NO_WORD_IDS)) + } + + #[pyo3(signature = (word_index, sequence_index = 0))] + #[allow(unused_variables)] + fn word_to_chars( + &self, + word_index: u32, + sequence_index: usize, + ) -> PyResult> { + Err(deferred("word_to_chars", NO_OFFSETS)) + } + + #[pyo3(signature = (token_index))] + #[allow(unused_variables)] + fn token_to_chars(&self, token_index: usize) -> PyResult> { + Err(deferred("token_to_chars", NO_OFFSETS)) + } + + #[pyo3(signature = (char_pos, sequence_index = 0))] + #[allow(unused_variables)] + fn char_to_token(&self, char_pos: usize, sequence_index: usize) -> PyResult> { + Err(deferred("char_to_token", NO_OFFSETS)) + } + + #[pyo3(signature = (char_pos, sequence_index = 0))] + #[allow(unused_variables)] + fn char_to_word(&self, char_pos: usize, sequence_index: usize) -> PyResult> { + Err(deferred("char_to_word", NO_OFFSETS)) + } +} + +/// The result of encoding a batch: a sequence of `Encoding`s. Index it +/// (`batch[0]`) or iterate it. +#[pyclass(frozen, name = "EncodingBatch", module = "tokenizers")] +pub struct PyEncodingBatch { + rows: Vec>, + tokenizer: Py, +} + +impl PyEncodingBatch { + pub(crate) fn new(rows: Vec>, tokenizer: Py) -> Self { + Self { rows, tokenizer } + } +} + +#[pymethods] +impl PyEncodingBatch { + fn __len__(&self) -> usize { + self.rows.len() + } + + fn __repr__(&self) -> String { + format!("EncodingBatch(size={})", self.rows.len()) + } + + fn __getitem__(&self, py: Python<'_>, index: isize) -> PyResult { + let len = self.rows.len() as isize; + let resolved = if index < 0 { index + len } else { index }; + if resolved < 0 || resolved >= len { + return Err(PyIndexError::new_err("EncodingBatch index out of range")); + } + Ok(PyEncoding::new( + self.rows[resolved as usize].clone(), + self.tokenizer.clone_ref(py), + )) + } +} diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs index 3f2128313..191a0988e 100644 --- a/bindings/python/src/lib.rs +++ b/bindings/python/src/lib.rs @@ -2,6 +2,7 @@ pub mod added_token; pub mod detached_lock; +pub mod encoding; pub mod error; pub mod models; pub mod normalizers; @@ -38,6 +39,8 @@ pub mod _native { #[pymodule_export] pub use super::added_token::PyAddedToken; #[pymodule_export] + pub use super::encoding::{PyEncoding, PyEncodingBatch}; + #[pymodule_export] pub use super::error::TokenizersError; #[pymodule_export] pub use super::tokenizer::PyTokenizer; diff --git a/bindings/python/src/tokenizer.rs b/bindings/python/src/tokenizer.rs index 7554f1a15..830d30784 100644 --- a/bindings/python/src/tokenizer.rs +++ b/bindings/python/src/tokenizer.rs @@ -20,6 +20,7 @@ use tk_train::{TokenizerTrainExt, Trainable}; use crate::added_token::{TokenInput, parse_tokens}; use crate::detached_lock::{Detached, DetachedRwLock}; +use crate::encoding::{PyEncoding, PyEncodingBatch}; use crate::error::{TokenizersError, to_pyerr}; use crate::models::{PyModel, wrap_model}; use crate::normalizers::{PyNormalizer, wrap_normalizer}; @@ -91,6 +92,17 @@ impl PyTokenizer { }) } + /// Resolve a run of ids to their token strings in one locked read, for + /// `Encoding.tokens`. Unknown ids (out of the vocabulary) become empty + /// strings rather than failing the whole lookup. + pub(crate) fn ids_to_tokens(&self, py: Python<'_>, ids: &[u32]) -> PyResult> { + self.read_spec(py, |spec| { + ids.iter() + .map(|&id| spec.id_to_token(id).unwrap_or_default()) + .collect() + }) + } + /// Write access to the spec; invalidates the compiled pipeline. fn mutate_spec( &self, @@ -280,6 +292,30 @@ fn encode_one( Ok(output.iter().map(|t| t.id).collect()) } +/// Encode a batch to raw id vectors — the work shared by `encode_batch_ids` +/// (which wraps each row in a numpy array) and `encode_batch` (which wraps the +/// batch in an `EncodingBatch`). Runs on rayon threads when parallelism is on +/// and the batch is worth splitting; the caller has already released the GIL. +fn encode_batch_core(compiled: &Compiled, texts: &[PyBackedStr]) -> PyResult>> { + if get_parallelism() && texts.len() > 1 { + USED_PARALLELISM.store(true, Ordering::SeqCst); + texts + .par_iter() + .map_init( + || (Vec::new(), compiled.pipe.get_model().init_scratch()), + |(pre_tokens, scratch), text| encode_one(&compiled.pipe, text, pre_tokens, scratch), + ) + .collect() + } else { + let mut pre_tokens = Vec::new(); + let mut scratch = compiled.pipe.get_model().init_scratch(); + texts + .iter() + .map(|text| encode_one(&compiled.pipe, text, &mut pre_tokens, &mut scratch)) + .collect() + } +} + #[pymethods] impl PyTokenizer { /// Create an untrained tokenizer from a model. @@ -346,8 +382,9 @@ impl PyTokenizer { /// Encode `text` into token ids. /// /// Runs entirely outside the interpreter lock and returns a `numpy.uint32` - /// array backed by the Rust output buffer (no copy). The `encode` name is - /// reserved for the upcoming `Encoding`-returning API. + /// array backed by the Rust output buffer (no copy). For ids plus the + /// masks and metadata models consume, use `encode`, which returns an + /// `Encoding` from the same work. #[pyo3(signature = (text, *, add_special_tokens = true) -> "npt.NDArray[np.uint32]")] fn encode_ids<'py>( &self, @@ -379,25 +416,7 @@ impl PyTokenizer { let batches = self.inner.with(py, |lock| -> PyResult>> { let compiled = get_or_compile(&lock)?; check_special_tokens_flag(&compiled, add_special_tokens)?; - if get_parallelism() && texts.len() > 1 { - USED_PARALLELISM.store(true, Ordering::SeqCst); - texts - .par_iter() - .map_init( - || (Vec::new(), compiled.pipe.get_model().init_scratch()), - |(pre_tokens, scratch), text| { - encode_one(&compiled.pipe, text, pre_tokens, scratch) - }, - ) - .collect() - } else { - let mut pre_tokens = Vec::new(); - let mut scratch = compiled.pipe.get_model().init_scratch(); - texts - .iter() - .map(|text| encode_one(&compiled.pipe, text, &mut pre_tokens, &mut scratch)) - .collect() - } + encode_batch_core(&compiled, &texts) })?; let list = PyList::empty(py); for ids in batches { @@ -431,6 +450,65 @@ impl PyTokenizer { to_thread(slf, "encode_batch_ids", texts, add_special_tokens) } + /// Encode `text` into an `Encoding`: token ids plus the masks and metadata + /// a model consumes. Same encode work as `encode_ids` (GIL released, no + /// copies); the `Encoding` wraps the ids and derives its fields on access. + #[pyo3(signature = (text, *, add_special_tokens = true) -> "Encoding")] + fn encode(slf: &Bound<'_, Self>, text: &str, add_special_tokens: bool) -> PyResult { + let py = slf.py(); + let ids = slf.get().inner.with(py, |lock| -> PyResult> { + let compiled = get_or_compile(&lock)?; + check_special_tokens_flag(&compiled, add_special_tokens)?; + let mut pre_tokens = Vec::new(); + let mut scratch = compiled.pipe.get_model().init_scratch(); + encode_one(&compiled.pipe, text, &mut pre_tokens, &mut scratch) + })?; + Ok(PyEncoding::new(ids.into(), slf.clone().unbind())) + } + + /// Encode a batch of texts into an `EncodingBatch`, in parallel across Rust + /// threads (respects `TOKENIZERS_PARALLELISM`), without holding the + /// interpreter lock. The batch version of `encode`. + #[pyo3(signature = (texts, *, add_special_tokens = true) -> "EncodingBatch")] + fn encode_batch( + slf: &Bound<'_, Self>, + texts: Vec, + add_special_tokens: bool, + ) -> PyResult { + let py = slf.py(); + let rows = slf + .get() + .inner + .with(py, |lock| -> PyResult>> { + let compiled = get_or_compile(&lock)?; + check_special_tokens_flag(&compiled, add_special_tokens)?; + encode_batch_core(&compiled, &texts) + })?; + let rows = rows.into_iter().map(Into::into).collect(); + Ok(PyEncodingBatch::new(rows, slf.clone().unbind())) + } + + /// Awaitable `encode`: same arguments and result, run in a worker thread. + #[pyo3(signature = (text, *, add_special_tokens = true) -> "Coroutine[Any, Any, Encoding]")] + fn async_encode<'py>( + slf: &Bound<'py, Self>, + text: &Bound<'py, PyAny>, + add_special_tokens: bool, + ) -> PyResult> { + to_thread(slf, "encode", text, add_special_tokens) + } + + /// Awaitable `encode_batch`: same arguments and result, run in a worker + /// thread while the batch encodes on Rust threads. + #[pyo3(signature = (texts, *, add_special_tokens = true) -> "Coroutine[Any, Any, EncodingBatch]")] + fn async_encode_batch<'py>( + slf: &Bound<'py, Self>, + texts: &Bound<'py, PyAny>, + add_special_tokens: bool, + ) -> PyResult> { + to_thread(slf, "encode_batch", texts, add_special_tokens) + } + /// Not implemented yet: decoding is not part of the encode pipeline. #[pyo3(signature = (ids, *, skip_special_tokens = true))] #[allow(unused_variables)] diff --git a/bindings/python/stubtest_allowlist.txt b/bindings/python/stubtest_allowlist.txt index 8e85580e5..98f948483 100644 --- a/bindings/python/stubtest_allowlist.txt +++ b/bindings/python/stubtest_allowlist.txt @@ -10,3 +10,7 @@ tokenizers.models.Model tokenizers.normalizers.Normalizer tokenizers.pre_tokenizers.PreTokenizer tokenizers.trainers.Trainer +# `__getitem__` is a slot method: its argument is positional-only at runtime, +# but pyo3 forbids a `signature` annotation on magic methods, so introspection +# emits it as positional-or-keyword. Runtime behaviour is correct. +tokenizers.EncodingBatch.__getitem__ diff --git a/bindings/python/tests/test_async.py b/bindings/python/tests/test_async.py index bc8a11d19..f9a1d9445 100644 --- a/bindings/python/tests/test_async.py +++ b/bindings/python/tests/test_async.py @@ -3,6 +3,8 @@ import numpy as np import pytest +from tokenizers import Encoding, EncodingBatch + from .conftest import SENTENCES, train_word_tokenizer @@ -20,6 +22,23 @@ async def go(): assert np.array_equal(got, want) +def test_async_encode_returns_encoding(): + tok = train_word_tokenizer() + + async def go(): + single = await tok.async_encode(SENTENCES[0], add_special_tokens=False) + batch = await tok.async_encode_batch(SENTENCES, add_special_tokens=False) + return single, batch + + single, batch = asyncio.run(go()) + assert isinstance(single, Encoding) + assert isinstance(batch, EncodingBatch) + assert single.ids == tok.encode(SENTENCES[0], add_special_tokens=False).ids + assert [batch[i].ids for i in range(len(batch))] == [ + tok.encode(line, add_special_tokens=False).ids for line in SENTENCES + ] + + def test_async_encodes_overlap(): tok = train_word_tokenizer() diff --git a/bindings/python/tests/test_encoding.py b/bindings/python/tests/test_encoding.py new file mode 100644 index 000000000..f75506817 --- /dev/null +++ b/bindings/python/tests/test_encoding.py @@ -0,0 +1,116 @@ +import numpy as np +import pytest + +from tokenizers import Encoding, EncodingBatch, Tokenizer, models, pre_tokenizers, trainers + +from .conftest import SENTENCES, train_word_tokenizer + + +@pytest.fixture(scope="module") +def special_tokenizer(): + """A BPE tokenizer whose vocabulary contains a special token (``), so + tests can feed that token in the input text.""" + tok = Tokenizer(models.BPE(unk_token="")) + tok.pre_tokenizer = pre_tokenizers.Whitespace() + tok.train_from_iterator( + ["hello world foo bar baz", "the quick brown fox"] * 50, + trainer=trainers.BpeTrainer(vocab_size=120, special_tokens=["", ""], show_progress=False), + ) + return tok + + +def test_encode_returns_encoding(word_tokenizer): + enc = word_tokenizer.encode(SENTENCES[0], add_special_tokens=False) + assert isinstance(enc, Encoding) + assert len(enc) == len(SENTENCES[0].split()) + assert repr(enc) == f"Encoding(length={len(enc)})" + + +def test_ids_match_encode_ids(word_tokenizer): + enc = word_tokenizer.encode(SENTENCES[0], add_special_tokens=False) + ids = word_tokenizer.encode_ids(SENTENCES[0], add_special_tokens=False) + assert enc.ids == ids.tolist() + assert np.array_equal(enc.ids_array(), ids) + assert enc.ids_array().dtype == np.uint32 + + +def test_ids_is_a_list_not_an_array(word_tokenizer): + # transformers' _pad does `input_ids + [pad] * n`; a numpy array would + # broadcast-add the padding into the token ids instead of concatenating. + enc = word_tokenizer.encode(SENTENCES[0], add_special_tokens=False) + assert isinstance(enc.ids, list) + assert enc.ids + [0, 0] == list(enc.ids) + [0, 0] + + +def test_tokens(word_tokenizer): + enc = word_tokenizer.encode(SENTENCES[0], add_special_tokens=False) + assert enc.tokens == SENTENCES[0].split() + + +def test_metadata_fields_are_constant_for_a_single_sequence(special_tokenizer): + # A special token appearing in the text is still an ordinary content token + # here: no post-processing means no special-tokens mask, one sequence. + enc = special_tokenizer.encode("hello world", add_special_tokens=False) + n = len(enc) + assert "" in enc.tokens + assert enc.type_ids == [0] * n + assert enc.attention_mask == [1] * n + assert enc.special_tokens_mask == [0] * n + assert enc.sequence_ids == [0] * n + assert enc.n_sequences == 1 + assert enc.token_to_sequence(0) == 0 + assert enc.token_to_sequence(n) is None + + +@pytest.mark.parametrize( + "access", + [ + lambda e: e.word_ids, + lambda e: e.offsets, + lambda e: e.char_to_token(0), + lambda e: e.char_to_word(0), + lambda e: e.token_to_chars(0), + lambda e: e.token_to_word(0), + lambda e: e.word_to_tokens(0), + lambda e: e.word_to_chars(0), + ], +) +def test_unavailable_features_raise(word_tokenizer, access): + enc = word_tokenizer.encode(SENTENCES[0], add_special_tokens=False) + with pytest.raises(NotImplementedError): + access(enc) + + +def test_encode_batch_returns_encoding_batch(word_tokenizer): + batch = word_tokenizer.encode_batch(SENTENCES, add_special_tokens=False) + assert isinstance(batch, EncodingBatch) + assert len(batch) == len(SENTENCES) + assert isinstance(batch[0], Encoding) + + +def test_batch_rows_match_single_encode(word_tokenizer): + batch = word_tokenizer.encode_batch(SENTENCES, add_special_tokens=False) + for i, line in enumerate(SENTENCES): + assert batch[i].ids == word_tokenizer.encode(line, add_special_tokens=False).ids + + +def test_batch_matches_encode_batch_ids(word_tokenizer): + batch = word_tokenizer.encode_batch(SENTENCES, add_special_tokens=False) + ids = word_tokenizer.encode_batch_ids(SENTENCES, add_special_tokens=False) + assert [batch[i].ids for i in range(len(batch))] == [row.tolist() for row in ids] + + +def test_batch_indexing_and_iteration(word_tokenizer): + batch = word_tokenizer.encode_batch(SENTENCES, add_special_tokens=False) + assert batch[-1].ids == batch[len(batch) - 1].ids + assert len(list(batch)) == len(SENTENCES) + with pytest.raises(IndexError): + _ = batch[len(batch)] + + +def test_encode_and_encode_ids_do_the_same_work(): + tok = train_word_tokenizer() + for line in SENTENCES: + assert ( + tok.encode(line, add_special_tokens=False).ids == tok.encode_ids(line, add_special_tokens=False).tolist() + ) From 520ca75c86b494b06616648fa125887f2f572ba1 Mon Sep 17 00:00:00 2001 From: SBrandeis <33657802+SBrandeis@users.noreply.github.com> Date: Thu, 23 Jul 2026 15:48:00 +0200 Subject: [PATCH 17/19] add fixme --- bindings/python/src/encoding.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bindings/python/src/encoding.rs b/bindings/python/src/encoding.rs index 3c8cafe68..55f46f819 100644 --- a/bindings/python/src/encoding.rs +++ b/bindings/python/src/encoding.rs @@ -47,6 +47,10 @@ impl PyEncoding { } /// The token ids, as a list. + // FIXME: copies Rust -> Python on every access — a fresh list, and each + // u32 boxed into an int object, with no caching. Ideally this would return + // a numpy uint32 array viewing the Arc buffer (zero-copy), which stays + // almost API-compatible: it indexes and iterates like a list, but isn't one. #[getter] fn ids(&self) -> Vec { self.ids.to_vec() From 1804a0b8379a2ac2e517f8a68035b832721ab409 Mon Sep 17 00:00:00 2001 From: SBrandeis <33657802+SBrandeis@users.noreply.github.com> Date: Thu, 23 Jul 2026 16:20:32 +0200 Subject: [PATCH 18/19] post-processor --- bindings/python/CHANGELOG.md | 23 +-- bindings/python/README.md | 38 +++-- bindings/python/examples/01_pretrained.py | 26 ++-- .../python/py_src/tokenizers/__init__.pyi | 22 ++- bindings/python/src/encoding.rs | 32 +++-- bindings/python/src/tokenizer.rs | 133 +++++++++--------- bindings/python/tests/test_encoding.py | 12 +- bindings/python/tests/test_pretrained.py | 12 +- .../tk-encode/src/tokenizer/pipeline.rs | 2 + 9 files changed, 151 insertions(+), 149 deletions(-) diff --git a/bindings/python/CHANGELOG.md b/bindings/python/CHANGELOG.md index bff62e2fa..fac03640a 100644 --- a/bindings/python/CHANGELOG.md +++ b/bindings/python/CHANGELOG.md @@ -15,23 +15,24 @@ arrays without a copy. Breaking changes — encoding: - `encode`/`encode_batch` return an `Encoding`/`EncodingBatch` carrying ids, - tokens, type ids, attention and special-tokens masks, and sequence ids - (`ids` is a `list`; `ids_array()` gives a numpy array). `encode_ids`/ - `encode_batch_ids` are the new names for the bare-`numpy.uint32` path — same - encode work, no `Encoding` wrapper. Not carried yet, and raising rather than - returning a guess: word ids and character offsets (and the char/word/token - mapping helpers built on them). Overflowing/stride, truncation, and padding + tokens, type ids, and the attention mask (`ids` is a `list`; `ids_array()` + gives a numpy array). `encode_ids`/`encode_batch_ids` are the new names for + the bare-`numpy.uint32` path — same encode work, no `Encoding` wrapper. Not + carried yet, and raising rather than returning a guess: special-tokens mask, + sequence ids, word ids, character offsets, and the char/word/token mapping + helpers built on them. Overflowing/stride, truncation, and padding (`enable_truncation`/`enable_padding` and their getters) are gone. - `encode` takes a single text: the `pair=` argument and the `is_pretokenized=` mode no longer exist (same for `encode_batch`). -- Not implemented yet (loud errors, never wrong ids): `decode`, - post-processor templates (pass `add_special_tokens=False`), and the +- `add_special_tokens=True` (the default) inserts the post-processor's template + tokens, as in 0.x; pass `False` to skip them. +- Not implemented yet (loud errors, never wrong ids): `decode` and the `Metaspace` pre-tokenizer. `decode_batch`, `DecodeStream`, `encode_batch_fast`, and `Tokenizer.post_process` are removed. - **`transformers`' `PreTrainedTokenizerFast` cannot run on 1.0 yet** — it - needs the not-yet-implemented `Encoding` fields (offsets), post-processing, - padding/truncation, and pair inputs. Pin `tokenizers<1.0` for `transformers` - until it targets 1.x. + needs the `Encoding` fields that still raise (offsets, special-tokens mask, + sequence ids), padding/truncation, and pair inputs. Pin `tokenizers<1.0` for + `transformers` until it targets 1.x. Breaking changes — components and introspection: diff --git a/bindings/python/README.md b/bindings/python/README.md index 467db067d..f9243b50d 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -14,28 +14,28 @@ import tokenizers as tk tok = tk.Tokenizer.from_file("tokenizer.json") -# encode returns an Encoding: ids plus the masks and metadata a model consumes. -# add_special_tokens=False skips template tokens like [CLS]/[SEP]; inserting -# them is not implemented yet in 1.0, so leaving it True raises a loud -# NotImplementedError on tokenizers that use such templates (BERT, Llama, …). -enc = tok.encode("Hello world", add_special_tokens=False) +# encode returns an Encoding: ids plus the metadata a model consumes. +# add_special_tokens=True (the default) inserts the tokenizer's template +# tokens, like BERT's [CLS]/[SEP]; pass False to leave them off. +enc = tok.encode("Hello world") enc.ids # list[int] enc.tokens # list[str] enc.attention_mask # list[int] -batch = tok.encode_batch(["Hello world", "How are you?"], add_special_tokens=False) +batch = tok.encode_batch(["Hello world", "How are you?"]) batch[0].ids # a batch is a sequence of Encodings # When you only want the ids, encode_ids skips the Encoding and returns them as # a numpy.uint32 array with no copy (encode_batch_ids returns a list of arrays). -ids = tok.encode_ids("Hello world", add_special_tokens=False) +ids = tok.encode_ids("Hello world") ``` `encode` and `encode_ids` run exactly the same work; `encode` wraps the ids in an `Encoding` and derives its fields on access, so you never pay for what you -don't read. Fields that need per-token provenance the pipeline does not compute -yet — `word_ids` and character `offsets` — raise `NotImplementedError` rather -than return a guess. +don't read. Fields that need per-token provenance the pipeline does not track +yet — which tokens are special (`special_tokens_mask`, `sequence_ids`), +`word_ids`, and character `offsets` — raise `NotImplementedError` rather than +return a guess. To load a tokenizer straight from the [Hugging Face Hub](https://huggingface.co), install the `hub` extra (`pip install 'tokenizers[hub]'`): @@ -81,31 +81,29 @@ plays well with threads and event loops: thread. ```python -enc = await tok.async_encode("Hello world", add_special_tokens=False) +enc = await tok.async_encode("Hello world") ``` ## Breaking changes vs 0.x 1.x is a ground-up rewrite with a smaller, faster API. The headline changes: -- `encode` returns an `Encoding` carrying ids, tokens, type ids, attention - and special-tokens masks, and sequence ids. Word ids and character offsets - are not computed yet and raise; truncation and padding +- `encode` returns an `Encoding` carrying ids, tokens, type ids, and the + attention mask. Special-tokens mask, sequence ids, word ids, and character + offsets are not computed yet and raise; truncation and padding (`enable_truncation`/`enable_padding`) are gone. `encode_ids` is the new name for a bare `numpy.uint32` id array. - `encode` takes a single text: no `pair=` argument, no `is_pretokenized=`. -- Not implemented yet (loud errors, never wrong ids): `decode`, - post-processor templates (`[CLS]`/`` insertion — pass - `add_special_tokens=False`), and the `Metaspace` pre-tokenizer - (t5-style files). +- Not implemented yet (loud errors, never wrong ids): `decode` and the + `Metaspace` pre-tokenizer (t5-style files). - Custom Python components (normalizers/pre-tokenizers written in Python) are not supported; components are plain values you assign, not objects you subclass. - `decoders`, `processors`, and the `implementations` helpers (`BertWordPieceTokenizer`, …) are gone. - **`transformers` cannot use 1.0 as its backend yet** — it needs the - not-yet-implemented pieces above (post-processing, offsets, decode). Pin - `tokenizers<1.0` for `transformers`. + `Encoding` fields that still raise (offsets, special-tokens mask, sequence + ids) plus `decode`. Pin `tokenizers<1.0` for `transformers`. The full list, including smaller removals and renames, is in the 1.0.0 entry of [CHANGELOG.md](CHANGELOG.md). diff --git a/bindings/python/examples/01_pretrained.py b/bindings/python/examples/01_pretrained.py index b4a1e5723..34832da0c 100644 --- a/bindings/python/examples/01_pretrained.py +++ b/bindings/python/examples/01_pretrained.py @@ -1,7 +1,7 @@ """The simplest starting point: load real tokenizer.json files and encode a -real corpus. Also demonstrates the two loud failure modes: pre-tokenizers the -pipeline does not support yet, and post-processing (special-token insertion), -which is not implemented yet. (Id parity against the released wheel is +real corpus, with and without the special tokens the post-processor inserts. +Also shows the one remaining loud failure mode: a pre-tokenizer the pipeline +does not support yet (Metaspace). (Id parity against the released wheel is checked by benches/bench_vs_release.py — the released package shares our name, so the comparison needs two processes.)""" @@ -32,17 +32,17 @@ assert first.attention_mask == [1] * len(first) print(f"{name}: {total} tokens, first token {first.tokens[0]!r}") -# Expected failure 1: post-processor would add special tokens -> loud error, -# not silently wrong ids +# Post-processing runs: add_special_tokens=True wraps the input with the +# template's special tokens ([CLS]/[SEP] for BERT); False leaves them off. bert = Tokenizer.from_file(DATA / "bert-base-uncased.json") -try: - bert.encode_ids("hello") - raise AssertionError("should have raised") -except NotImplementedError as e: - print(f"bert with add_special_tokens=True: NotImplementedError({e})") - -# Expected failure 2: pipeline-unsupported component (Metaspace) -> loud error -# at compile time, with the reason +wrapped = bert.encode("hello world") +plain = bert.encode("hello world", add_special_tokens=False) +assert wrapped.tokens[0] == "[CLS]" and wrapped.tokens[-1] == "[SEP]" +assert len(wrapped) == len(plain) + 2 +print(f"bert add_special_tokens: {wrapped.tokens}") + +# Loud failure mode: a pipeline-unsupported component (Metaspace) -> error at +# compile time, with the reason t5 = Tokenizer.from_file(DATA / "t5-base.json") try: t5.encode_ids("hello", add_special_tokens=False) diff --git a/bindings/python/py_src/tokenizers/__init__.pyi b/bindings/python/py_src/tokenizers/__init__.pyi index d33cc9b41..21a4a3096 100644 --- a/bindings/python/py_src/tokenizers/__init__.pyi +++ b/bindings/python/py_src/tokenizers/__init__.pyi @@ -51,12 +51,11 @@ class Encoding: `Encoding` costs the same to produce as a bare id array — `Tokenizer.encode` runs exactly the work `encode_ids` does. - `encode` only produces an `Encoding` for a single sequence with no - post-processor-inserted special tokens (it raises otherwise), so the - segment, attention, special-token and sequence values are constant: one - sequence numbered 0, nothing padded, nothing special. Anything that would - need per-token provenance the pipeline does not compute — word ids and - character offsets — raises rather than returning a plausible-looking guess. + `encode` handles a single sequence, so `type_ids` and `attention_mask` are + constant (one segment, nothing padded). Fields that need per-token + provenance the pipeline does not track yet — which tokens are special + (`special_tokens_mask`, `sequence_ids`), word ids, and character offsets — + raise rather than returning a plausible-looking guess. """ def __len__(self, /) -> int: ... def __repr__(self, /) -> str: ... @@ -92,18 +91,17 @@ class Encoding: @property def sequence_ids(self, /) -> list[int |None]: """ - The sequence each token belongs to: all 0 (single sequence). + The sequence each token belongs to — not available: it depends on which + tokens are special, which the pipeline does not mark yet. """ @property def special_tokens_mask(self, /) -> list[int]: """ - Special-tokens mask, one entry per token: all 0 (no post-processing). + Special-tokens mask — not available: the pipeline does not mark which + tokens are special yet (a backing structure for this is coming). """ def token_to_chars(self, /, token_index: int) -> tuple[int, int] |None: ... - def token_to_sequence(self, /, token_index: int) -> int |None: - """ - The sequence a token belongs to (0), or None for an out-of-range index. - """ + def token_to_sequence(self, /, token_index: int) -> int |None: ... def token_to_word(self, /, token_index: int) -> int |None: ... @property def tokens(self, /) -> list[str]: diff --git a/bindings/python/src/encoding.rs b/bindings/python/src/encoding.rs index 55f46f819..f3c6c69c3 100644 --- a/bindings/python/src/encoding.rs +++ b/bindings/python/src/encoding.rs @@ -8,6 +8,7 @@ use crate::tokenizer::PyTokenizer; const NO_OFFSETS: &str = "character offsets are not tracked by the encode pipeline"; const NO_WORD_IDS: &str = "word ids are not emitted by the encode pipeline"; +const NO_SPECIALS: &str = "the encode pipeline does not mark which tokens are special"; fn deferred(what: &str, why: &str) -> PyErr { PyNotImplementedError::new_err(format!("{what} is not available yet: {why}")) @@ -18,12 +19,11 @@ fn deferred(what: &str, why: &str) -> PyErr { /// `Encoding` costs the same to produce as a bare id array — `Tokenizer.encode` /// runs exactly the work `encode_ids` does. /// -/// `encode` only produces an `Encoding` for a single sequence with no -/// post-processor-inserted special tokens (it raises otherwise), so the -/// segment, attention, special-token and sequence values are constant: one -/// sequence numbered 0, nothing padded, nothing special. Anything that would -/// need per-token provenance the pipeline does not compute — word ids and -/// character offsets — raises rather than returning a plausible-looking guess. +/// `encode` handles a single sequence, so `type_ids` and `attention_mask` are +/// constant (one segment, nothing padded). Fields that need per-token +/// provenance the pipeline does not track yet — which tokens are special +/// (`special_tokens_mask`, `sequence_ids`), word ids, and character offsets — +/// raise rather than returning a plausible-looking guess. #[pyclass(frozen, name = "Encoding", module = "tokenizers")] pub struct PyEncoding { ids: Arc<[u32]>, @@ -82,16 +82,18 @@ impl PyEncoding { vec![1; self.ids.len()] } - /// Special-tokens mask, one entry per token: all 0 (no post-processing). + /// Special-tokens mask — not available: the pipeline does not mark which + /// tokens are special yet (a backing structure for this is coming). #[getter] - fn special_tokens_mask(&self) -> Vec { - vec![0; self.ids.len()] + fn special_tokens_mask(&self) -> PyResult> { + Err(deferred("special_tokens_mask", NO_SPECIALS)) } - /// The sequence each token belongs to: all 0 (single sequence). + /// The sequence each token belongs to — not available: it depends on which + /// tokens are special, which the pipeline does not mark yet. #[getter] - fn sequence_ids(&self) -> Vec> { - vec![Some(0); self.ids.len()] + fn sequence_ids(&self) -> PyResult>> { + Err(deferred("sequence_ids", NO_SPECIALS)) } /// Number of sequences in this encoding: always 1. @@ -100,10 +102,10 @@ impl PyEncoding { 1 } - /// The sequence a token belongs to (0), or None for an out-of-range index. #[pyo3(signature = (token_index))] - fn token_to_sequence(&self, token_index: usize) -> Option { - (token_index < self.ids.len()).then_some(0) + #[allow(unused_variables)] + fn token_to_sequence(&self, token_index: usize) -> PyResult> { + Err(deferred("token_to_sequence", NO_SPECIALS)) } /// Word id per token — not available: the pipeline does not emit word diff --git a/bindings/python/src/tokenizer.rs b/bindings/python/src/tokenizer.rs index 830d30784..8d9568205 100644 --- a/bindings/python/src/tokenizer.rs +++ b/bindings/python/src/tokenizer.rs @@ -14,7 +14,6 @@ use tk_encode::Tokenizer as SpecTokenizer; use tk_encode::pipeline::{ Model as _, PipelineModelScratch, PipelineToken, PipelineTokenizer, Span, }; -use tk_encode::tokenizer::PostProcessor as _; use tk_encode::utils::parallelism::get_parallelism; use tk_train::{TokenizerTrainExt, Trainable}; @@ -32,22 +31,11 @@ use crate::trainers::PyTrainer; /// that really used it — forking before any parallel work stays quiet. pub static USED_PARALLELISM: AtomicBool = AtomicBool::new(false); -/// The compiled encode path plus the facts about the spec the encode calls -/// need without re-locking it. -#[derive(Clone)] -struct Compiled { - pipe: Arc, - /// Whether the spec's post-processor would add special tokens. Post-processing - /// is not wired into the pipeline yet, so encode(add_special_tokens=True) - /// must fail loudly instead of silently dropping them. - post_adds_special_tokens: bool, -} - struct Inner { /// Source of truth: the mutable, serializable tokenizer definition. spec: SpecTokenizer, /// Memoized compilation of `spec`; invalidated by every mutation. - compiled: Option, + compiled: Option>, } fn poisoned(_: std::sync::PoisonError) -> PyErr { @@ -179,6 +167,44 @@ impl PyTokenizer { } Ok(()) } + + /// Compile if needed, then encode one text to raw ids with the GIL + /// released. Shared by `encode` and `encode_ids`. + fn run_encode( + &self, + py: Python<'_>, + text: &str, + add_special_tokens: bool, + ) -> PyResult> { + self.inner.with(py, |lock| { + let pipe = get_or_compile(&lock)?; + let mut pre_tokens = Vec::new(); + // TODO: reuse scratches across calls instead of building one per encode — + // see the ScratchPool pattern in https://github.com/huggingface/tokenizers/pull/2223 + let mut scratch = pipe.get_model().init_scratch(); + encode_one( + &pipe, + text, + &mut pre_tokens, + add_special_tokens, + &mut scratch, + ) + }) + } + + /// The batch counterpart of `run_encode`, shared by `encode_batch` and + /// `encode_batch_ids`. + fn run_encode_batch( + &self, + py: Python<'_>, + texts: &[PyBackedStr], + add_special_tokens: bool, + ) -> PyResult>> { + self.inner.with(py, |lock| { + let pipe = get_or_compile(&lock)?; + encode_batch_core(&pipe, texts, add_special_tokens) + }) + } } /// Wrap a bound method call in `asyncio.to_thread`, returning the coroutine. @@ -238,7 +264,7 @@ fn pretokenize( /// Get the compiled pipeline, building it from the spec on first use after a /// mutation. The `Detached` parameter is the proof this runs off the GIL. -fn get_or_compile(lock: &Detached<'_, Inner>) -> PyResult { +fn get_or_compile(lock: &Detached<'_, Inner>) -> PyResult> { { let guard = lock.read().map_err(poisoned)?; if let Some(compiled) = &guard.compiled { @@ -252,38 +278,22 @@ fn get_or_compile(lock: &Detached<'_, Inner>) -> PyResult { "this tokenizer cannot be compiled to an encode pipeline: {e}" )) })?; - let post_adds_special_tokens = guard - .spec - .get_post_processor() - .is_some_and(|p| p.added_tokens(false) > 0); - guard.compiled = Some(Compiled { - pipe: Arc::new(pipe), - post_adds_special_tokens, - }); + guard.compiled = Some(Arc::new(pipe)); } Ok(guard.compiled.clone().expect("just set")) } -fn check_special_tokens_flag(compiled: &Compiled, add_special_tokens: bool) -> PyResult<()> { - if add_special_tokens && compiled.post_adds_special_tokens { - return Err(PyNotImplementedError::new_err( - "this tokenizer's post-processor adds special tokens, but post-processing is not \ - implemented in the encode pipeline yet; pass add_special_tokens=False to encode \ - without them", - )); - } - Ok(()) -} - fn encode_one( pipe: &PipelineTokenizer, text: &str, pre_tokens: &mut Vec, + add_special_tokens: bool, scratch: &mut PipelineModelScratch, ) -> PyResult> { let mut output: Vec = Vec::new(); - pipe.encode_generic::<{ PipelineTokenizer::STAGE_MODEL }>( + pipe.encode_generic::<{ PipelineTokenizer::STAGE_POSTPROCESS }>( text, + add_special_tokens, pre_tokens, scratch, &mut output, @@ -296,22 +306,36 @@ fn encode_one( /// (which wraps each row in a numpy array) and `encode_batch` (which wraps the /// batch in an `EncodingBatch`). Runs on rayon threads when parallelism is on /// and the batch is worth splitting; the caller has already released the GIL. -fn encode_batch_core(compiled: &Compiled, texts: &[PyBackedStr]) -> PyResult>> { +fn encode_batch_core( + pipe: &PipelineTokenizer, + texts: &[PyBackedStr], + add_special_tokens: bool, +) -> PyResult>> { if get_parallelism() && texts.len() > 1 { USED_PARALLELISM.store(true, Ordering::SeqCst); texts .par_iter() .map_init( - || (Vec::new(), compiled.pipe.get_model().init_scratch()), - |(pre_tokens, scratch), text| encode_one(&compiled.pipe, text, pre_tokens, scratch), + || (Vec::new(), pipe.get_model().init_scratch()), + |(pre_tokens, scratch), text| { + encode_one(pipe, text, pre_tokens, add_special_tokens, scratch) + }, ) .collect() } else { let mut pre_tokens = Vec::new(); - let mut scratch = compiled.pipe.get_model().init_scratch(); + let mut scratch = pipe.get_model().init_scratch(); texts .iter() - .map(|text| encode_one(&compiled.pipe, text, &mut pre_tokens, &mut scratch)) + .map(|text| { + encode_one( + pipe, + text, + &mut pre_tokens, + add_special_tokens, + &mut scratch, + ) + }) .collect() } } @@ -392,13 +416,7 @@ impl PyTokenizer { text: &str, add_special_tokens: bool, ) -> PyResult>> { - let ids = self.inner.with(py, |lock| -> PyResult> { - let compiled = get_or_compile(&lock)?; - check_special_tokens_flag(&compiled, add_special_tokens)?; - let mut pre_tokens = Vec::new(); - let mut scratch = compiled.pipe.get_model().init_scratch(); - encode_one(&compiled.pipe, text, &mut pre_tokens, &mut scratch) - })?; + let ids = self.run_encode(py, text, add_special_tokens)?; Ok(ids.into_pyarray(py)) } @@ -413,11 +431,7 @@ impl PyTokenizer { texts: Vec, add_special_tokens: bool, ) -> PyResult> { - let batches = self.inner.with(py, |lock| -> PyResult>> { - let compiled = get_or_compile(&lock)?; - check_special_tokens_flag(&compiled, add_special_tokens)?; - encode_batch_core(&compiled, &texts) - })?; + let batches = self.run_encode_batch(py, &texts, add_special_tokens)?; let list = PyList::empty(py); for ids in batches { list.append(ids.into_pyarray(py))?; @@ -455,14 +469,7 @@ impl PyTokenizer { /// copies); the `Encoding` wraps the ids and derives its fields on access. #[pyo3(signature = (text, *, add_special_tokens = true) -> "Encoding")] fn encode(slf: &Bound<'_, Self>, text: &str, add_special_tokens: bool) -> PyResult { - let py = slf.py(); - let ids = slf.get().inner.with(py, |lock| -> PyResult> { - let compiled = get_or_compile(&lock)?; - check_special_tokens_flag(&compiled, add_special_tokens)?; - let mut pre_tokens = Vec::new(); - let mut scratch = compiled.pipe.get_model().init_scratch(); - encode_one(&compiled.pipe, text, &mut pre_tokens, &mut scratch) - })?; + let ids = slf.get().run_encode(slf.py(), text, add_special_tokens)?; Ok(PyEncoding::new(ids.into(), slf.clone().unbind())) } @@ -475,15 +482,9 @@ impl PyTokenizer { texts: Vec, add_special_tokens: bool, ) -> PyResult { - let py = slf.py(); let rows = slf .get() - .inner - .with(py, |lock| -> PyResult>> { - let compiled = get_or_compile(&lock)?; - check_special_tokens_flag(&compiled, add_special_tokens)?; - encode_batch_core(&compiled, &texts) - })?; + .run_encode_batch(slf.py(), &texts, add_special_tokens)?; let rows = rows.into_iter().map(Into::into).collect(); Ok(PyEncodingBatch::new(rows, slf.clone().unbind())) } diff --git a/bindings/python/tests/test_encoding.py b/bindings/python/tests/test_encoding.py index f75506817..3a5355db2 100644 --- a/bindings/python/tests/test_encoding.py +++ b/bindings/python/tests/test_encoding.py @@ -47,24 +47,22 @@ def test_tokens(word_tokenizer): assert enc.tokens == SENTENCES[0].split() -def test_metadata_fields_are_constant_for_a_single_sequence(special_tokenizer): - # A special token appearing in the text is still an ordinary content token - # here: no post-processing means no special-tokens mask, one sequence. +def test_constant_metadata_fields_for_a_single_sequence(special_tokenizer): + # type_ids and attention_mask are constant for one unpadded sequence. enc = special_tokenizer.encode("hello world", add_special_tokens=False) n = len(enc) assert "" in enc.tokens assert enc.type_ids == [0] * n assert enc.attention_mask == [1] * n - assert enc.special_tokens_mask == [0] * n - assert enc.sequence_ids == [0] * n assert enc.n_sequences == 1 - assert enc.token_to_sequence(0) == 0 - assert enc.token_to_sequence(n) is None @pytest.mark.parametrize( "access", [ + lambda e: e.special_tokens_mask, + lambda e: e.sequence_ids, + lambda e: e.token_to_sequence(0), lambda e: e.word_ids, lambda e: e.offsets, lambda e: e.char_to_token(0), diff --git a/bindings/python/tests/test_pretrained.py b/bindings/python/tests/test_pretrained.py index 65a505e74..73dc024bb 100644 --- a/bindings/python/tests/test_pretrained.py +++ b/bindings/python/tests/test_pretrained.py @@ -11,12 +11,14 @@ def test_gpt2_encodes_corpus(gpt2_file, corpus): assert all(ids.dtype == np.uint32 for ids in batch) -def test_bert_special_tokens_gate(bert_file): +def test_bert_add_special_tokens(bert_file): tok = Tokenizer.from_file(bert_file) - with pytest.raises(NotImplementedError, match="post-process"): - tok.encode_ids("hello") - ids = tok.encode_ids("hello", add_special_tokens=False) - assert len(ids) > 0 + wrapped = tok.encode_ids("hello") # add_special_tokens=True by default + plain = tok.encode_ids("hello", add_special_tokens=False) + # The post-processor wraps the content with [CLS] ... [SEP]. + assert len(wrapped) == len(plain) + 2 + assert tok.id_to_token(int(wrapped[0])) == "[CLS]" + assert tok.id_to_token(int(wrapped[-1])) == "[SEP]" def test_metaspace_fails_loudly_at_compile(t5_file): diff --git a/tokenizers/tk-encode/src/tokenizer/pipeline.rs b/tokenizers/tk-encode/src/tokenizer/pipeline.rs index 1526fc40f..42a6e7dbc 100644 --- a/tokenizers/tk-encode/src/tokenizer/pipeline.rs +++ b/tokenizers/tk-encode/src/tokenizer/pipeline.rs @@ -521,6 +521,8 @@ impl PipelineTokenizer { pub fn encode(&self, input: &str, add_special_tokens: bool) -> Result> { let mut output = Vec::new(); let mut pre_tokens = Vec::new(); + // TODO: reuse scratches across calls instead of building one per encode — + // see the ScratchPool pattern in https://github.com/huggingface/tokenizers/pull/2223 let mut scratch = self.model.init_scratch(); self.encode_generic::<{ Self::STAGE_POSTPROCESS }>( From 4162de82f45cc1efa090960a02e6a5b9453653e3 Mon Sep 17 00:00:00 2001 From: Simon Brandeis <33657802+SBrandeis@users.noreply.github.com> Date: Thu, 23 Jul 2026 22:12:09 +0200 Subject: [PATCH 19/19] feat(bindings): e2e and golden tests for Python bindings (#2235) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * golden end-to-end tests through transformers v5 Same suite, two builds: `make golden-release` runs it on the tokenizers wheel transformers resolves (always green, defines the target behavior); `make golden` runs it on the in-tree build, where the failures enumerate what the 1.0 rewrite is still missing. Four layers: the tokenizers API driven directly, the transformers tokenizer API, inference (generate/ pipeline), and training (Trainer/collators). Co-Authored-By: Claude Fable 5 * deepen golden tests with the transformers example recipes Scenarios lifted from the maintained examples/pytorch/ scripts at v5.14.1: run_qa's overflow/stride/sequence_ids/offsets span labeling, run_ner's word_ids label alignment, run_mlm's special_tokens_mask + masking collator, run_translation's text_target and seq2seq collator, plus token-classification pipeline spans, encoder-decoder generate, and the T5 Unigram+Metaspace archetype in the direct-API layer. 31 tests, all green on the release wheel. Co-Authored-By: Claude Fable 5 * golden production flows: chat serving, RAG, SFT Story-shaped tests mirroring how production stacks drive the tokenizer: chat serving (ChatML template + control tokens as added specials + history truncation to a token budget + left-padded batch generate + TextIteratorStreamer streaming == one-shot decode), RAG (token-budget chunking via offsets with exact source spans + mean-pooled embeddings + retrieval), and SFT (TRL-style completion-only masking, tokenizer pickled through datasets.map(num_proc=2), Trainer steps). 34 tests green on the release wheel. Co-Authored-By: Claude Fable 5 * golden conformance layer; rename the scenario suite to e2e tests/golden now holds true golden-master tests: goldens/*.jsonl record everything the released wheel (0.23.1) produces on the 8 bench-model tokenizers — ids, tokens, offsets, masks, word ids, decodes and pair encodings for fixture excerpts and edge-case strings, plus ids digests over the fixture corpora. generate.py regenerates them from .release/ (never hand-edited, guarded); test_golden.py replays the inputs on the current build, one test per output domain per model, so failures read as a conformance matrix. Opt-in via TOKENIZERS_GOLDEN (make golden) until the rewrite is green, then the gate should be dropped. The transformers scenario suite moves to tests/e2e (make e2e / e2e-release) — flows there, values here. CI: e2e-release and golden-freshness (regen must reproduce the committed goldens byte for byte) required; one informational in-tree job runs both. First catch, dev build vs release on llama-2: identical ids but tokens[0] renders as '▁' instead of '' — the Prepend normalizer leaks into the inserted special token's string. Co-Authored-By: Claude Fable 5 * add huggingface_hub to the dev venv from_pretrained needs it (the wheel's optional hub extra), so the network-marked pretrained test failed in a fresh venv — it only worked after `make examples` pulled the dep in via datasets. Co-Authored-By: Claude Fable 5 * review pass: scope HF_TOKEN per step, unshadow offset loop var Co-Authored-By: Claude Fable 5 * iter Makefile * fix tk-encode doctest: fence PipelinePostProcessor example as text rustdoc compiles indented doc blocks as Rust doctests; the example is an illustration, not compilable code. Co-Authored-By: Claude Fable 5 * fix Python lint under ruff 0.16 defaults; pin ruff in CI ruff 0.16.0 widened the default rule set, breaking CI with no code change: import sorting (I001), loop vars captured by bench lambdas (B023), implicit string concat in lists (ISC004), open() without a context manager (SIM115), zip-for-pairwise (RUF007), and a blind pytest.raises(Exception) (B017). Pin the CI ruff so rule-set changes land deliberately. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- .github/workflows/python-e2e-golden.yml | 153 ++++++++++++ .github/workflows/python.yml | 6 +- bindings/python/.gitignore | 4 + bindings/python/Makefile | 52 ++++- bindings/python/README.md | 14 +- bindings/python/benches/bench_vs_release.py | 25 +- bindings/python/examples/01_pretrained.py | 1 - .../python/examples/02_train_and_encode.py | 1 - .../python/examples/06_train_with_datasets.py | 1 - bindings/python/tests/conftest.py | 1 - bindings/python/tests/e2e/__init__.py | 0 bindings/python/tests/e2e/conftest.py | 37 +++ .../tests/e2e/relax_transformers_pin.py | 19 ++ bindings/python/tests/e2e/requirements.txt | 6 + bindings/python/tests/e2e/test_direct_api.py | 96 ++++++++ .../python/tests/e2e/test_production_flows.py | 189 +++++++++++++++ .../tests/e2e/test_transformers_inference.py | 78 +++++++ .../tests/e2e/test_transformers_tokenizer.py | 218 ++++++++++++++++++ .../tests/e2e/test_transformers_training.py | 123 ++++++++++ bindings/python/tests/golden/__init__.py | 0 bindings/python/tests/golden/conftest.py | 21 ++ bindings/python/tests/golden/generate.py | 172 ++++++++++++++ .../golden/goldens/bert-base-uncased.jsonl | 64 +++++ .../tests/golden/goldens/deepseek-v4.jsonl | 64 +++++ .../python/tests/golden/goldens/glm-5.2.jsonl | 64 +++++ .../python/tests/golden/goldens/gpt-oss.jsonl | 64 +++++ .../python/tests/golden/goldens/gpt2.jsonl | 64 +++++ .../python/tests/golden/goldens/llama-2.jsonl | 64 +++++ .../python/tests/golden/goldens/llama-3.jsonl | 64 +++++ .../python/tests/golden/goldens/t5-base.jsonl | 64 +++++ bindings/python/tests/golden/test_golden.py | 133 +++++++++++ bindings/python/tests/test_async.py | 1 - bindings/python/tests/test_components.py | 3 +- bindings/python/tests/test_encoding.py | 1 - bindings/python/tests/test_parity_trainer.py | 1 - bindings/python/tests/test_pretrained.py | 1 - bindings/python/tests/test_tokenizer.py | 1 - bindings/python/tests/test_trainers.py | 1 - .../tk-encode/src/tokenizer/pipeline.rs | 17 +- 39 files changed, 1847 insertions(+), 41 deletions(-) create mode 100644 .github/workflows/python-e2e-golden.yml create mode 100644 bindings/python/tests/e2e/__init__.py create mode 100644 bindings/python/tests/e2e/conftest.py create mode 100644 bindings/python/tests/e2e/relax_transformers_pin.py create mode 100644 bindings/python/tests/e2e/requirements.txt create mode 100644 bindings/python/tests/e2e/test_direct_api.py create mode 100644 bindings/python/tests/e2e/test_production_flows.py create mode 100644 bindings/python/tests/e2e/test_transformers_inference.py create mode 100644 bindings/python/tests/e2e/test_transformers_tokenizer.py create mode 100644 bindings/python/tests/e2e/test_transformers_training.py create mode 100644 bindings/python/tests/golden/__init__.py create mode 100644 bindings/python/tests/golden/conftest.py create mode 100644 bindings/python/tests/golden/generate.py create mode 100644 bindings/python/tests/golden/goldens/bert-base-uncased.jsonl create mode 100644 bindings/python/tests/golden/goldens/deepseek-v4.jsonl create mode 100644 bindings/python/tests/golden/goldens/glm-5.2.jsonl create mode 100644 bindings/python/tests/golden/goldens/gpt-oss.jsonl create mode 100644 bindings/python/tests/golden/goldens/gpt2.jsonl create mode 100644 bindings/python/tests/golden/goldens/llama-2.jsonl create mode 100644 bindings/python/tests/golden/goldens/llama-3.jsonl create mode 100644 bindings/python/tests/golden/goldens/t5-base.jsonl create mode 100644 bindings/python/tests/golden/test_golden.py diff --git a/.github/workflows/python-e2e-golden.yml b/.github/workflows/python-e2e-golden.yml new file mode 100644 index 000000000..cc481dc5d --- /dev/null +++ b/.github/workflows/python-e2e-golden.yml @@ -0,0 +1,153 @@ +name: Python e2e & golden tests + +# Two complementary suites for the 1.0 bindings rewrite (bindings/python/tests): +# e2e (tests/e2e) — transformers scenario tests. Run on the wheel +# transformers resolves (required green) and on the +# in-tree build (informational while incomplete). +# golden (tests/golden) — exact-output conformance against committed goldens +# generated from the released wheel. Required: the +# goldens must reproduce from the release. The +# in-tree run is informational while incomplete. + +on: + push: + branches: + - main + paths-ignore: + - bindings/node/** + pull_request: + paths-ignore: + - bindings/node/** + +env: + # CI runners have no GPU: resolve torch from the CPU index instead of + # pulling ~2 GB of CUDA wheels. + UV_TORCH_BACKEND: cpu + +jobs: + e2e-release: + name: e2e on released tokenizers (must pass) + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Install uv + uses: astral-sh/setup-uv@v6 + + - name: Cache Hub downloads + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: ~/.cache/huggingface + key: e2e-hub-${{ hashFiles('bindings/python/tests/e2e/*') }} + restore-keys: e2e-hub- + + - name: Run the e2e suite + working-directory: ./bindings/python + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + run: make e2e-release + + golden-freshness: + name: goldens reproduce from the released wheel (must pass) + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Install uv + uses: astral-sh/setup-uv@v6 + + - name: Cache test data + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: tokenizers/data + key: golden-data-${{ hashFiles('tokenizers/Makefile', 'tokenizers/tk-encode/examples/bench_models.json') }} + + - name: Download model tokenizers and fixture corpora + working-directory: tokenizers + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + run: make bench-models fixtures HF="uvx --from huggingface_hub hf" + + # Regenerating must reproduce the committed goldens byte for byte. A + # difference means either an uncommitted regen, or a new tokenizers + # release that changed behavior — both need a human decision. + - name: Regenerate and diff the goldens + working-directory: ./bindings/python + run: | + make golden-regen + if [ -n "$(git status --porcelain -- tests/golden/goldens)" ]; then + git status --porcelain -- tests/golden/goldens + git diff -- tests/golden/goldens | head -100 + echo "::error::committed goldens do not reproduce from the released wheel — run 'make golden-regen' and commit, or investigate the release" + exit 1 + fi + + in-tree: + name: e2e + golden on the in-tree build (informational) + runs-on: ubuntu-latest + continue-on-error: true + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Install Rust + uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable + + - name: Cache cargo registry / git / target + uses: Swatinem/rust-cache@23869a5bd66c73db3c0ac40331f3206eb23791dc # v2.9.1 + with: + workspaces: bindings/python + shared-key: python + cache-bin: false + + - name: Install uv + uses: astral-sh/setup-uv@v6 + + - name: Cache test data + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: tokenizers/data + key: golden-data-${{ hashFiles('tokenizers/Makefile', 'tokenizers/tk-encode/examples/bench_models.json') }} + + - name: Cache Hub downloads + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: ~/.cache/huggingface + key: e2e-hub-${{ hashFiles('bindings/python/tests/e2e/*') }} + restore-keys: e2e-hub- + + - name: Download model tokenizers and fixture corpora + working-directory: tokenizers + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + run: make bench-models fixtures HF="uvx --from huggingface_hub hf" + + - name: Run the golden conformance layer + working-directory: ./bindings/python + run: | + set -o pipefail + make golden 2>&1 | tee golden.log + + - name: Run the e2e suite + if: always() + working-directory: ./bindings/python + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + run: | + set -o pipefail + make e2e 2>&1 | tee e2e.log + + # The point of these runs while the rewrite is in progress: surface + # what still fails, without hunting through the logs. + - name: Post the failure overview to the step summary + if: always() + working-directory: ./bindings/python + run: | + for suite in golden e2e; do + echo "## ${suite} on the in-tree build" + echo '```' + sed -n '/short test summary info/,$p' "${suite}.log" 2>/dev/null || echo "no ${suite} results" + echo '```' + done >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index c2813bc98..1a93e813e 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -170,11 +170,13 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v6 + # Pinned: a ruff release can change the default rule set and break CI + # without any code change (0.16.0 did). Bump deliberately. - name: Lint the Python sources with ruff working-directory: ./bindings/python run: | - uvx ruff check benches examples tests - uvx ruff format --check benches examples tests + uvx ruff@0.16.0 check benches examples tests + uvx ruff@0.16.0 format --check benches examples tests audit: name: Audit dependencies diff --git a/bindings/python/.gitignore b/bindings/python/.gitignore index dfeb86359..a7ee81fc1 100644 --- a/bindings/python/.gitignore +++ b/bindings/python/.gitignore @@ -1,6 +1,10 @@ .venv/ .venv-ft/ +.venv-e2e/ +.venv-e2e-release/ .release/ +e2e.log +golden.log python_bench.json python_bench.md .smoke/ diff --git a/bindings/python/Makefile b/bindings/python/Makefile index 7aef0366e..ddd6ad203 100644 --- a/bindings/python/Makefile +++ b/bindings/python/Makefile @@ -5,9 +5,11 @@ PYTHON := .venv/bin/python dev: .venv . .venv/bin/activate && maturin develop --release +# huggingface_hub backs from_pretrained (the wheel's optional `hub` extra); +# the pretrained tests and example 01 need it. .venv: uv venv .venv - uv pip install --python $(PYTHON) maturin numpy pytest ruff mypy + uv pip install --python $(PYTHON) maturin numpy pytest ruff mypy huggingface_hub # Regenerate the .pyi stubs from the built extension. Run after `make dev`. .PHONY: stubs @@ -19,10 +21,17 @@ test: dev $(PYTHON) -m pytest -q $(PYTHON) -m mypy.stubtest tokenizers --allowlist stubtest_allowlist.txt +# Examples 01-03 read this corpus; fetch it through the tokenizers repo's +# data rule, pointing it at the dev venv's `hf` CLI so nothing global is +# needed. +TOKENIZERS_DIR := ../../tokenizers +$(TOKENIZERS_DIR)/data/big.txt: | .venv + $(MAKE) -C $(TOKENIZERS_DIR) data/big.txt HF=$(abspath .venv/bin/hf) + # Run the end-to-end examples. `datasets` (for 06) is installed on demand so # `make dev` stays lean; 06 downloads wikitext-2 (~12 MB) on first run. .PHONY: examples -examples: dev +examples: dev $(TOKENIZERS_DIR)/data/big.txt uv pip install --python $(PYTHON) datasets $(PYTHON) examples/01_pretrained.py $(PYTHON) examples/02_train_and_encode.py @@ -31,6 +40,43 @@ examples: dev $(PYTHON) examples/05_train_bytelevel_bpe.py $(PYTHON) examples/06_train_with_datasets.py +# End-to-end tests (tests/e2e): the transformers scenario suite, two builds. +# e2e — the in-tree build; failures list what the rewrite still misses +# e2e-release — the tokenizers wheel transformers resolves; must always pass +# Dedicated venvs: transformers+torch are heavy, and each venv carries its own +# tokenizers. The dev venv also needs transformers' import-time tokenizers pin +# lifted, or it refuses our 1.0.0-dev version (see relax_transformers_pin.py). +.venv-e2e: + uv venv $@ --python 3.12 + uv pip install --python $@/bin/python -r tests/e2e/requirements.txt maturin + $@/bin/python tests/e2e/relax_transformers_pin.py + +.PHONY: e2e +e2e: .venv-e2e + . .venv-e2e/bin/activate && maturin develop --release + .venv-e2e/bin/python -m pytest tests/e2e -q -ra --continue-on-collection-errors + +.venv-e2e-release: + uv venv $@ --python 3.12 + uv pip install --python $@/bin/python -r tests/e2e/requirements.txt + +.PHONY: e2e-release +e2e-release: .venv-e2e-release + .venv-e2e-release/bin/python -m pytest tests/e2e -q + +# Golden conformance layer (tests/golden): exact encode/decode outputs on the +# data/ fixtures, diffed against committed goldens. The goldens are only ever +# regenerated from the released wheel (golden-regen) — never hand-edited. +# `golden` is opt-in (env gate) while the rewrite is incomplete; drop the gate +# once it runs green so plain pytest enforces conformance everywhere. +.PHONY: golden +golden: dev + TOKENIZERS_GOLDEN=1 $(PYTHON) -m pytest tests/golden -q -ra + +.PHONY: golden-regen +golden-regen: .venv .release + PYTHONPATH=.release $(PYTHON) tests/golden/generate.py + # The released PyPI wheel shares our package name, so it lives in its own # directory that the bench subprocess puts on PYTHONPATH. .release: | .venv @@ -50,4 +96,4 @@ lint: .venv .PHONY: clean clean: - rm -rf .venv .release target tools/stub-gen/target + rm -rf .venv .venv-e2e .venv-e2e-release .release target tools/stub-gen/target diff --git a/bindings/python/README.md b/bindings/python/README.md index f9243b50d..a7c88ecad 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -136,11 +136,15 @@ per-version with `maturin build --no-default-features`. Other targets: ```sh -make test # pytest suite in tests/ -make examples # run the end-to-end examples (needs ../../tokenizers/data) -make bench # benchmark against the released tokenizers wheel from PyPI -make stubs # regenerate the .pyi type stubs from the built extension -make lint # cargo fmt + clippy, ruff over the python sources +make test # pytest suite in tests/ +make examples # run the end-to-end examples (needs ../../tokenizers/data) +make bench # benchmark against the released tokenizers wheel from PyPI +make e2e # transformers end-to-end suite (tests/e2e) on this build +make e2e-release # same suite on the released wheel — must always pass +make golden # exact-output conformance vs the committed goldens (tests/golden) +make golden-regen # regenerate the goldens from the released wheel +make stubs # regenerate the .pyi type stubs from the built extension +make lint # cargo fmt + clippy, ruff over the python sources ``` The examples and the benchmark read test data from `../../tokenizers/data`. diff --git a/bindings/python/benches/bench_vs_release.py b/bindings/python/benches/bench_vs_release.py index d974b15be..c8fc56850 100644 --- a/bindings/python/benches/bench_vs_release.py +++ b/bindings/python/benches/bench_vs_release.py @@ -37,7 +37,6 @@ from pathlib import Path import numpy as np - import tokenizers # Keep in sync with fixture_bench.rs (CHUNK_BYTES, MAX_CHUNKS). @@ -117,7 +116,7 @@ def bench_release_side(models: list[dict], fixtures: list[dict], iters: int) -> for fixture in fixtures: chunks = fixture["chunks"] encoded = tok.encode_batch_fast(chunks, add_special_tokens=False) - t = timed(lambda: tok.encode_batch_fast(chunks, add_special_tokens=False), iters) + t = timed(lambda tok=tok, chunks=chunks: tok.encode_batch_fast(chunks, add_special_tokens=False), iters) rows.append( { "mbps": fixture["bytes"] / t / 1e6, @@ -125,7 +124,7 @@ def bench_release_side(models: list[dict], fixtures: list[dict], iters: int) -> } ) os.environ["TOKENIZERS_PARALLELISM"] = "true" - t = timed(lambda: tok.encode_batch_fast(all_chunks, add_special_tokens=False), iters) + t = timed(lambda tok=tok: tok.encode_batch_fast(all_chunks, add_special_tokens=False), iters) out["models"][model["name"]] = {"fixtures": rows, "multi_thread_mbps": nbytes / t / 1e6} return out @@ -136,7 +135,7 @@ def bench_local_side(tok, fixtures: list[dict], release_row: dict, iters: int) - for fixture, rel in zip(fixtures, release_row["fixtures"], strict=True): chunks = fixture["chunks"] encoded = tok.encode_batch_ids(chunks, add_special_tokens=False) - t = timed(lambda: tok.encode_batch_ids(chunks, add_special_tokens=False), iters) + t = timed(lambda chunks=chunks: tok.encode_batch_ids(chunks, add_special_tokens=False), iters) mbps = fixture["bytes"] / t / 1e6 row["fixtures"].append( { @@ -169,13 +168,17 @@ def render_markdown(report: dict) -> str: lines = [ f"## Python bindings: this branch vs `tokenizers` {report['release_version']} (PyPI)", "", - f"{report['fixture_count']} fixtures (~10 KiB chunks, ≤100/fixture), median of " - f"{report['iters']} runs, {report['cpus']} CPUs. Single-thread numbers aggregate " - "all fixtures (speedup range = slowest…fastest fixture); multi-thread runs the " - "flattened corpus. Speedup >1 means this branch is faster.", + ( + f"{report['fixture_count']} fixtures (~10 KiB chunks, ≤100/fixture), median of " + f"{report['iters']} runs, {report['cpus']} CPUs. Single-thread numbers aggregate " + "all fixtures (speedup range = slowest…fastest fixture); multi-thread runs the " + "flattened corpus. Speedup >1 means this branch is faster." + ), "", - "| model | ids | branch 1t (MB/s) | release 1t (MB/s) | speedup 1t (range) " - "| branch mt (MB/s) | release mt (MB/s) | speedup mt |", + ( + "| model | ids | branch 1t (MB/s) | release 1t (MB/s) | speedup 1t (range) " + "| branch mt (MB/s) | release mt (MB/s) | speedup mt |" + ), "|---|---|---|---|---|---|---|---|", ] for row in report["models"]: @@ -224,7 +227,7 @@ def main() -> int: f"`uv pip install --target {args.release_dir} tokenizers`" ) - models = json.load(open(args.manifest)) if args.manifest else DEFAULT_MODELS + models = json.loads(args.manifest.read_text()) if args.manifest else DEFAULT_MODELS for model in models: model["path"] = str(args.data_dir / model.get("file", model["name"] + ".json")) missing = [m["name"] for m in models if not Path(m["path"]).is_file()] diff --git a/bindings/python/examples/01_pretrained.py b/bindings/python/examples/01_pretrained.py index 34832da0c..1136b135a 100644 --- a/bindings/python/examples/01_pretrained.py +++ b/bindings/python/examples/01_pretrained.py @@ -8,7 +8,6 @@ from pathlib import Path import numpy as np - from tokenizers import Tokenizer, TokenizersError DATA = Path(__file__).resolve().parents[3] / "tokenizers" / "data" diff --git a/bindings/python/examples/02_train_and_encode.py b/bindings/python/examples/02_train_and_encode.py index a61af6f2a..904a87d39 100644 --- a/bindings/python/examples/02_train_and_encode.py +++ b/bindings/python/examples/02_train_and_encode.py @@ -7,7 +7,6 @@ from pathlib import Path import numpy as np - from tokenizers import AddedToken, Tokenizer, models, normalizers, pre_tokenizers, trainers DATA = Path(__file__).resolve().parents[3] / "tokenizers" / "data" diff --git a/bindings/python/examples/06_train_with_datasets.py b/bindings/python/examples/06_train_with_datasets.py index cc2ad3e0d..6a6593ff1 100644 --- a/bindings/python/examples/06_train_with_datasets.py +++ b/bindings/python/examples/06_train_with_datasets.py @@ -5,7 +5,6 @@ Needs the `datasets` package; downloads wikitext-2 (~12 MB) on first run.""" import datasets - from tokenizers import Tokenizer, models, normalizers, pre_tokenizers tokenizer = Tokenizer(models.BPE()) diff --git a/bindings/python/tests/conftest.py b/bindings/python/tests/conftest.py index ce515e14c..93533fdf4 100644 --- a/bindings/python/tests/conftest.py +++ b/bindings/python/tests/conftest.py @@ -1,7 +1,6 @@ from pathlib import Path import pytest - from tokenizers import Tokenizer, models, pre_tokenizers, trainers DATA = Path(__file__).resolve().parents[3] / "tokenizers" / "data" diff --git a/bindings/python/tests/e2e/__init__.py b/bindings/python/tests/e2e/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/bindings/python/tests/e2e/conftest.py b/bindings/python/tests/e2e/conftest.py new file mode 100644 index 000000000..7610ee303 --- /dev/null +++ b/bindings/python/tests/e2e/conftest.py @@ -0,0 +1,37 @@ +"""End-to-end tests: the behavior the 1.0 bindings rewrite works toward. + +Each file drives the bindings the way a real application would, one layer +further from the library than the last: + +- test_direct_api.py — the `tokenizers` API itself +- test_transformers_tokenizer.py — the transformers tokenizer API, which wraps `tokenizers` +- test_transformers_inference.py — transformers inference (`generate`, `pipeline`) +- test_transformers_training.py — transformers training (`Trainer`, data collators) +- test_production_flows.py — whole production stories: chat serving, RAG, SFT + +Against the tokenizers wheel that transformers v5 resolves, every test passes: +`make e2e-release`. Against the in-tree build (`make e2e`) the failures +enumerate what the rewrite is still missing — expected until it is complete. +A test that fails on both sides is a bug in the test. + +These files check *flows*; the golden layer in tests/golden checks *values* +(every id, offset, and decode against the released wheel, on the data/ +fixtures). + +The suite runs in its own venv (transformers + torch, see requirements.txt) +and downloads a few small models from the Hub on first run. Without +transformers installed — e.g. during a plain `make test` — it skips itself. +""" + +import pytest + +pytest.importorskip("transformers", reason="e2e tests run in a dedicated venv — use `make e2e`") + +# Randomly-initialized miniatures of the real architectures: full tokenizer +# and model plumbing at a few MB per download. Their outputs are gibberish, +# so tests assert mechanics (ids, shapes, round-trips), never quality. +TINY_GPT2 = "hf-internal-testing/tiny-random-gpt2" +TINY_BERT_CLS = "hf-internal-testing/tiny-random-BertForSequenceClassification" +TINY_BERT_MLM = "hf-internal-testing/tiny-random-BertForMaskedLM" +TINY_BERT_NER = "hf-internal-testing/tiny-random-BertForTokenClassification" +TINY_T5 = "hf-internal-testing/tiny-random-t5" diff --git a/bindings/python/tests/e2e/relax_transformers_pin.py b/bindings/python/tests/e2e/relax_transformers_pin.py new file mode 100644 index 000000000..595d36318 --- /dev/null +++ b/bindings/python/tests/e2e/relax_transformers_pin.py @@ -0,0 +1,19 @@ +"""Let transformers import with the in-tree tokenizers build installed. + +transformers 5.x refuses to import unless the installed tokenizers version +satisfies the range in its dependency_versions_table.py (checked at import +time), which rejects our 1.0.0-dev build. This rewrites that one entry in the +venv's copy of the table to an unversioned "tokenizers". Run it once per +venv, after installing transformers; `make e2e` does. + +The file is located without importing transformers: importing it would run +the very check being lifted. +""" + +import importlib.util +import pathlib +import re + +table = pathlib.Path(importlib.util.find_spec("transformers").origin).with_name("dependency_versions_table.py") +table.write_text(re.sub(r'"tokenizers": "tokenizers[^"]*"', '"tokenizers": "tokenizers"', table.read_text())) +print(f"tokenizers pin relaxed in {table}") diff --git a/bindings/python/tests/e2e/requirements.txt b/bindings/python/tests/e2e/requirements.txt new file mode 100644 index 000000000..a7f51dbf2 --- /dev/null +++ b/bindings/python/tests/e2e/requirements.txt @@ -0,0 +1,6 @@ +# Environment for the e2e tests. The transformers stack is heavy and +# resolves its own tokenizers wheel, so it lives in dedicated venvs +# (.venv-e2e / .venv-e2e-release) — see the e2e targets in the Makefile. +transformers[torch]>=5,<6 +datasets +pytest diff --git a/bindings/python/tests/e2e/test_direct_api.py b/bindings/python/tests/e2e/test_direct_api.py new file mode 100644 index 000000000..9653796a2 --- /dev/null +++ b/bindings/python/tests/e2e/test_direct_api.py @@ -0,0 +1,96 @@ +"""Layer 1: the `tokenizers` API driven directly. + +The library's bread and butter: load a pretrained tokenizer, turn raw strings +into padded/truncated id batches a model can consume, then map ids back to +text (decode) and tokens back to source positions (offsets). +""" + +import pytest +from tokenizers import Tokenizer + + +@pytest.fixture +def bert(): + # Function-scoped: enable_truncation/enable_padding mutate the tokenizer, + # so every test gets a fresh instance (loaded from the local Hub cache). + return Tokenizer.from_pretrained("bert-base-uncased") + + +def test_encode_wraps_with_special_tokens(bert): + enc = bert.encode("Hello world") + + assert enc.tokens == ["[CLS]", "hello", "world", "[SEP]"] + assert enc.ids == [101, 7592, 2088, 102] + assert enc.type_ids == [0, 0, 0, 0] + assert enc.attention_mask == [1, 1, 1, 1] + assert enc.special_tokens_mask == [1, 0, 0, 1] + + +def test_question_context_pair(bert): + # QA and reranking models take two sequences in one encoding, told apart + # by type_ids; sequence_ids gives None on the template's special tokens. + enc = bert.encode("Where is Paris?", "Paris is in France.") + + assert enc.tokens.count("[SEP]") == 2 + boundary = enc.tokens.index("[SEP]") + assert all(t == 0 for t in enc.type_ids[: boundary + 1]) + assert all(t == 1 for t in enc.type_ids[boundary + 1 :]) + assert enc.sequence_ids[0] is None + assert enc.sequence_ids[1] == 0 + assert enc.sequence_ids[-2] == 1 + assert enc.sequence_ids[-1] is None + + +def test_batch_padded_and_truncated_to_fixed_shape(bert): + # Fixed-shape batches: long inputs truncated (keeping the special-token + # template), short ones padded, attention_mask flagging real tokens. + bert.enable_truncation(max_length=6) + bert.enable_padding(length=6, pad_token="[PAD]", pad_id=0) + + short, long = bert.encode_batch(["One two", "one two three four five six seven eight"]) + + assert short.tokens == ["[CLS]", "one", "two", "[SEP]", "[PAD]", "[PAD]"] + assert short.attention_mask == [1, 1, 1, 1, 0, 0] + assert len(long.ids) == 6 + assert long.tokens[0] == "[CLS]" + assert long.tokens[-1] == "[SEP]" + assert long.attention_mask == [1, 1, 1, 1, 1, 1] + + +def test_offsets_point_back_into_the_source(bert): + # Offsets power everything that maps tokens to source positions, e.g. + # highlighting an answer span. Special tokens have no source, WordPiece + # continuations drop their "##" marker. + text = "Tokenization highlights substrings." + enc = bert.encode(text) + + for token, (start, end), special in zip(enc.tokens, enc.offsets, enc.special_tokens_mask): + if not special: + assert text[start:end].lower() == token.removeprefix("##") + + +def test_decode_round_trips(bert): + enc = bert.encode("hello world") + + assert bert.decode(enc.ids, skip_special_tokens=True) == "hello world" + + +def test_byte_level_round_trip(): + # GPT-2's byte-level BPE is the other big archetype: no [UNK], any text + # survives encode/decode byte for byte. + gpt2 = Tokenizer.from_pretrained("openai-community/gpt2") + text = "Byte-level BPE round-trips emoji 🤗 and accents é!" + + assert gpt2.encode("Hello world").ids == [15496, 995] + assert gpt2.decode(gpt2.encode(text).ids) == text + + +def test_sentencepiece_unigram_archetype(): + # T5's Unigram + Metaspace pipeline: the third big pretrained archetype + # after WordPiece and byte-level BPE. + t5 = Tokenizer.from_pretrained("google-t5/t5-small") + enc = t5.encode("Hello world") + + assert enc.tokens == ["▁Hello", "▁world", ""] + assert enc.ids == [8774, 296, 1] + assert t5.decode(enc.ids, skip_special_tokens=True) == "Hello world" diff --git a/bindings/python/tests/e2e/test_production_flows.py b/bindings/python/tests/e2e/test_production_flows.py new file mode 100644 index 000000000..e0b91b940 --- /dev/null +++ b/bindings/python/tests/e2e/test_production_flows.py @@ -0,0 +1,189 @@ +"""Layer 5: production flows. + +Where the recipe tests pinpoint single features, each test here plays out one +whole production story from the three domains where transformers earns its +keep — LLM chat serving, retrieval (RAG), and supervised fine-tuning — the +way serving stacks (vLLM, TGI, `transformers serve`), embedding pipelines +(sentence-transformers, text splitters), and SFT trainers (TRL) drive the +tokenizer. Models stay tiny and random (see conftest): the assertions pin the +tokenizer's side of the story, not model quality. +""" + +import itertools +import math +from threading import Thread + +import datasets +import torch +from transformers import ( + AutoModel, + AutoModelForCausalLM, + AutoTokenizer, + DataCollatorForSeq2Seq, + TextIteratorStreamer, + Trainer, + TrainingArguments, +) + +from .conftest import TINY_BERT_MLM, TINY_GPT2 + +# The ChatML conversation format used by the SmolLM/Qwen instruct families. +CHATML_TEMPLATE = ( + "{% for message in messages %}" + "{{ '<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n' }}" + "{% endfor %}" + "{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}" +) + + +def chat_tokenizer(**kwargs): + # A serving or training stack customizes its tokenizer once at startup: + # chat template, the template's control tokens as specials, a pad token. + tok = AutoTokenizer.from_pretrained(TINY_GPT2, **kwargs) + tok.chat_template = CHATML_TEMPLATE + tok.add_special_tokens({"additional_special_tokens": ["<|im_start|>", "<|im_end|>"]}) + tok.pad_token = tok.eos_token + return tok + + +def test_chat_serving_flow(): + tok = chat_tokenizer(padding_side="left") # prompts flush against generation + model = AutoModelForCausalLM.from_pretrained(TINY_GPT2) + model.resize_token_embeddings(len(tok)) + + # The control tokens registered as specials encode to single ids and are + # stripped from decoded replies. + im_start = tok.convert_tokens_to_ids("<|im_start|>") + assert tok("<|im_start|>", add_special_tokens=False)["input_ids"] == [im_start] + + # Fit the conversation into the context budget by dropping the oldest + # exchanges; the system prompt always survives. + def n_tokens(conversation): + text = tok.apply_chat_template(conversation, tokenize=False, add_generation_prompt=True) + return len(tok(text, add_special_tokens=False)["input_ids"]) + + conversation = [{"role": "system", "content": "You are a concise assistant."}] + for i in range(3): + conversation += [ + {"role": "user", "content": f"Tell me an interesting fact, number {i}, please."}, + {"role": "assistant", "content": f"Here is interesting fact number {i} for you."}, + ] + conversation.append({"role": "user", "content": "Now summarize them all."}) + + budget = n_tokens(conversation) - 1 + while n_tokens(conversation) > budget: + del conversation[1:3] + assert n_tokens(conversation) <= budget + assert conversation[0]["role"] == "system" + assert len(conversation) == 6 # exactly one exchange dropped + + # Serve a batch of two conversations, left-padded to one rectangle. + other = [{"role": "user", "content": "Hi!"}] + prompts = [tok.apply_chat_template(c, tokenize=False, add_generation_prompt=True) for c in (conversation, other)] + inputs = tok(prompts, padding=True, return_tensors="pt") + prompt_len = inputs["input_ids"].shape[1] + out = model.generate(**inputs, max_new_tokens=8, do_sample=False) + replies = tok.batch_decode(out[:, prompt_len:], skip_special_tokens=True) + assert len(replies) == 2 + assert all(isinstance(reply, str) for reply in replies) + + # Stream the same request token by token: the concatenated stream must + # equal the one-shot decode, however tokens split mid-word or mid-byte. + single = tok(prompts[0], return_tensors="pt") + streamer = TextIteratorStreamer(tok, skip_prompt=True, skip_special_tokens=True) + worker = Thread( + target=model.generate, + kwargs={**single, "max_new_tokens": 8, "do_sample": False, "streamer": streamer}, + ) + worker.start() + streamed = "".join(streamer) + worker.join() + one_shot = model.generate(**single, max_new_tokens=8, do_sample=False) + assert streamed == tok.decode(one_shot[0, single["input_ids"].shape[1] :], skip_special_tokens=True) + + +def test_rag_chunk_embed_retrieve_flow(): + tok = AutoTokenizer.from_pretrained(TINY_BERT_MLM) + encoder = AutoModel.from_pretrained(TINY_BERT_MLM) + document = " ".join(f"Sentence number {i} talks at length about topic {i % 7}." for i in range(80)) + + # Chunk by token budget, keeping each chunk's char span so retrieval can + # point back into the original document. + encoding = tok(document, add_special_tokens=False, return_offsets_mapping=True) + offsets = encoding["offset_mapping"] + budget = 48 + spans = [ + (window[0][0], window[-1][1]) for window in (offsets[i : i + budget] for i in range(0, len(offsets), budget)) + ] + chunks = [document[start:end] for start, end in spans] + + # The spans tile the document: in order, non-overlapping, nothing lost + # but the whitespace between chunks. + assert len(chunks) > 3 + assert spans[0][0] == 0 + assert spans[-1][1] == len(document) + assert all(a[1] <= b[0] for a, b in itertools.pairwise(spans)) + + # Embed chunks and query the sentence-transformers way: mean-pool hidden + # states over the attention mask, so padding never dilutes the vector. + def embed(texts): + batch = tok(texts, padding=True, truncation=True, max_length=64, return_tensors="pt") + with torch.no_grad(): + hidden = encoder(**batch).last_hidden_state + mask = batch["attention_mask"].unsqueeze(-1) + vectors = (hidden * mask).sum(1) / mask.sum(1) + return torch.nn.functional.normalize(vectors, dim=1) + + chunk_vectors = embed(chunks) + query_vector = embed(["Which sentence talks about topic 3?"]) + assert torch.isfinite(chunk_vectors).all() + + # Whatever the (random) model ranks first, the service returns an exact + # substring of the source document. + best = int((chunk_vectors @ query_vector.T).argmax()) + start, end = spans[best] + assert document[start:end] == chunks[best] + + +def test_sft_finetuning_flow(tmp_path): + tok = chat_tokenizer() + model = AutoModelForCausalLM.from_pretrained(TINY_GPT2) + model.resize_token_embeddings(len(tok)) + + raw = datasets.Dataset.from_list( + [{"prompt": f"Question number {i}?", "response": f"Answer number {i}."} for i in range(8)] + ) + + def to_features(example): + # Prompt and reply are tokenized separately then concatenated — the + # TRL recipe — so no BPE merge can blur the boundary; the loss covers + # the reply only. + messages = [{"role": "user", "content": example["prompt"]}] + prompt = tok.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) + prompt_ids = tok(prompt, add_special_tokens=False)["input_ids"] + reply_ids = tok(example["response"] + "<|im_end|>", add_special_tokens=False)["input_ids"] + return { + "input_ids": prompt_ids + reply_ids, + "attention_mask": [1] * (len(prompt_ids) + len(reply_ids)), + "labels": [-100] * len(prompt_ids) + reply_ids, + } + + # num_proc=2 pickles the tokenizer into worker processes, like any real + # preprocessing job over a large dataset. + features = raw.map(to_features, num_proc=2, remove_columns=raw.column_names) + + first = features[0] + boundary = first["labels"].count(-100) + assert 0 < boundary < len(first["labels"]) + assert first["labels"][boundary:] == first["input_ids"][boundary:] + + trainer = Trainer( + model=model, + args=TrainingArguments(output_dir=str(tmp_path), max_steps=3, per_device_train_batch_size=4, report_to=[]), + train_dataset=features, + # Pads ragged input_ids with the pad token and labels with -100. + data_collator=DataCollatorForSeq2Seq(tok, label_pad_token_id=-100), + ) + result = trainer.train() + + assert math.isfinite(result.training_loss) diff --git a/bindings/python/tests/e2e/test_transformers_inference.py b/bindings/python/tests/e2e/test_transformers_inference.py new file mode 100644 index 000000000..ab6c51bcf --- /dev/null +++ b/bindings/python/tests/e2e/test_transformers_inference.py @@ -0,0 +1,78 @@ +"""Layer 3: transformers inference. + +The tokenizer feeds prompts into a model and turns its output ids back into +text. Models are tiny and random (see conftest), so the assertions check what +the tokenizer is responsible for — prompt round-trips, shapes, masks — never +generation quality. +""" + +from transformers import AutoModelForCausalLM, AutoModelForSeq2SeqLM, AutoTokenizer, pipeline + +from .conftest import TINY_BERT_MLM, TINY_BERT_NER, TINY_GPT2, TINY_T5 + + +def test_batched_generate_round_trips_the_prompt(): + # Decoder-only generation pads on the left so prompts sit flush against + # the generated positions. GPT-2 has no pad token; reusing EOS is the + # standard recipe. + tok = AutoTokenizer.from_pretrained(TINY_GPT2, padding_side="left") + tok.pad_token = tok.eos_token + model = AutoModelForCausalLM.from_pretrained(TINY_GPT2) + prompts = ["Hello world", "The quick brown fox jumps"] + + inputs = tok(prompts, padding=True, return_tensors="pt") + out = model.generate(**inputs, max_new_tokens=5, do_sample=False) + texts = tok.batch_decode(out, skip_special_tokens=True) + + assert out.shape == (2, inputs["input_ids"].shape[1] + 5) + for prompt, text in zip(prompts, texts): + assert text.startswith(prompt) + + +def test_text_generation_pipeline(): + # pipeline() bundles tokenizer + model + decoding into one call. + generate = pipeline("text-generation", model=TINY_GPT2) + + result = generate("Hello world", max_new_tokens=5, do_sample=False) + + assert result[0]["generated_text"].startswith("Hello world") + + +def test_encoder_decoder_generate(): + tok = AutoTokenizer.from_pretrained(TINY_T5) + model = AutoModelForSeq2SeqLM.from_pretrained(TINY_T5) + prompts = ["translate English to German: Hello world", "summarize: A long day"] + + batch = tok(prompts, padding=True, return_tensors="pt") + out = model.generate(**batch, max_new_tokens=5, do_sample=False) + texts = tok.batch_decode(out, skip_special_tokens=True) + + assert out.shape[0] == 2 + assert all(isinstance(t, str) for t in texts) + + +def test_token_classification_pipeline_spans(): + # Entity spans come straight from the tokenizer's offsets: whatever the + # (random) model labels, each span must slice the input text exactly. + text = "my name is sylvain and i work at huggingface in brooklyn" + ner = pipeline("token-classification", model=TINY_BERT_NER, aggregation_strategy="simple") + + entities = ner(text) + + assert entities + for entity in entities: + assert text[entity["start"] : entity["end"]] == entity["word"].replace("##", "") + + +def test_fill_mask_pipeline(): + # Fill-mask leans on tokenizer internals: locating the mask token's + # position and decoding single-token candidates back to strings. + fill = pipeline("fill-mask", model=TINY_BERT_MLM) + + candidates = fill("Paris is the [MASK] of France.") + + assert len(candidates) == 5 + for candidate in candidates: + assert candidate.keys() == {"score", "token", "token_str", "sequence"} + assert "[MASK]" not in candidate["sequence"] + assert candidates[0]["sequence"].startswith("paris is the") diff --git a/bindings/python/tests/e2e/test_transformers_tokenizer.py b/bindings/python/tests/e2e/test_transformers_tokenizer.py new file mode 100644 index 000000000..71541e352 --- /dev/null +++ b/bindings/python/tests/e2e/test_transformers_tokenizer.py @@ -0,0 +1,218 @@ +"""Layer 2: the transformers tokenizer API. + +`AutoTokenizer` wraps a `tokenizers.Tokenizer` (its "backend") and funnels +every call below into it. This is the API most users actually type, so it is +the contract the bindings have to serve. The heavier scenarios are lifted +from the maintained example scripts in transformers' examples/pytorch/ — the +canonical data-preparation recipes users copy. +""" + +import pytest +import tokenizers +from transformers import AutoTokenizer + +from .conftest import TINY_GPT2, TINY_T5 + + +@pytest.fixture(scope="module") +def bert(): + return AutoTokenizer.from_pretrained("bert-base-uncased") + + +def test_the_backend_is_tokenizers(bert): + assert bert.is_fast + assert isinstance(bert.backend_tokenizer, tokenizers.Tokenizer) + + +def test_call_returns_model_inputs(bert): + out = bert("Hello world") + + assert out["input_ids"] == [101, 7592, 2088, 102] + assert out["token_type_ids"] == [0, 0, 0, 0] + assert out["attention_mask"] == [1, 1, 1, 1] + + +def test_batch_with_dynamic_padding(bert): + out = bert(["One two", "one two three four five"], padding=True) + + short, long = out["input_ids"] + assert len(short) == len(long) + assert short[-1] == bert.pad_token_id + assert out["attention_mask"][0] == [1, 1, 1, 1, 0, 0, 0] + assert out["attention_mask"][1] == [1] * 7 + + +def test_truncation_to_max_length(bert): + out = bert("one two three four five six seven eight", truncation=True, max_length=6) + + assert len(out["input_ids"]) == 6 + # The special-token template survives truncation. + assert out["input_ids"][0] == bert.cls_token_id + assert out["input_ids"][-1] == bert.sep_token_id + + +def test_return_tensors_pt(bert): + import torch + + out = bert(["One two", "one two three four five"], padding=True, return_tensors="pt") + + assert out["input_ids"].shape == (2, 7) + assert out["input_ids"].dtype == torch.int64 + assert out["attention_mask"].shape == (2, 7) + + +def test_pairs_get_type_ids(bert): + out = bert("Where is Paris?", "Paris is in France.") + + assert set(out["token_type_ids"]) == {0, 1} + + +def test_offsets_and_word_ids_align_tokens_to_text(bert): + # NER-style alignment: word_ids groups sub-word tokens back into words, + # offsets locate them in the raw string. Special tokens map to nothing. + text = "Tokenization highlights substrings." + out = bert(text, return_offsets_mapping=True) + + word_ids = out.word_ids() + assert word_ids[0] is None + assert word_ids[-1] is None + assert word_ids[1:3] == [0, 0] # "token" + "##ization" + + for (start, end), word_id in zip(out["offset_mapping"], word_ids): + if word_id is None: + assert (start, end) == (0, 0) + else: + assert text[start:end] != "" + + +def test_batch_decode_round_trips(bert): + texts = ["hello world", "how are you?"] + out = bert(texts) + + assert bert.batch_decode(out["input_ids"], skip_special_tokens=True) == texts + + +def test_save_pretrained_round_trips(bert, tmp_path): + bert.save_pretrained(str(tmp_path)) + reloaded = AutoTokenizer.from_pretrained(str(tmp_path)) + + text = "Round-tripping through save_pretrained." + assert reloaded(text)["input_ids"] == bert(text)["input_ids"] + + +def test_chat_template_renders_then_tokenizes(): + tok = AutoTokenizer.from_pretrained(TINY_GPT2) + tok.chat_template = "{% for m in messages %}<|{{ m.role }}|>{{ m.content }}\n{% endfor %}" + messages = [{"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Hello!"}] + + text = tok.apply_chat_template(messages, tokenize=False) + assert text == "<|user|>Hi\n<|assistant|>Hello!\n" + # Byte-level BPE, so the rendered conversation round-trips exactly. + assert tok.decode(tok(text)["input_ids"]) == text + + +def test_qa_features_with_overflow_and_stride(bert): + # The question-answering recipe (run_qa.py, prepare_train_features): a + # long context becomes several overlapping features, each carrying + # offsets into the original string and sequence_ids to tell question + # from context — everything needed to label answer spans for training. + filler = "The quick brown fox jumps over the lazy dog. " * 12 + context = filler + "The Eiffel Tower is located in Paris." + filler + answer = "Paris" + start_char = context.index(answer) + end_char = start_char + len(answer) + + features = bert( + ["Where is the Eiffel Tower?"], + [context], + truncation="only_second", # never truncate the question + max_length=64, + stride=32, + return_overflowing_tokens=True, + return_offsets_mapping=True, + ) + + n = len(features["input_ids"]) + assert n > 1 + assert features["overflow_to_sample_mapping"] == [0] * n + + recovered = [] + for i in range(n): + seq_ids = features.sequence_ids(i) + offsets = features["offset_mapping"][i] + token_start = seq_ids.index(1) + token_end = len(seq_ids) - 1 + while seq_ids[token_end] != 1: + token_end -= 1 + if not (offsets[token_start][0] <= start_char and offsets[token_end][1] >= end_char): + continue # the answer is outside this feature's window + # Walk in to the answer's exact token span, as run_qa does to label + # training data, then read it back out of the source string. + while token_start < len(offsets) and offsets[token_start][0] <= start_char: + token_start += 1 + while offsets[token_end][1] >= end_char: + token_end -= 1 + recovered.append(context[offsets[token_start - 1][0] : offsets[token_end + 1][1]]) + + # The stride windows overlap, so several features see the answer and all + # recover it exactly; the far-away windows don't contain it at all. + assert recovered + assert set(recovered) == {answer} + assert len(recovered) < n + + +def test_ner_labels_align_through_word_ids(bert): + # The token-classification recipe (run_ner.py, tokenize_and_align_labels): + # one label per word in, one label per token out — the word's label on its + # first sub-token, -100 on continuations and special tokens. + words = ["My", "name", "is", "Sylvain", "and", "I", "work", "at", "HuggingFace"] + word_labels = [0, 0, 0, 1, 0, 0, 0, 0, 2] + + encoding = bert([words], is_split_into_words=True) + + word_ids = encoding.word_ids(batch_index=0) + labels = [] + previous = None + for word_idx in word_ids: + if word_idx is None: + labels.append(-100) + elif word_idx != previous: + labels.append(word_labels[word_idx]) + else: + labels.append(-100) + previous = word_idx + + assert len(labels) == len(encoding["input_ids"][0]) + assert labels[0] == labels[-1] == -100 + # Each word contributes its label exactly once, in order. + assert [label for label in labels if label != -100] == word_labels + # The alignment is only interesting if some word split into sub-tokens + # ("Sylvain" and "HuggingFace" do). + assert len(word_ids) > len(words) + 2 + + +def test_seq2seq_targets_via_text_target(): + # The translation recipe (run_translation.py, preprocess_function): + # text_target= tokenizes with the target-side rules, and the label ids + # end with EOS so generation learns where to stop. + tok = AutoTokenizer.from_pretrained(TINY_T5) + + inputs = tok(["translate English to German: Hello"], max_length=32, truncation=True) + labels = tok(text_target=["Hallo"], max_length=32, truncation=True) + inputs["labels"] = labels["input_ids"] + + assert inputs["labels"][0][-1] == tok.eos_token_id + assert tok.decode(inputs["labels"][0], skip_special_tokens=True) == "Hallo" + + +def test_train_new_from_iterator(bert): + # Retrain the same pipeline on a new corpus — transformers drives the + # tokenizers trainers under the hood. + corpus = ["the cat sat on the mat"] * 20 + + new_tok = bert.train_new_from_iterator(corpus, vocab_size=60) + + assert new_tok.is_fast + assert len(new_tok) <= 60 + ids = new_tok("the cat")["input_ids"] + assert new_tok.batch_decode([ids], skip_special_tokens=True) == ["the cat"] diff --git a/bindings/python/tests/e2e/test_transformers_training.py b/bindings/python/tests/e2e/test_transformers_training.py new file mode 100644 index 000000000..f9917717b --- /dev/null +++ b/bindings/python/tests/e2e/test_transformers_training.py @@ -0,0 +1,123 @@ +"""Layer 4: transformers training. + +The tokenizer's job in a training loop: turn a labeled corpus into ragged id +lists up front, then let a data collator pad each batch on the fly. A few +optimizer steps on a tiny random model prove the loop consumes them. +""" + +import math + +import torch +from transformers import ( + AutoModelForSequenceClassification, + AutoTokenizer, + DataCollatorForLanguageModeling, + DataCollatorForSeq2Seq, + DataCollatorWithPadding, + Trainer, + TrainingArguments, +) + +from .conftest import TINY_BERT_CLS, TINY_BERT_MLM, TINY_GPT2, TINY_T5 + + +def test_data_collator_pads_a_ragged_batch(): + # DataCollatorWithPadding is tokenizer.pad() in disguise: ragged encodings + # in, rectangular tensors out. + tok = AutoTokenizer.from_pretrained(TINY_BERT_CLS) + features = [tok(text) for text in ("One two", "one two three four five")] + + batch = DataCollatorWithPadding(tok)(features) + + assert batch["input_ids"].shape == batch["attention_mask"].shape + assert batch["input_ids"][0, -1] == tok.pad_token_id + assert batch["attention_mask"][0, -1] == 0 + assert batch["attention_mask"][1].tolist() == [1] * batch["input_ids"].shape[1] + + +def test_causal_lm_collator_masks_padding_in_labels(): + # For causal LM the labels are the input ids, except padding must not + # contribute to the loss: it becomes -100. + tok = AutoTokenizer.from_pretrained(TINY_GPT2) + tok.pad_token = tok.eos_token + features = [tok(text) for text in ("Tiny", "a longer line of text")] + + batch = DataCollatorForLanguageModeling(tok, mlm=False)(features) + + padded = batch["attention_mask"] == 0 + assert (batch["labels"][padded] == -100).all() + assert (batch["labels"][~padded] == batch["input_ids"][~padded]).all() + + +def test_mlm_collator_masks_only_real_tokens(): + # The masked-LM recipe (run_mlm.py): tokenize with the special-tokens + # mask so the collator knows which positions it must never mask, then + # let it pick tokens to corrupt and put their original ids in the labels. + tok = AutoTokenizer.from_pretrained(TINY_BERT_MLM) + texts = ["a reasonably long sentence used for masked language modeling"] * 4 + features = [tok(t, return_special_tokens_mask=True) for t in texts] + + torch.manual_seed(0) + batch = DataCollatorForLanguageModeling(tok, mlm=True, mlm_probability=0.5)(features) + + selected = batch["labels"] != -100 + original = torch.tensor([f["input_ids"] for f in features]) + assert selected.any() + assert (batch["input_ids"] == tok.mask_token_id).any() + assert (batch["labels"][selected] == original[selected]).all() + # [CLS] and [SEP] are never selected (all rows are the same length here, + # so the last column is [SEP], not padding). + assert (batch["labels"][:, 0] == -100).all() + assert (batch["labels"][:, -1] == -100).all() + + +def test_seq2seq_collator_pads_labels_with_minus_100(): + # The translation recipe again (run_translation.py): inputs pad with the + # pad token, labels with -100 so padding never contributes to the loss. + tok = AutoTokenizer.from_pretrained(TINY_T5) + inputs = tok(["short", "a much longer input sentence right here"]) + targets = tok(text_target=["ok", "a longer target"]) + features = [ + { + "input_ids": inputs["input_ids"][i], + "attention_mask": inputs["attention_mask"][i], + "labels": targets["input_ids"][i], + } + for i in range(2) + ] + + batch = DataCollatorForSeq2Seq(tok, label_pad_token_id=-100)(features) + + assert batch["input_ids"][0, -1] == tok.pad_token_id + assert batch["labels"][0, -1] == -100 + assert batch["labels"][1, -1] == tok.eos_token_id + + +def test_trainer_runs_a_few_steps(tmp_path): + texts = ["a delightful film", "an utter disappointment", "warm and funny", "dull beyond belief"] * 4 + labels = [1, 0, 1, 0] * 4 + + tok = AutoTokenizer.from_pretrained(TINY_BERT_CLS) + model = AutoModelForSequenceClassification.from_pretrained(TINY_BERT_CLS) + encodings = tok(texts, truncation=True, max_length=32) + + class SentimentDataset(torch.utils.data.Dataset): + def __len__(self): + return len(labels) + + def __getitem__(self, i): + return { + "input_ids": encodings["input_ids"][i], + "attention_mask": encodings["attention_mask"][i], + "labels": labels[i], + } + + trainer = Trainer( + model=model, + args=TrainingArguments(output_dir=str(tmp_path), max_steps=3, per_device_train_batch_size=4, report_to=[]), + train_dataset=SentimentDataset(), + data_collator=DataCollatorWithPadding(tok), + ) + result = trainer.train() + + assert math.isfinite(result.training_loss) diff --git a/bindings/python/tests/golden/__init__.py b/bindings/python/tests/golden/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/bindings/python/tests/golden/conftest.py b/bindings/python/tests/golden/conftest.py new file mode 100644 index 000000000..458c6619b --- /dev/null +++ b/bindings/python/tests/golden/conftest.py @@ -0,0 +1,21 @@ +"""Golden conformance layer: byte-exact agreement with the released wheel. + +The files under goldens/ record what the released tokenizers wheel produces +on the data/ fixtures and a set of edge-case strings (see generate.py). +test_golden.py replays those inputs on the current build and diffs every id, +token, offset, mask and decoded string against the record. + +Run with `make golden`; regenerate the goldens with `make golden-regen`. + +The env gate below makes the layer opt-in while the 1.0 rewrite is +incomplete, so a plain `make test` or the required CI jobs stay green while +offsets/decode/pairs are still missing. Delete it once `make golden` runs +green — from then on conformance should be enforced by default, everywhere. +""" + +import os + +import pytest + +if not os.environ.get("TOKENIZERS_GOLDEN"): + pytest.skip("golden conformance layer is opt-in for now — run `make golden`", allow_module_level=True) diff --git a/bindings/python/tests/golden/generate.py b/bindings/python/tests/golden/generate.py new file mode 100644 index 000000000..4db256f64 --- /dev/null +++ b/bindings/python/tests/golden/generate.py @@ -0,0 +1,172 @@ +"""Regenerate the golden files: the exact-output contract of the bindings. + +A golden file records everything encode and decode produce for one tokenizer +on a fixed set of inputs — ids, tokens, offsets, masks, word ids, decoded +strings for short samples, plus ids digests over the data/ fixture corpora +for breadth. test_golden.py diffs the current build against these files, so +any behavioral deviation from the released wheel surfaces, including ones +nobody thought to assert. + +Goldens are never hand-edited. They come from the released tokenizers wheel — +the reference the 1.0 rewrite must match: `make golden-regen` installs it +into .release/ and runs this script with PYTHONPATH pointing there (the +release shares our package name; same trick as benches/bench_vs_release.py). + +Inputs: every model in tk-encode/examples/bench_models.json, the corpora +under data/fixtures (a short excerpt captured verbatim, the rest as a capped +digest), and the edge-case strings below. Regenerating needs the data +fetched: `make -C ../../tokenizers bench-models fixtures`. + +Output format: one JSON-lines file per model under goldens/ — meta line, +then one line per sample/pair/digest, so a behavior change diffs line by line. +""" + +import hashlib +import json +import sys +from pathlib import Path + +import tokenizers +from tokenizers import Tokenizer + +REPO = Path(__file__).resolve().parents[4] +DATA = REPO / "tokenizers" / "data" +BENCH_MODELS = REPO / "tokenizers" / "tk-encode" / "examples" / "bench_models.json" +GOLDENS = Path(__file__).parent / "goldens" + +EXCERPT_CHARS = 160 +# Fixture files are ~5 MB each; the first 200k chars already exercise the +# whole distribution, and keep regeneration under a minute. +DIGEST_CAP_CHARS = 200_000 + +EDGE_CASES = { + "empty": "", + "spaces": " ", + "hello": "Hello world", + "punctuation": "Hello, world!! How's it going? (fine; thanks...)", + "whitespace-mix": "line one\nline two\r\n\tindented\n trailing ", + "accents": "café naïve résumé — déjà vu", + # The same letters in decomposed form (base letter + combining accent): + # ids must come out identical to the composed spelling above once the + # tokenizer normalizes, or reveal that it does not. + "accents-decomposed": "cafe\u0301 nai\u0308ve re\u0301sume\u0301", + "emoji": "🤗 emoji, families 👨‍👩‍👧‍👦, flags 🇫🇷 and skin tones 👍🏽", + "cjk": "漢字とひらがなとカタカナが混ざった文章です。", + "korean": "한국어 텍스트 조각", + "rtl": "مرحبا بالعالم — שלום עולם", + "numbers": "1234567890, 3.14159, 1,000,000th", + "code": "def f(x):\n return x**2 # squared\nprint(f'{f(3)=}')", + "long-word": "Donaudampfschifffahrtsgesellschaftskapitänsmützenabzeichen", + "url-email": "https://example.com/a/b?q=1&r=2#frag user.name+tag@example.co.uk", + # no-break, thin and ideographic spaces + "unicode-spaces": "a\u00a0b\u2009c\u3000d", + "math": "∑ᵢ xᵢ² ≤ ∫₀^∞ e⁻ˣ dx ≈ 1", +} + +PAIRS = [ + ("What is the capital of France?", "Paris is the capital of France."), + ("Question in English?", "Ответ на русском языке, с цифрами 123."), +] + + +def text_digest(text: str) -> str: + return hashlib.sha256(text.encode()).hexdigest() + + +def ids_digest(ids) -> str: + return hashlib.sha256(",".join(map(str, ids)).encode()).hexdigest() + + +def sample_record(tok: Tokenizer, source: str, text: str) -> dict: + enc = tok.encode(text) + return { + "kind": "sample", + "source": source, + "text": text, + "ids": enc.ids, + "ids_no_specials": tok.encode(text, add_special_tokens=False).ids, + "tokens": enc.tokens, + "offsets": [list(span) for span in enc.offsets], + "type_ids": enc.type_ids, + "special_tokens_mask": enc.special_tokens_mask, + "word_ids": enc.word_ids, + "decoded": tok.decode(enc.ids, skip_special_tokens=True), + "decoded_with_specials": tok.decode(enc.ids, skip_special_tokens=False), + } + + +def pair_record(tok: Tokenizer, text: str, pair: str) -> dict: + enc = tok.encode(text, pair) + return { + "kind": "pair", + "text": text, + "pair": pair, + "ids": enc.ids, + "tokens": enc.tokens, + "type_ids": enc.type_ids, + "sequence_ids": enc.sequence_ids, + "special_tokens_mask": enc.special_tokens_mask, + "offsets": [list(span) for span in enc.offsets], + } + + +def digest_record(tok: Tokenizer, relative: str, text: str) -> dict: + ids = tok.encode(text).ids + return { + "kind": "digest", + "file": relative, + "cap_chars": DIGEST_CAP_CHARS, + "text_sha256": text_digest(text), + "n_tokens": len(ids), + "ids_sha256": ids_digest(ids), + } + + +def fixture_files() -> list[Path]: + files = sorted((DATA / "fixtures" / "lang").glob("*.txt")) + sorted( + (DATA / "fixtures" / "modalities").glob("*.txt") + ) + if not files: + sys.exit(f"no fixture corpora under {DATA / 'fixtures'} — run `make -C {REPO / 'tokenizers'} fixtures`") + return files + + +def main(): + if not tokenizers.__version__.startswith("0."): + sys.exit( + f"goldens must come from the released 0.x wheel, not {tokenizers.__version__} — run `make golden-regen`" + ) + + fixtures = fixture_files() + GOLDENS.mkdir(exist_ok=True) + for model in json.loads(BENCH_MODELS.read_text()): + tokenizer_file = DATA / model["file"] + if not tokenizer_file.is_file(): + sys.exit(f"{tokenizer_file} missing — run `make -C {REPO / 'tokenizers'} bench-models`") + tok = Tokenizer.from_file(str(tokenizer_file)) + + records = [ + { + "kind": "meta", + "model": model["name"], + "tokenizer_file": model["file"], + "tokenizers_version": tokenizers.__version__, + } + ] + records += [sample_record(tok, f"edge:{name}", text) for name, text in EDGE_CASES.items()] + records += [ + sample_record(tok, f"{f.relative_to(DATA).as_posix()}[:{EXCERPT_CHARS}]", f.read_text()[:EXCERPT_CHARS]) + for f in fixtures + ] + records += [pair_record(tok, text, pair) for text, pair in PAIRS] + records += [ + digest_record(tok, f.relative_to(DATA).as_posix(), f.read_text()[:DIGEST_CAP_CHARS]) for f in fixtures + ] + + path = GOLDENS / f"{model['name']}.jsonl" + path.write_text("\n".join(json.dumps(r, ensure_ascii=False, separators=(",", ":")) for r in records) + "\n") + print(f"{path.name}: {len(records) - 1} records from tokenizers {tokenizers.__version__}") + + +if __name__ == "__main__": + main() diff --git a/bindings/python/tests/golden/goldens/bert-base-uncased.jsonl b/bindings/python/tests/golden/goldens/bert-base-uncased.jsonl new file mode 100644 index 000000000..f3b136835 --- /dev/null +++ b/bindings/python/tests/golden/goldens/bert-base-uncased.jsonl @@ -0,0 +1,64 @@ +{"kind":"meta","model":"bert-base-uncased","tokenizer_file":"bert-base-uncased.json","tokenizers_version":"0.23.1"} +{"kind":"sample","source":"edge:empty","text":"","ids":[101,102],"ids_no_specials":[],"tokens":["[CLS]","[SEP]"],"offsets":[[0,0],[0,0]],"type_ids":[0,0],"special_tokens_mask":[1,1],"word_ids":[null,null],"decoded":"","decoded_with_specials":"[CLS] [SEP]"} +{"kind":"sample","source":"edge:spaces","text":" ","ids":[101,102],"ids_no_specials":[],"tokens":["[CLS]","[SEP]"],"offsets":[[0,0],[0,0]],"type_ids":[0,0],"special_tokens_mask":[1,1],"word_ids":[null,null],"decoded":"","decoded_with_specials":"[CLS] [SEP]"} +{"kind":"sample","source":"edge:hello","text":"Hello world","ids":[101,7592,2088,102],"ids_no_specials":[7592,2088],"tokens":["[CLS]","hello","world","[SEP]"],"offsets":[[0,0],[0,5],[6,11],[0,0]],"type_ids":[0,0,0,0],"special_tokens_mask":[1,0,0,1],"word_ids":[null,0,1,null],"decoded":"hello world","decoded_with_specials":"[CLS] hello world [SEP]"} +{"kind":"sample","source":"edge:punctuation","text":"Hello, world!! How's it going? (fine; thanks...)","ids":[101,7592,1010,2088,999,999,2129,1005,1055,2009,2183,1029,1006,2986,1025,4283,1012,1012,1012,1007,102],"ids_no_specials":[7592,1010,2088,999,999,2129,1005,1055,2009,2183,1029,1006,2986,1025,4283,1012,1012,1012,1007],"tokens":["[CLS]","hello",",","world","!","!","how","'","s","it","going","?","(","fine",";","thanks",".",".",".",")","[SEP]"],"offsets":[[0,0],[0,5],[5,6],[7,12],[12,13],[13,14],[15,18],[18,19],[19,20],[21,23],[24,29],[29,30],[31,32],[32,36],[36,37],[38,44],[44,45],[45,46],[46,47],[47,48],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[null,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,null],"decoded":"hello, world!! how ' s it going? ( fine ; thanks... )","decoded_with_specials":"[CLS] hello, world!! how ' s it going? ( fine ; thanks... ) [SEP]"} +{"kind":"sample","source":"edge:whitespace-mix","text":"line one\nline two\r\n\tindented\n trailing ","ids":[101,2240,2028,2240,2048,27427,14088,12542,102],"ids_no_specials":[2240,2028,2240,2048,27427,14088,12542],"tokens":["[CLS]","line","one","line","two","ind","##ented","trailing","[SEP]"],"offsets":[[0,0],[0,4],[5,8],[9,13],[14,17],[20,23],[23,28],[31,39],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,1],"word_ids":[null,0,1,2,3,4,4,5,null],"decoded":"line one line two indented trailing","decoded_with_specials":"[CLS] line one line two indented trailing [SEP]"} +{"kind":"sample","source":"edge:accents","text":"café naïve résumé — déjà vu","ids":[101,7668,15743,13746,1517,2139,3900,24728,102],"ids_no_specials":[7668,15743,13746,1517,2139,3900,24728],"tokens":["[CLS]","cafe","naive","resume","—","de","##ja","vu","[SEP]"],"offsets":[[0,0],[0,4],[5,10],[11,17],[18,19],[20,22],[22,24],[25,27],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,1],"word_ids":[null,0,1,2,3,4,4,5,null],"decoded":"cafe naive resume — deja vu","decoded_with_specials":"[CLS] cafe naive resume — deja vu [SEP]"} +{"kind":"sample","source":"edge:accents-decomposed","text":"café naïve résumé","ids":[101,7668,15743,13746,102],"ids_no_specials":[7668,15743,13746],"tokens":["[CLS]","cafe","naive","resume","[SEP]"],"offsets":[[0,0],[0,4],[6,12],[13,20],[0,0]],"type_ids":[0,0,0,0,0],"special_tokens_mask":[1,0,0,0,1],"word_ids":[null,0,1,2,null],"decoded":"cafe naive resume","decoded_with_specials":"[CLS] cafe naive resume [SEP]"} +{"kind":"sample","source":"edge:emoji","text":"🤗 emoji, families 👨‍👩‍👧‍👦, flags 🇫🇷 and skin tones 👍🏽","ids":[101,100,7861,29147,2072,1010,2945,100,1010,9245,100,1998,3096,12623,100,102],"ids_no_specials":[100,7861,29147,2072,1010,2945,100,1010,9245,100,1998,3096,12623,100],"tokens":["[CLS]","[UNK]","em","##oj","##i",",","families","[UNK]",",","flags","[UNK]","and","skin","tones","[UNK]","[SEP]"],"offsets":[[0,0],[0,1],[2,4],[4,6],[6,7],[7,8],[9,17],[18,25],[25,26],[27,32],[33,35],[36,39],[40,44],[45,50],[51,53],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[null,0,1,1,1,2,3,4,5,6,7,8,9,10,11,null],"decoded":"emoji, families, flags and skin tones","decoded_with_specials":"[CLS] [UNK] emoji, families [UNK], flags [UNK] and skin tones [UNK] [SEP]"} +{"kind":"sample","source":"edge:cjk","text":"漢字とひらがなとカタカナが混ざった文章です。","ids":[101,1904,100,1666,30199,30211,30177,30193,30192,30226,30235,30226,30241,30177,100,1656,30189,30187,1861,1932,1665,30184,1636,102],"ids_no_specials":[1904,100,1666,30199,30211,30177,30193,30192,30226,30235,30226,30241,30177,100,1656,30189,30187,1861,1932,1665,30184,1636],"tokens":["[CLS]","漢","[UNK]","と","##ひ","##ら","##か","##な","##と","##カ","##タ","##カ","##ナ","##か","[UNK]","さ","##っ","##た","文","章","て","##す","。","[SEP]"],"offsets":[[0,0],[0,1],[1,2],[2,3],[3,4],[4,5],[5,6],[6,7],[7,8],[8,9],[9,10],[10,11],[11,12],[12,13],[13,14],[14,15],[15,16],[16,17],[17,18],[18,19],[19,20],[20,21],[21,22],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[null,0,1,2,2,2,2,2,2,2,2,2,2,2,3,4,4,4,5,6,7,7,8,null],"decoded":"漢 とひらかなとカタカナか さった 文 章 てす 。","decoded_with_specials":"[CLS] 漢 [UNK] とひらかなとカタカナか [UNK] さった 文 章 てす 。 [SEP]"} +{"kind":"sample","source":"edge:korean","text":"한국어 텍스트 조각","ids":[101,1469,30006,30021,29991,30014,30020,29999,30008,1467,30009,30020,29997,30017,30003,30017,1464,30011,29991,30006,30020,102],"ids_no_specials":[1469,30006,30021,29991,30014,30020,29999,30008,1467,30009,30020,29997,30017,30003,30017,1464,30011,29991,30006,30020],"tokens":["[CLS]","ᄒ","##ᅡ","##ᆫ","##ᄀ","##ᅮ","##ᆨ","##ᄋ","##ᅥ","ᄐ","##ᅦ","##ᆨ","##ᄉ","##ᅳ","##ᄐ","##ᅳ","ᄌ","##ᅩ","##ᄀ","##ᅡ","##ᆨ","[SEP]"],"offsets":[[0,0],[0,1],[0,1],[0,1],[1,2],[1,2],[1,2],[2,3],[2,3],[4,5],[4,5],[4,5],[5,6],[5,6],[6,7],[6,7],[8,9],[8,9],[9,10],[9,10],[9,10],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[null,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,2,2,2,2,2,null],"decoded":"한국어 텍스트 조각","decoded_with_specials":"[CLS] 한국어 텍스트 조각 [SEP]"} +{"kind":"sample","source":"edge:rtl","text":"مرحبا بالعالم — שלום עולם","ids":[101,1295,17149,29820,29816,25573,1271,25573,23673,29830,25573,23673,22192,1517,1266,29799,29792,29800,1259,29792,29799,29800,102],"ids_no_specials":[1295,17149,29820,29816,25573,1271,25573,23673,29830,25573,23673,22192,1517,1266,29799,29792,29800,1259,29792,29799,29800],"tokens":["[CLS]","م","##ر","##ح","##ب","##ا","ب","##ا","##ل","##ع","##ا","##ل","##م","—","ש","##ל","##ו","##ם","ע","##ו","##ל","##ם","[SEP]"],"offsets":[[0,0],[0,1],[1,2],[2,3],[3,4],[4,5],[6,7],[7,8],[8,9],[9,10],[10,11],[11,12],[12,13],[14,15],[16,17],[17,18],[18,19],[19,20],[21,22],[22,23],[23,24],[24,25],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[null,0,0,0,0,0,1,1,1,1,1,1,1,2,3,3,3,3,4,4,4,4,null],"decoded":"مرحبا بالعالم — שלום עולם","decoded_with_specials":"[CLS] مرحبا بالعالم — שלום עולם [SEP]"} +{"kind":"sample","source":"edge:numbers","text":"1234567890, 3.14159, 1,000,000th","ids":[101,13138,19961,2575,2581,2620,21057,1010,1017,1012,15471,28154,1010,1015,1010,2199,1010,2199,2705,102],"ids_no_specials":[13138,19961,2575,2581,2620,21057,1010,1017,1012,15471,28154,1010,1015,1010,2199,1010,2199,2705],"tokens":["[CLS]","123","##45","##6","##7","##8","##90",",","3",".","141","##59",",","1",",","000",",","000","##th","[SEP]"],"offsets":[[0,0],[0,3],[3,5],[5,6],[6,7],[7,8],[8,10],[10,11],[12,13],[13,14],[14,17],[17,19],[19,20],[21,22],[22,23],[23,26],[26,27],[27,30],[30,32],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[null,0,0,0,0,0,0,1,2,3,4,4,5,6,7,8,9,10,10,null],"decoded":"1234567890, 3. 14159, 1, 000, 000th","decoded_with_specials":"[CLS] 1234567890, 3. 14159, 1, 000, 000th [SEP]"} +{"kind":"sample","source":"edge:code","text":"def f(x):\n return x**2 # squared\nprint(f'{f(3)=}')","ids":[101,13366,1042,1006,1060,1007,1024,2709,1060,1008,1008,1016,1001,19942,6140,1006,1042,1005,1063,1042,1006,1017,1007,1027,1065,1005,1007,102],"ids_no_specials":[13366,1042,1006,1060,1007,1024,2709,1060,1008,1008,1016,1001,19942,6140,1006,1042,1005,1063,1042,1006,1017,1007,1027,1065,1005,1007],"tokens":["[CLS]","def","f","(","x",")",":","return","x","*","*","2","#","squared","print","(","f","'","{","f","(","3",")","=","}","'",")","[SEP]"],"offsets":[[0,0],[0,3],[4,5],[5,6],[6,7],[7,8],[8,9],[14,20],[21,22],[22,23],[23,24],[24,25],[27,28],[29,36],[37,42],[42,43],[43,44],[44,45],[45,46],[46,47],[47,48],[48,49],[49,50],[50,51],[51,52],[52,53],[53,54],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[null,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,null],"decoded":"def f ( x ) : return x * * 2 # squared print ( f ' { f ( 3 ) = } ' )","decoded_with_specials":"[CLS] def f ( x ) : return x * * 2 # squared print ( f ' { f ( 3 ) = } ' ) [SEP]"} +{"kind":"sample","source":"edge:long-word","text":"Donaudampfschifffahrtsgesellschaftskapitänsmützenabzeichen","ids":[101,24260,14066,8737,10343,5428,4246,7011,8093,3215,8449,5349,26527,8337,23270,6962,28120,10431,7875,4371,17322,2078,102],"ids_no_specials":[24260,14066,8737,10343,5428,4246,7011,8093,3215,8449,5349,26527,8337,23270,6962,28120,10431,7875,4371,17322,2078],"tokens":["[CLS]","dona","##uda","##mp","##fs","##chi","##ff","##fa","##hr","##ts","##ges","##ell","##schaft","##ska","##pit","##ans","##mut","##zen","##ab","##ze","##iche","##n","[SEP]"],"offsets":[[0,0],[0,4],[4,7],[7,9],[9,11],[11,14],[14,16],[16,18],[18,20],[20,22],[22,25],[25,28],[28,34],[34,37],[37,40],[40,43],[43,46],[46,49],[49,51],[51,53],[53,57],[57,58],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,null],"decoded":"donaudampfschifffahrtsgesellschaftskapitansmutzenabzeichen","decoded_with_specials":"[CLS] donaudampfschifffahrtsgesellschaftskapitansmutzenabzeichen [SEP]"} +{"kind":"sample","source":"edge:url-email","text":"https://example.com/a/b?q=1&r=2#frag user.name+tag@example.co.uk","ids":[101,16770,1024,1013,1013,2742,1012,4012,1013,1037,1013,1038,1029,1053,1027,1015,1004,1054,1027,1016,1001,25312,2290,5310,1012,2171,1009,6415,1030,2742,1012,2522,1012,2866,102],"ids_no_specials":[16770,1024,1013,1013,2742,1012,4012,1013,1037,1013,1038,1029,1053,1027,1015,1004,1054,1027,1016,1001,25312,2290,5310,1012,2171,1009,6415,1030,2742,1012,2522,1012,2866],"tokens":["[CLS]","https",":","/","/","example",".","com","/","a","/","b","?","q","=","1","&","r","=","2","#","fra","##g","user",".","name","+","tag","@","example",".","co",".","uk","[SEP]"],"offsets":[[0,0],[0,5],[5,6],[6,7],[7,8],[8,15],[15,16],[16,19],[19,20],[20,21],[21,22],[22,23],[23,24],[24,25],[25,26],[26,27],[27,28],[28,29],[29,30],[30,31],[31,32],[32,35],[35,36],[37,41],[41,42],[42,46],[46,47],[47,50],[50,51],[51,58],[58,59],[59,61],[61,62],[62,64],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[null,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,20,21,22,23,24,25,26,27,28,29,30,31,null],"decoded":"https : / / example. com / a / b? q = 1 & r = 2 # frag user. name + tag @ example. co. uk","decoded_with_specials":"[CLS] https : / / example. com / a / b? q = 1 & r = 2 # frag user. name + tag @ example. co. uk [SEP]"} +{"kind":"sample","source":"edge:unicode-spaces","text":"a b c　d","ids":[101,1037,1038,1039,1040,102],"ids_no_specials":[1037,1038,1039,1040],"tokens":["[CLS]","a","b","c","d","[SEP]"],"offsets":[[0,0],[0,1],[2,3],[4,5],[6,7],[0,0]],"type_ids":[0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,1],"word_ids":[null,0,1,2,3,null],"decoded":"a b c d","decoded_with_specials":"[CLS] a b c d [SEP]"} +{"kind":"sample","source":"edge:math","text":"∑ᵢ xᵢ² ≤ ∫₀^∞ e⁻ˣ dx ≈ 1","ids":[101,100,1060,19109,10701,1608,100,1034,1601,1041,30079,29718,1040,2595,1606,1015,102],"ids_no_specials":[100,1060,19109,10701,1608,100,1034,1601,1041,30079,29718,1040,2595,1606,1015],"tokens":["[CLS]","[UNK]","x","##ᵢ","##²","≤","[UNK]","^","∞","e","##⁻","##ˣ","d","##x","≈","1","[SEP]"],"offsets":[[0,0],[0,2],[3,4],[4,5],[5,6],[7,8],[9,11],[11,12],[12,13],[14,15],[15,16],[16,17],[18,19],[19,20],[21,22],[23,24],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[null,0,1,1,1,2,3,4,5,6,6,6,7,7,8,9,null],"decoded":"xᵢ² ≤ ^ ∞ e⁻ˣ dx ≈ 1","decoded_with_specials":"[CLS] [UNK] xᵢ² ≤ [UNK] ^ ∞ e⁻ˣ dx ≈ 1 [SEP]"} +{"kind":"sample","source":"fixtures/lang/amh_Ethi.txt[:160]","text":"- ፍርድ ቤቱ በአቶ ዮናታን ተስፋዬ ላይ የ6 አመት ከ6 ወር ጽኑ የእስር ቅጣት አስተላለፈ\n- አንድነት ዶክተር ቴድሮስ ለዓለም ጤና ድርጅት ዋና ዳይሬክተርነት በመመረጣቸው ደስታውን ገለጸ\n- ህንድ መጠነ ሰፊ የድንጋይ ከሰል ሃይል ጣብያዎች የመገንባት እ","ids":[101,1011,100,100,100,100,100,100,100,100,100,100,100,100,100,100,1011,100,100,100,100,100,100,100,100,100,100,100,1011,100,100,100,100,100,100,100,100,100,102],"ids_no_specials":[1011,100,100,100,100,100,100,100,100,100,100,100,100,100,100,1011,100,100,100,100,100,100,100,100,100,100,100,1011,100,100,100,100,100,100,100,100,100],"tokens":["[CLS]","-","[UNK]","[UNK]","[UNK]","[UNK]","[UNK]","[UNK]","[UNK]","[UNK]","[UNK]","[UNK]","[UNK]","[UNK]","[UNK]","[UNK]","-","[UNK]","[UNK]","[UNK]","[UNK]","[UNK]","[UNK]","[UNK]","[UNK]","[UNK]","[UNK]","[UNK]","-","[UNK]","[UNK]","[UNK]","[UNK]","[UNK]","[UNK]","[UNK]","[UNK]","[UNK]","[SEP]"],"offsets":[[0,0],[0,1],[2,5],[6,8],[9,12],[13,17],[18,22],[23,25],[26,28],[29,32],[33,35],[36,38],[39,41],[42,46],[47,50],[51,57],[58,59],[60,65],[66,70],[71,75],[76,80],[81,83],[84,88],[89,91],[92,100],[101,108],[109,114],[115,118],[119,120],[121,124],[125,128],[129,131],[132,137],[138,141],[142,145],[146,151],[152,158],[159,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[null,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,null],"decoded":"- - -","decoded_with_specials":"[CLS] - [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] - [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] - [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [SEP]"} +{"kind":"sample","source":"fixtures/lang/arb_Arab.txt[:160]","text":"[بالصور] ترقوميا : الموت يطفىء باقة ورد قيد التفتح !!\nالخليل – دوت كوم – من غسان عبد الحميد - لم يخطر في بال أهالي \"ترقوميا\" التي انخلع قلبها وهي تودع التراب جث","ids":[101,1031,1271,25573,23673,29826,29836,17149,1033,1273,17149,29834,29836,22192,14498,25573,1024,1270,23673,22192,29836,29817,1300,29828,29833,29837,29815,1271,25573,29834,19433,1298,17149,15394,1292,14498,15394,1270,23673,29817,29833,29817,29820,999,999,1270,23673,29821,23673,14498,23673,1516,1278,29836,29817,1293,29836,22192,1516,1295,15915,1289,29824,18511,1288,29816,15394,1270,23673,29820,22192,14498,15394,1011,1294,22192,1300,29821,29828,17149,1291,14498,1271,25573,23673,1270,14157,25573,23673,14498,1000,1273,17149,29834,29836,22192,14498,25573,1000,1270,23673,29817,14498,1270,15915,29821,23673,29830,1292,23673,29816,14157,25573,1298,14157,14498,1273,29836,15394,29830,1270,23673,29817,17149,25573,29816,1275,29818,102],"ids_no_specials":[1031,1271,25573,23673,29826,29836,17149,1033,1273,17149,29834,29836,22192,14498,25573,1024,1270,23673,22192,29836,29817,1300,29828,29833,29837,29815,1271,25573,29834,19433,1298,17149,15394,1292,14498,15394,1270,23673,29817,29833,29817,29820,999,999,1270,23673,29821,23673,14498,23673,1516,1278,29836,29817,1293,29836,22192,1516,1295,15915,1289,29824,18511,1288,29816,15394,1270,23673,29820,22192,14498,15394,1011,1294,22192,1300,29821,29828,17149,1291,14498,1271,25573,23673,1270,14157,25573,23673,14498,1000,1273,17149,29834,29836,22192,14498,25573,1000,1270,23673,29817,14498,1270,15915,29821,23673,29830,1292,23673,29816,14157,25573,1298,14157,14498,1273,29836,15394,29830,1270,23673,29817,17149,25573,29816,1275,29818],"tokens":["[CLS]","[","ب","##ا","##ل","##ص","##و","##ر","]","ت","##ر","##ق","##و","##م","##ي","##ا",":","ا","##ل","##م","##و","##ت","ي","##ط","##ف","##ى","##ء","ب","##ا","##ق","##ة","و","##ر","##د","ق","##ي","##د","ا","##ل","##ت","##ف","##ت","##ح","!","!","ا","##ل","##خ","##ل","##ي","##ل","–","د","##و","##ت","ك","##و","##م","–","م","##ن","غ","##س","##ان","ع","##ب","##د","ا","##ل","##ح","##م","##ي","##د","-","ل","##م","ي","##خ","##ط","##ر","ف","##ي","ب","##ا","##ل","ا","##ه","##ا","##ل","##ي","\"","ت","##ر","##ق","##و","##م","##ي","##ا","\"","ا","##ل","##ت","##ي","ا","##ن","##خ","##ل","##ع","ق","##ل","##ب","##ه","##ا","و","##ه","##ي","ت","##و","##د","##ع","ا","##ل","##ت","##ر","##ا","##ب","ج","##ث","[SEP]"],"offsets":[[0,0],[0,1],[1,2],[2,3],[3,4],[4,5],[5,6],[6,7],[7,8],[9,10],[10,11],[11,12],[12,13],[13,14],[14,15],[15,16],[17,18],[19,20],[20,21],[21,22],[22,23],[23,24],[25,26],[26,27],[27,28],[28,29],[29,30],[31,32],[32,33],[33,34],[34,35],[36,37],[37,38],[38,39],[40,41],[41,42],[42,43],[44,45],[45,46],[46,47],[47,48],[48,49],[49,50],[51,52],[52,53],[54,55],[55,56],[56,57],[57,58],[58,59],[59,60],[61,62],[63,64],[64,65],[65,66],[67,68],[68,69],[69,70],[71,72],[73,74],[74,75],[76,77],[77,78],[78,80],[81,82],[82,83],[83,84],[85,86],[86,87],[87,88],[88,89],[89,90],[90,91],[92,93],[94,95],[95,96],[97,98],[98,99],[99,100],[100,101],[102,103],[103,104],[105,106],[106,107],[107,108],[109,110],[110,111],[111,112],[112,113],[113,114],[115,116],[116,117],[117,118],[118,119],[119,120],[120,121],[121,122],[122,123],[123,124],[125,126],[126,127],[127,128],[128,129],[130,131],[131,132],[132,133],[133,134],[134,135],[136,137],[137,138],[138,139],[139,140],[140,141],[142,143],[143,144],[144,145],[146,147],[147,148],[148,149],[149,150],[151,152],[152,153],[153,154],[154,155],[155,156],[156,157],[158,159],[159,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[null,0,1,1,1,1,1,1,2,3,3,3,3,3,3,3,4,5,5,5,5,5,6,6,6,6,6,7,7,7,7,8,8,8,9,9,9,10,10,10,10,10,10,11,12,13,13,13,13,13,13,14,15,15,15,16,16,16,17,18,18,19,19,19,20,20,20,21,21,21,21,21,21,22,23,23,24,24,24,24,25,25,26,26,26,27,27,27,27,27,28,29,29,29,29,29,29,29,30,31,31,31,31,32,32,32,32,32,33,33,33,33,33,34,34,34,35,35,35,35,36,36,36,36,36,36,37,37,null],"decoded":"[ بالصور ] ترقوميا : الموت يطفىء باقة ورد قيد التفتح!! الخليل – دوت كوم – من غسان عبد الحميد - لم يخطر في بال اهالي \" ترقوميا \" التي انخلع قلبها وهي تودع التراب جث","decoded_with_specials":"[CLS] [ بالصور ] ترقوميا : الموت يطفىء باقة ورد قيد التفتح!! الخليل – دوت كوم – من غسان عبد الحميد - لم يخطر في بال اهالي \" ترقوميا \" التي انخلع قلبها وهي تودع التراب جث [SEP]"} +{"kind":"sample","source":"fixtures/lang/ben_Beng.txt[:160]","text":"গ্রহ নীহারিকা\nগ্রহ নীহারিকা (ইংরেজি ভাষায়: Planetary nebula) এক বিশেষ ধরনের গ্যাসীয় নীহারিকা। যেসব তারার ভর কম, নির্দিষ্টভাবে বলতে গেলে যেসব তারার ভর সূর্যের ","ids":[101,1355,29908,29913,1366,29916,29913,29914,29908,29915,29889,29914,1355,29908,29913,1366,29916,29913,29914,29908,29915,29889,29914,1006,1349,29882,29908,29917,29894,29915,1369,29914,29911,29914,29907,1024,17700,25677,1007,1351,29889,1368,29915,29910,29917,29911,1365,29908,29902,29917,29908,1355,29907,29914,29912,29916,29907,1366,29916,29913,29914,29908,29915,29889,29914,1344,1371,29917,29912,29904,1362,29914,29908,29914,29908,1369,29908,1353,29906,1010,1366,29915,29908,29900,29915,29911,29895,29905,29914,29904,29917,1368,29909,29898,29917,1355,29917,29909,29917,1371,29917,29912,29904,1362,29914,29908,29914,29908,1369,29908,1376,29908,29907,29917,29908,102],"ids_no_specials":[1355,29908,29913,1366,29916,29913,29914,29908,29915,29889,29914,1355,29908,29913,1366,29916,29913,29914,29908,29915,29889,29914,1006,1349,29882,29908,29917,29894,29915,1369,29914,29911,29914,29907,1024,17700,25677,1007,1351,29889,1368,29915,29910,29917,29911,1365,29908,29902,29917,29908,1355,29907,29914,29912,29916,29907,1366,29916,29913,29914,29908,29915,29889,29914,1344,1371,29917,29912,29904,1362,29914,29908,29914,29908,1369,29908,1353,29906,1010,1366,29915,29908,29900,29915,29911,29895,29905,29914,29904,29917,1368,29909,29898,29917,1355,29917,29909,29917,1371,29917,29912,29904,1362,29914,29908,29914,29908,1369,29908,1376,29908,29907,29917,29908],"tokens":["[CLS]","গ","##র","##হ","ন","##ী","##হ","##া","##র","##ি","##ক","##া","গ","##র","##হ","ন","##ী","##হ","##া","##র","##ি","##ক","##া","(","ই","##ং","##র","##ে","##জ","##ি","ভ","##া","##ষ","##া","##য",":","planetary","nebula",")","এ","##ক","ব","##ি","##শ","##ে","##ষ","ধ","##র","##ন","##ে","##র","গ","##য","##া","##স","##ী","##য","ন","##ী","##হ","##া","##র","##ি","##ক","##া","।","য","##ে","##স","##ব","ত","##া","##র","##া","##র","ভ","##র","ক","##ম",",","ন","##ি","##র","##দ","##ি","##ষ","##ট","##ভ","##া","##ব","##ে","ব","##ল","##ত","##ে","গ","##ে","##ল","##ে","য","##ে","##স","##ব","ত","##া","##র","##া","##র","ভ","##র","স","##র","##য","##ে","##র","[SEP]"],"offsets":[[0,0],[0,1],[2,3],[3,4],[5,6],[6,7],[7,8],[8,9],[9,10],[10,11],[11,12],[12,13],[14,15],[16,17],[17,18],[19,20],[20,21],[21,22],[22,23],[23,24],[24,25],[25,26],[26,27],[28,29],[29,30],[30,31],[31,32],[32,33],[33,34],[34,35],[36,37],[37,38],[38,39],[39,40],[40,41],[42,43],[44,53],[54,60],[60,61],[62,63],[63,64],[65,66],[66,67],[67,68],[68,69],[69,70],[71,72],[72,73],[73,74],[74,75],[75,76],[77,78],[79,80],[80,81],[81,82],[82,83],[83,84],[86,87],[87,88],[88,89],[89,90],[90,91],[91,92],[92,93],[93,94],[94,95],[96,97],[97,98],[98,99],[99,100],[101,102],[102,103],[103,104],[104,105],[105,106],[107,108],[108,109],[110,111],[111,112],[112,113],[114,115],[115,116],[116,117],[118,119],[119,120],[120,121],[122,123],[123,124],[124,125],[125,126],[126,127],[128,129],[129,130],[130,131],[131,132],[133,134],[134,135],[135,136],[136,137],[138,139],[139,140],[140,141],[141,142],[143,144],[144,145],[145,146],[146,147],[147,148],[149,150],[150,151],[152,153],[154,155],[156,157],[157,158],[158,159],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[null,0,0,0,1,1,1,1,1,1,1,1,2,2,2,3,3,3,3,3,3,3,3,4,5,5,5,5,5,5,6,6,6,6,6,7,8,9,10,11,11,12,12,12,12,12,13,13,13,13,13,14,14,14,14,14,14,15,15,15,15,15,15,15,15,16,17,17,17,17,18,18,18,18,18,19,19,20,20,21,22,22,22,22,22,22,22,22,22,22,22,23,23,23,23,24,24,24,24,25,25,25,25,26,26,26,26,26,27,27,28,28,28,28,28,null],"decoded":"গরহ নীহারিকা গরহ নীহারিকা ( ইংরেজি ভাষায : planetary nebula ) এক বিশেষ ধরনের গযাসীয নীহারিকা । যেসব তারার ভর কম, নিরদিষটভাবে বলতে গেলে যেসব তারার ভর সরযের","decoded_with_specials":"[CLS] গরহ নীহারিকা গরহ নীহারিকা ( ইংরেজি ভাষায : planetary nebula ) এক বিশেষ ধরনের গযাসীয নীহারিকা । যেসব তারার ভর কম, নিরদিষটভাবে বলতে গেলে যেসব তারার ভর সরযের [SEP]"} +{"kind":"sample","source":"fixtures/lang/cmn_Hani.txt[:160]","text":"強力建議大家未來盡量避免英航阿~~~\n班機原訂航程\n6/9 台北→香港\n6/9 香港→倫敦 (原訂23:15起飛，4:50am抵達)\n6/10 倫敦→斯德哥爾摩 (原訂7:40am起飛，11:05am抵達)\n就在第二段，英航no.25班機在香港機場兩度離開閘口又兩度返航\n第一次因有旅客身體不適 (好，不怪他，消耗時間也","ids":[101,100,1778,100,100,1810,1825,100,100,100,100,100,100,1941,100,1971,1066,1066,1066,100,100,1787,100,100,100,1020,1013,1023,100,1781,1585,1979,100,1020,1013,1023,1979,100,1585,100,100,1006,1787,100,2603,1024,2321,100,100,1989,1018,1024,2753,3286,100,100,1007,1020,1013,2184,100,100,1585,100,1848,100,100,100,1006,1787,100,1021,1024,2871,3286,100,100,1989,2340,1024,5709,3286,100,100,1007,100,100,100,1752,100,1989,1941,100,2053,1012,2423,100,100,100,1979,100,100,1806,100,100,100,100,100,1788,100,100,100,100,100,100,1740,100,100,1873,100,100,100,100,1744,100,1006,100,1989,1744,100,100,1989,100,100,100,1969,1750,102],"ids_no_specials":[100,1778,100,100,1810,1825,100,100,100,100,100,100,1941,100,1971,1066,1066,1066,100,100,1787,100,100,100,1020,1013,1023,100,1781,1585,1979,100,1020,1013,1023,1979,100,1585,100,100,1006,1787,100,2603,1024,2321,100,100,1989,1018,1024,2753,3286,100,100,1007,1020,1013,2184,100,100,1585,100,1848,100,100,100,1006,1787,100,1021,1024,2871,3286,100,100,1989,2340,1024,5709,3286,100,100,1007,100,100,100,1752,100,1989,1941,100,2053,1012,2423,100,100,100,1979,100,100,1806,100,100,100,100,100,1788,100,100,100,100,100,100,1740,100,100,1873,100,100,100,100,1744,100,1006,100,1989,1744,100,100,1989,100,100,100,1969,1750],"tokens":["[CLS]","[UNK]","力","[UNK]","[UNK]","大","家","[UNK]","[UNK]","[UNK]","[UNK]","[UNK]","[UNK]","英","[UNK]","阿","~","~","~","[UNK]","[UNK]","原","[UNK]","[UNK]","[UNK]","6","/","9","[UNK]","北","→","香","[UNK]","6","/","9","香","[UNK]","→","[UNK]","[UNK]","(","原","[UNK]","23",":","15","[UNK]","[UNK]","，","4",":","50","##am","[UNK]","[UNK]",")","6","/","10","[UNK]","[UNK]","→","[UNK]","德","[UNK]","[UNK]","[UNK]","(","原","[UNK]","7",":","40","##am","[UNK]","[UNK]","，","11",":","05","##am","[UNK]","[UNK]",")","[UNK]","[UNK]","[UNK]","二","[UNK]","，","英","[UNK]","no",".","25","[UNK]","[UNK]","[UNK]","香","[UNK]","[UNK]","場","[UNK]","[UNK]","[UNK]","[UNK]","[UNK]","口","[UNK]","[UNK]","[UNK]","[UNK]","[UNK]","[UNK]","一","[UNK]","[UNK]","有","[UNK]","[UNK]","[UNK]","[UNK]","不","[UNK]","(","[UNK]","，","不","[UNK]","[UNK]","，","[UNK]","[UNK]","[UNK]","間","也","[SEP]"],"offsets":[[0,0],[0,1],[1,2],[2,3],[3,4],[4,5],[5,6],[6,7],[7,8],[8,9],[9,10],[10,11],[11,12],[12,13],[13,14],[14,15],[15,16],[16,17],[17,18],[19,20],[20,21],[21,22],[22,23],[23,24],[24,25],[26,27],[27,28],[28,29],[30,31],[31,32],[32,33],[33,34],[34,35],[36,37],[37,38],[38,39],[40,41],[41,42],[42,43],[43,44],[44,45],[46,47],[47,48],[48,49],[49,51],[51,52],[52,54],[54,55],[55,56],[56,57],[57,58],[58,59],[59,61],[61,63],[63,64],[64,65],[65,66],[67,68],[68,69],[69,71],[72,73],[73,74],[74,75],[75,76],[76,77],[77,78],[78,79],[79,80],[81,82],[82,83],[83,84],[84,85],[85,86],[86,88],[88,90],[90,91],[91,92],[92,93],[93,95],[95,96],[96,98],[98,100],[100,101],[101,102],[102,103],[104,105],[105,106],[106,107],[107,108],[108,109],[109,110],[110,111],[111,112],[112,114],[114,115],[115,117],[117,118],[118,119],[119,120],[120,121],[121,122],[122,123],[123,124],[124,125],[125,126],[126,127],[127,128],[128,129],[129,130],[130,131],[131,132],[132,133],[133,134],[134,135],[136,137],[137,138],[138,139],[139,140],[140,141],[141,142],[142,143],[143,144],[144,145],[145,146],[146,147],[148,149],[149,150],[150,151],[151,152],[152,153],[153,154],[154,155],[155,156],[156,157],[157,158],[158,159],[159,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[null,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,71,72,73,74,75,76,77,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,null],"decoded":"力 大 家 英 阿 ~ ~ ~ 原 6 / 9 北 → 香 6 / 9 香 → ( 原 23 : 15 ， 4 : 50am ) 6 / 10 → 德 ( 原 7 : 40am ， 11 : 05am ) 二 ， 英 no. 25 香 場 口 一 有 不 ( ， 不 ， 間 也","decoded_with_specials":"[CLS] [UNK] 力 [UNK] [UNK] 大 家 [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] 英 [UNK] 阿 ~ ~ ~ [UNK] [UNK] 原 [UNK] [UNK] [UNK] 6 / 9 [UNK] 北 → 香 [UNK] 6 / 9 香 [UNK] → [UNK] [UNK] ( 原 [UNK] 23 : 15 [UNK] [UNK] ， 4 : 50am [UNK] [UNK] ) 6 / 10 [UNK] [UNK] → [UNK] 德 [UNK] [UNK] [UNK] ( 原 [UNK] 7 : 40am [UNK] [UNK] ， 11 : 05am [UNK] [UNK] ) [UNK] [UNK] [UNK] 二 [UNK] ， 英 [UNK] no. 25 [UNK] [UNK] [UNK] 香 [UNK] [UNK] 場 [UNK] [UNK] [UNK] [UNK] [UNK] 口 [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] 一 [UNK] [UNK] 有 [UNK] [UNK] [UNK] [UNK] 不 [UNK] ( [UNK] ， 不 [UNK] [UNK] ， [UNK] [UNK] [UNK] 間 也 [SEP]"} +{"kind":"sample","source":"fixtures/lang/ell_Grek.txt[:160]","text":"Θυμήσου με\nΗ πόλη προσφέρει πολλές ευκαιρίες - πήγαινε στο εμπορικό κέντρο, στο σαλόνι ομορφιάς , σε κλάμπ και διασκέδασε με την ψυχή σου.\nΔημιούργησε μαγευτικά","ids":[101,1162,29735,29728,24824,29733,26789,1166,29723,1161,1170,29730,29727,24824,1170,29732,29730,29733,29736,29723,29732,29723,18199,1170,29730,29727,29727,29723,19579,1159,29735,29726,14608,18199,29732,18199,29723,19579,1011,1170,24824,29721,14608,18199,16177,29723,1173,29734,29730,1159,29728,29731,29730,29732,18199,29726,29730,1164,29723,16177,29734,29732,29730,1010,1173,29734,29730,1173,14608,29727,29730,16177,18199,1169,29728,29730,29732,29736,27432,19579,1010,1173,29723,1164,29727,14608,29728,29731,1164,14608,18199,1158,27432,29733,29726,29723,29722,14608,29733,29723,1166,29723,1174,24824,16177,1178,29735,29737,24824,1173,26789,1012,1158,24824,29728,18199,26789,29732,29721,24824,29733,29723,1166,14608,29721,29723,29735,29734,18199,29726,14608,102],"ids_no_specials":[1162,29735,29728,24824,29733,26789,1166,29723,1161,1170,29730,29727,24824,1170,29732,29730,29733,29736,29723,29732,29723,18199,1170,29730,29727,29727,29723,19579,1159,29735,29726,14608,18199,29732,18199,29723,19579,1011,1170,24824,29721,14608,18199,16177,29723,1173,29734,29730,1159,29728,29731,29730,29732,18199,29726,29730,1164,29723,16177,29734,29732,29730,1010,1173,29734,29730,1173,14608,29727,29730,16177,18199,1169,29728,29730,29732,29736,27432,19579,1010,1173,29723,1164,29727,14608,29728,29731,1164,14608,18199,1158,27432,29733,29726,29723,29722,14608,29733,29723,1166,29723,1174,24824,16177,1178,29735,29737,24824,1173,26789,1012,1158,24824,29728,18199,26789,29732,29721,24824,29733,29723,1166,14608,29721,29723,29735,29734,18199,29726,14608],"tokens":["[CLS]","θ","##υ","##μ","##η","##σ","##ου","μ","##ε","η","π","##ο","##λ","##η","π","##ρ","##ο","##σ","##φ","##ε","##ρ","##ε","##ι","π","##ο","##λ","##λ","##ε","##ς","ε","##υ","##κ","##α","##ι","##ρ","##ι","##ε","##ς","-","π","##η","##γ","##α","##ι","##ν","##ε","σ","##τ","##ο","ε","##μ","##π","##ο","##ρ","##ι","##κ","##ο","κ","##ε","##ν","##τ","##ρ","##ο",",","σ","##τ","##ο","σ","##α","##λ","##ο","##ν","##ι","ο","##μ","##ο","##ρ","##φ","##ια","##ς",",","σ","##ε","κ","##λ","##α","##μ","##π","κ","##α","##ι","δ","##ια","##σ","##κ","##ε","##δ","##α","##σ","##ε","μ","##ε","τ","##η","##ν","ψ","##υ","##χ","##η","σ","##ου",".","δ","##η","##μ","##ι","##ου","##ρ","##γ","##η","##σ","##ε","μ","##α","##γ","##ε","##υ","##τ","##ι","##κ","##α","[SEP]"],"offsets":[[0,0],[0,1],[1,2],[2,3],[3,4],[4,5],[5,7],[8,9],[9,10],[11,12],[13,14],[14,15],[15,16],[16,17],[18,19],[19,20],[20,21],[21,22],[22,23],[23,24],[24,25],[25,26],[26,27],[28,29],[29,30],[30,31],[31,32],[32,33],[33,34],[35,36],[36,37],[37,38],[38,39],[39,40],[40,41],[41,42],[42,43],[43,44],[45,46],[47,48],[48,49],[49,50],[50,51],[51,52],[52,53],[53,54],[55,56],[56,57],[57,58],[59,60],[60,61],[61,62],[62,63],[63,64],[64,65],[65,66],[66,67],[68,69],[69,70],[70,71],[71,72],[72,73],[73,74],[74,75],[76,77],[77,78],[78,79],[80,81],[81,82],[82,83],[83,84],[84,85],[85,86],[87,88],[88,89],[89,90],[90,91],[91,92],[92,94],[94,95],[96,97],[98,99],[99,100],[101,102],[102,103],[103,104],[104,105],[105,106],[107,108],[108,109],[109,110],[111,112],[112,114],[114,115],[115,116],[116,117],[117,118],[118,119],[119,120],[120,121],[122,123],[123,124],[125,126],[126,127],[127,128],[129,130],[130,131],[131,132],[132,133],[134,135],[135,137],[137,138],[139,140],[140,141],[141,142],[142,143],[143,145],[145,146],[146,147],[147,148],[148,149],[149,150],[151,152],[152,153],[153,154],[154,155],[155,156],[156,157],[157,158],[158,159],[159,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[null,0,0,0,0,0,0,1,1,2,3,3,3,3,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,7,8,8,8,8,8,8,8,9,9,9,10,10,10,10,10,10,10,10,11,11,11,11,11,11,12,13,13,13,14,14,14,14,14,14,15,15,15,15,15,15,15,16,17,17,18,18,18,18,18,19,19,19,20,20,20,20,20,20,20,20,20,21,21,22,22,22,23,23,23,23,24,24,25,26,26,26,26,26,26,26,26,26,26,27,27,27,27,27,27,27,27,27,null],"decoded":"θυμησου με η πολη προσφερει πολλες ευκαιριες - πηγαινε στο εμπορικο κεντρο, στο σαλονι ομορφιας, σε κλαμπ και διασκεδασε με την ψυχη σου. δημιουργησε μαγευτικα","decoded_with_specials":"[CLS] θυμησου με η πολη προσφερει πολλες ευκαιριες - πηγαινε στο εμπορικο κεντρο, στο σαλονι ομορφιας, σε κλαμπ και διασκεδασε με την ψυχη σου. δημιουργησε μαγευτικα [SEP]"} +{"kind":"sample","source":"fixtures/lang/eng_Latn.txt[:160]","text":"|Viewing Single Post From: Spoilers for the Week of February 11th|\n|Lil||Feb 1 2013, 09:58 AM|\nDon't care about Chloe/Taniel/Jen-Jen. Don't care about Sami, rea","ids":[101,1064,10523,2309,2695,2013,1024,27594,2545,2005,1996,2733,1997,2337,6252,1064,1064,13451,1064,1064,13114,1015,2286,1010,5641,1024,5388,2572,1064,2123,1005,1056,2729,2055,9318,1013,9092,9257,1013,15419,1011,15419,1012,2123,1005,1056,2729,2055,17015,1010,2128,2050,102],"ids_no_specials":[1064,10523,2309,2695,2013,1024,27594,2545,2005,1996,2733,1997,2337,6252,1064,1064,13451,1064,1064,13114,1015,2286,1010,5641,1024,5388,2572,1064,2123,1005,1056,2729,2055,9318,1013,9092,9257,1013,15419,1011,15419,1012,2123,1005,1056,2729,2055,17015,1010,2128,2050],"tokens":["[CLS]","|","viewing","single","post","from",":","spoil","##ers","for","the","week","of","february","11th","|","|","lil","|","|","feb","1","2013",",","09",":","58","am","|","don","'","t","care","about","chloe","/","tan","##iel","/","jen","-","jen",".","don","'","t","care","about","sami",",","re","##a","[SEP]"],"offsets":[[0,0],[0,1],[1,8],[9,15],[16,20],[21,25],[25,26],[27,32],[32,35],[36,39],[40,43],[44,48],[49,51],[52,60],[61,65],[65,66],[67,68],[68,71],[71,72],[72,73],[73,76],[77,78],[79,83],[83,84],[85,87],[87,88],[88,90],[91,93],[93,94],[95,98],[98,99],[99,100],[101,105],[106,111],[112,117],[117,118],[118,121],[121,124],[124,125],[125,128],[128,129],[129,132],[132,133],[134,137],[137,138],[138,139],[140,144],[145,150],[151,155],[155,156],[157,159],[159,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[null,0,1,2,3,4,5,6,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,34,35,36,37,38,39,40,41,42,43,44,45,46,47,47,null],"decoded":"| viewing single post from : spoilers for the week of february 11th | | lil | | feb 1 2013, 09 : 58 am | don ' t care about chloe / taniel / jen - jen. don ' t care about sami, rea","decoded_with_specials":"[CLS] | viewing single post from : spoilers for the week of february 11th | | lil | | feb 1 2013, 09 : 58 am | don ' t care about chloe / taniel / jen - jen. don ' t care about sami, rea [SEP]"} +{"kind":"sample","source":"fixtures/lang/heb_Hebr.txt[:160]","text":"סיכונים – אלמנט מרכזי, חיוני לפעילות מסחרית. להבין מה הסיכון הוא מאוד חשוב. החוויה האנושית עולה כי מי יודע איך לקחת סיכונים בזמן, היא לנצח גדולים. לזכור פוליטיק","ids":[101,1258,29796,29798,29792,29803,29796,29800,1516,1241,29799,29801,29803,29795,1255,29811,29798,29793,29796,1010,1248,29796,29792,29803,29796,1253,29807,29805,29796,29799,29792,29813,1255,29804,29794,29811,29796,29813,1012,1253,29128,29789,29796,29802,1255,29128,1245,29804,29796,29798,29792,29802,1245,29792,29788,1255,29788,29792,29791,1248,29812,29792,29789,1012,1245,29794,29792,29792,29796,29128,1245,29788,29803,29792,29812,29796,29813,1259,29792,29799,29128,1252,29796,1255,29796,1250,29792,29791,29805,1241,29796,29797,1253,29810,29794,29813,1258,29796,29798,29792,29803,29796,29800,1242,29793,29801,29802,1010,1245,29796,29788,1253,29803,29809,29794,1243,29791,29792,29799,29796,29800,1012,1253,29793,29798,29792,29811,1261,29792,29799,29796,29795,29796,29810,102],"ids_no_specials":[1258,29796,29798,29792,29803,29796,29800,1516,1241,29799,29801,29803,29795,1255,29811,29798,29793,29796,1010,1248,29796,29792,29803,29796,1253,29807,29805,29796,29799,29792,29813,1255,29804,29794,29811,29796,29813,1012,1253,29128,29789,29796,29802,1255,29128,1245,29804,29796,29798,29792,29802,1245,29792,29788,1255,29788,29792,29791,1248,29812,29792,29789,1012,1245,29794,29792,29792,29796,29128,1245,29788,29803,29792,29812,29796,29813,1259,29792,29799,29128,1252,29796,1255,29796,1250,29792,29791,29805,1241,29796,29797,1253,29810,29794,29813,1258,29796,29798,29792,29803,29796,29800,1242,29793,29801,29802,1010,1245,29796,29788,1253,29803,29809,29794,1243,29791,29792,29799,29796,29800,1012,1253,29793,29798,29792,29811,1261,29792,29799,29796,29795,29796,29810],"tokens":["[CLS]","ס","##י","##כ","##ו","##נ","##י","##ם","–","א","##ל","##מ","##נ","##ט","מ","##ר","##כ","##ז","##י",",","ח","##י","##ו","##נ","##י","ל","##פ","##ע","##י","##ל","##ו","##ת","מ","##ס","##ח","##ר","##י","##ת",".","ל","##ה","##ב","##י","##ן","מ","##ה","ה","##ס","##י","##כ","##ו","##ן","ה","##ו","##א","מ","##א","##ו","##ד","ח","##ש","##ו","##ב",".","ה","##ח","##ו","##ו","##י","##ה","ה","##א","##נ","##ו","##ש","##י","##ת","ע","##ו","##ל","##ה","כ","##י","מ","##י","י","##ו","##ד","##ע","א","##י","##ך","ל","##ק","##ח","##ת","ס","##י","##כ","##ו","##נ","##י","##ם","ב","##ז","##מ","##ן",",","ה","##י","##א","ל","##נ","##צ","##ח","ג","##ד","##ו","##ל","##י","##ם",".","ל","##ז","##כ","##ו","##ר","פ","##ו","##ל","##י","##ט","##י","##ק","[SEP]"],"offsets":[[0,0],[0,1],[1,2],[2,3],[3,4],[4,5],[5,6],[6,7],[8,9],[10,11],[11,12],[12,13],[13,14],[14,15],[16,17],[17,18],[18,19],[19,20],[20,21],[21,22],[23,24],[24,25],[25,26],[26,27],[27,28],[29,30],[30,31],[31,32],[32,33],[33,34],[34,35],[35,36],[37,38],[38,39],[39,40],[40,41],[41,42],[42,43],[43,44],[45,46],[46,47],[47,48],[48,49],[49,50],[51,52],[52,53],[54,55],[55,56],[56,57],[57,58],[58,59],[59,60],[61,62],[62,63],[63,64],[65,66],[66,67],[67,68],[68,69],[70,71],[71,72],[72,73],[73,74],[74,75],[76,77],[77,78],[78,79],[79,80],[80,81],[81,82],[83,84],[84,85],[85,86],[86,87],[87,88],[88,89],[89,90],[91,92],[92,93],[93,94],[94,95],[96,97],[97,98],[99,100],[100,101],[102,103],[103,104],[104,105],[105,106],[107,108],[108,109],[109,110],[111,112],[112,113],[113,114],[114,115],[116,117],[117,118],[118,119],[119,120],[120,121],[121,122],[122,123],[124,125],[125,126],[126,127],[127,128],[128,129],[130,131],[131,132],[132,133],[134,135],[135,136],[136,137],[137,138],[139,140],[140,141],[141,142],[142,143],[143,144],[144,145],[145,146],[147,148],[148,149],[149,150],[150,151],[151,152],[153,154],[154,155],[155,156],[156,157],[157,158],[158,159],[159,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[null,0,0,0,0,0,0,0,1,2,2,2,2,2,3,3,3,3,3,4,5,5,5,5,5,6,6,6,6,6,6,6,7,7,7,7,7,7,8,9,9,9,9,9,10,10,11,11,11,11,11,11,12,12,12,13,13,13,13,14,14,14,14,15,16,16,16,16,16,16,17,17,17,17,17,17,17,18,18,18,18,19,19,20,20,21,21,21,21,22,22,22,23,23,23,23,24,24,24,24,24,24,24,25,25,25,25,26,27,27,27,28,28,28,28,29,29,29,29,29,29,30,31,31,31,31,31,32,32,32,32,32,32,32,null],"decoded":"סיכונים – אלמנט מרכזי, חיוני לפעילות מסחרית. להבין מה הסיכון הוא מאוד חשוב. החוויה האנושית עולה כי מי יודע איך לקחת סיכונים בזמן, היא לנצח גדולים. לזכור פוליטיק","decoded_with_specials":"[CLS] סיכונים – אלמנט מרכזי, חיוני לפעילות מסחרית. להבין מה הסיכון הוא מאוד חשוב. החוויה האנושית עולה כי מי יודע איך לקחת סיכונים בזמן, היא לנצח גדולים. לזכור פוליטיק [SEP]"} +{"kind":"sample","source":"fixtures/lang/hin_Deva.txt[:160]","text":"PHOTOS: न्यूज पढ़ते-पढ़ते अचानक ये क्या करने लगी एंकर!\nकुछ समय पहले एक टीवी चैनल पर एंकर खबर पढ़ रही थी और पीछे की स्क्रीन पर 10 मिनिट का पोर्न वीडियो चलता रहा,","ids":[101,7760,1024,1327,29868,29855,100,1011,100,1311,29854,29876,29863,29851,1332,1315,29868,29876,1315,29869,29863,1334,29853,29878,1314,29851,29869,999,100,1338,29867,29868,1328,29875,29870,1314,29851,1320,29878,29871,29878,1318,29863,29870,1328,29869,1314,29851,29869,1316,29865,29869,100,1333,29875,29878,1324,29878,100,100,1315,29878,1338,29851,29869,29878,29863,1328,29869,2184,1331,29877,29863,29877,29856,1315,29876,1328,29879,29869,29863,1335,29878,29857,29877,29868,29879,1318,29870,29859,29876,1333,29875,29876,1010,102],"ids_no_specials":[7760,1024,1327,29868,29855,100,1011,100,1311,29854,29876,29863,29851,1332,1315,29868,29876,1315,29869,29863,1334,29853,29878,1314,29851,29869,999,100,1338,29867,29868,1328,29875,29870,1314,29851,1320,29878,29871,29878,1318,29863,29870,1328,29869,1314,29851,29869,1316,29865,29869,100,1333,29875,29878,1324,29878,100,100,1315,29878,1338,29851,29869,29878,29863,1328,29869,2184,1331,29877,29863,29877,29856,1315,29876,1328,29879,29869,29863,1335,29878,29857,29877,29868,29879,1318,29870,29859,29876,1333,29875,29876,1010],"tokens":["[CLS]","photos",":","न","##य","##ज","[UNK]","-","[UNK]","अ","##च","##ा","##न","##क","य","क","##य","##ा","क","##र","##न","ल","##ग","##ी","ए","##क","##र","!","[UNK]","स","##म","##य","प","##ह","##ल","ए","##क","ट","##ी","##व","##ी","च","##न","##ल","प","##र","ए","##क","##र","ख","##ब","##र","[UNK]","र","##ह","##ी","थ","##ी","[UNK]","[UNK]","क","##ी","स","##क","##र","##ी","##न","प","##र","10","म","##ि","##न","##ि","##ट","क","##ा","प","##ो","##र","##न","व","##ी","##ड","##ि","##य","##ो","च","##ल","##त","##ा","र","##ह","##ा",",","[SEP]"],"offsets":[[0,0],[0,6],[6,7],[8,9],[10,11],[12,13],[14,18],[19,20],[20,24],[26,27],[27,28],[28,29],[29,30],[30,31],[32,33],[35,36],[37,38],[38,39],[40,41],[41,42],[42,43],[45,46],[46,47],[47,48],[49,50],[51,52],[52,53],[53,54],[55,58],[59,60],[60,61],[61,62],[63,64],[64,65],[65,66],[68,69],[69,70],[71,72],[72,73],[73,74],[74,75],[76,77],[78,79],[79,80],[81,82],[82,83],[84,85],[86,87],[87,88],[89,90],[90,91],[91,92],[93,95],[97,98],[98,99],[99,100],[101,102],[102,103],[104,106],[107,110],[112,113],[113,114],[115,116],[117,118],[119,120],[120,121],[121,122],[123,124],[124,125],[126,128],[129,130],[130,131],[131,132],[132,133],[133,134],[135,136],[136,137],[138,139],[139,140],[140,141],[142,143],[144,145],[145,146],[146,147],[147,148],[148,149],[149,150],[151,152],[152,153],[153,154],[154,155],[156,157],[157,158],[158,159],[159,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[null,0,1,2,2,2,3,4,5,6,6,6,6,6,7,8,8,8,9,9,9,10,10,10,11,11,11,12,13,14,14,14,15,15,15,16,16,17,17,17,17,18,18,18,19,19,20,20,20,21,21,21,22,23,23,23,24,24,25,26,27,27,28,28,28,28,28,29,29,30,31,31,31,31,31,32,32,33,33,33,33,34,34,34,34,34,34,35,35,35,35,36,36,36,37,null],"decoded":"photos : नयज - अचानक य कया करन लगी एकर! समय पहल एक टीवी चनल पर एकर खबर रही थी की सकरीन पर 10 मिनिट का पोरन वीडियो चलता रहा,","decoded_with_specials":"[CLS] photos : नयज [UNK] - [UNK] अचानक य कया करन लगी एकर! [UNK] समय पहल एक टीवी चनल पर एकर खबर [UNK] रही थी [UNK] [UNK] की सकरीन पर 10 मिनिट का पोरन वीडियो चलता रहा, [SEP]"} +{"kind":"sample","source":"fixtures/lang/jpn_Jpan.txt[:160]","text":"Omni Dallas Parkwest Hotelでは4ツ星ホテルで、アイアンホース・ゴルフコース、Love Field Airport とテキサス・スタジアムから7.8kmかかるのところにあります。 優れたホテルにオープンし、ダラスにある古代の建築の象徴です。\n部屋\n快適なゲストルームには、モダンな設備を備えたプレ","ids":[101,18168,3490,5759,2380,19650,3309,30191,30198,2549,30238,1866,1722,30239,30259,30191,1635,1693,30221,30219,30263,30248,30265,30233,1738,1704,30259,30246,30230,30265,30233,1635,2293,2492,3199,1666,30239,30227,30231,30233,1738,1707,30235,30232,30219,30251,30177,30211,2581,1012,1022,22287,30177,30177,30213,30197,30192,30181,30215,30194,30172,30212,30203,30184,1636,100,1688,30187,30248,30239,30259,30194,30225,30265,30246,30263,30183,1635,1709,30257,30233,30194,30172,30213,1789,1760,1671,100,100,1671,100,100,1665,30184,1636,1960,100,100,100,1667,30229,30233,30240,30259,30265,30251,30194,30198,1635,1727,30235,30263,30193,100,100,1690,100,1649,30187,30246,30260,102],"ids_no_specials":[18168,3490,5759,2380,19650,3309,30191,30198,2549,30238,1866,1722,30239,30259,30191,1635,1693,30221,30219,30263,30248,30265,30233,1738,1704,30259,30246,30230,30265,30233,1635,2293,2492,3199,1666,30239,30227,30231,30233,1738,1707,30235,30232,30219,30251,30177,30211,2581,1012,1022,22287,30177,30177,30213,30197,30192,30181,30215,30194,30172,30212,30203,30184,1636,100,1688,30187,30248,30239,30259,30194,30225,30265,30246,30263,30183,1635,1709,30257,30233,30194,30172,30213,1789,1760,1671,100,100,1671,100,100,1665,30184,1636,1960,100,100,100,1667,30229,30233,30240,30259,30265,30251,30194,30198,1635,1727,30235,30263,30193,100,100,1690,100,1649,30187,30246,30260],"tokens":["[CLS]","om","##ni","dallas","park","##west","hotel","##て","##は","##4","##ツ","星","ホ","##テ","##ル","##て","、","ア","##イ","##ア","##ン","##ホ","##ー","##ス","・","コ","##ル","##フ","##コ","##ー","##ス","、","love","field","airport","と","##テ","##キ","##サ","##ス","・","ス","##タ","##シ","##ア","##ム","##か","##ら","##7",".","8","##km","##か","##か","##る","##の","##と","##こ","##ろ","##に","##あ","##り","##ま","##す","。","[UNK]","れ","##た","##ホ","##テ","##ル","##に","##オ","##ー","##フ","##ン","##し","、","タ","##ラ","##ス","##に","##あ","##る","古","代","の","[UNK]","[UNK]","の","[UNK]","[UNK]","て","##す","。","部","[UNK]","[UNK]","[UNK]","な","##ケ","##ス","##ト","##ル","##ー","##ム","##に","##は","、","モ","##タ","##ン","##な","[UNK]","[UNK]","を","[UNK]","え","##た","##フ","##レ","[SEP]"],"offsets":[[0,0],[0,2],[2,4],[5,11],[12,16],[16,20],[21,26],[26,27],[27,28],[28,29],[29,30],[30,31],[31,32],[32,33],[33,34],[34,35],[35,36],[36,37],[37,38],[38,39],[39,40],[40,41],[41,42],[42,43],[43,44],[44,45],[45,46],[46,47],[47,48],[48,49],[49,50],[50,51],[51,55],[56,61],[62,69],[70,71],[71,72],[72,73],[73,74],[74,75],[75,76],[76,77],[77,78],[78,79],[79,80],[80,81],[81,82],[82,83],[83,84],[84,85],[85,86],[86,88],[88,89],[89,90],[90,91],[91,92],[92,93],[93,94],[94,95],[95,96],[96,97],[97,98],[98,99],[99,100],[100,101],[102,103],[103,104],[104,105],[105,106],[106,107],[107,108],[108,109],[109,110],[110,111],[111,112],[112,113],[113,114],[114,115],[115,116],[116,117],[117,118],[118,119],[119,120],[120,121],[121,122],[122,123],[123,124],[124,125],[125,126],[126,127],[127,128],[128,129],[129,130],[130,131],[131,132],[133,134],[134,135],[136,137],[137,138],[138,139],[139,140],[140,141],[141,142],[142,143],[143,144],[144,145],[145,146],[146,147],[147,148],[148,149],[149,150],[150,151],[151,152],[152,153],[153,154],[154,155],[155,156],[156,157],[157,158],[158,159],[159,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[null,0,0,1,2,2,3,3,3,3,3,4,5,5,5,5,6,7,7,7,7,7,7,7,8,9,9,9,9,9,9,10,11,12,13,14,14,14,14,14,15,16,16,16,16,16,16,16,16,17,18,18,18,18,18,18,18,18,18,18,18,18,18,18,19,20,21,21,21,21,21,21,21,21,21,21,21,22,23,23,23,23,23,23,24,25,26,27,28,29,30,31,32,32,33,34,35,36,37,38,38,38,38,38,38,38,38,38,39,40,40,40,40,41,42,43,44,45,45,45,45,null],"decoded":"omni dallas parkwest hotelては4ツ 星 ホテルて 、 アイアンホース ・ コルフコース 、 love field airport とテキサス ・ スタシアムから7. 8kmかかるのところにあります 。 れたホテルにオーフンし 、 タラスにある 古 代 の の てす 。 部 なケストルームには 、 モタンな を えたフレ","decoded_with_specials":"[CLS] omni dallas parkwest hotelては4ツ 星 ホテルて 、 アイアンホース ・ コルフコース 、 love field airport とテキサス ・ スタシアムから7. 8kmかかるのところにあります 。 [UNK] れたホテルにオーフンし 、 タラスにある 古 代 の [UNK] [UNK] の [UNK] [UNK] てす 。 部 [UNK] [UNK] [UNK] なケストルームには 、 モタンな [UNK] [UNK] を [UNK] えたフレ [SEP]"} +{"kind":"sample","source":"fixtures/lang/kat_Geor.txt[:160]","text":"ჩვენ მსოფლიოს სხვადასხვა ქვეყანაში ვცხოვრობთ და სხვადასხვა ენაზე ვსაუბრობთ, მაგრამ საერთო მიზანი გვაერთიანებს. ჩვენი მთავარი მიზანი სამყაროს შემოქმედისა და ბიბლ","ids":[101,100,100,100,100,100,1441,29974,100,100,1443,29988,29974,29990,29975,29987,29986,29975,29980,1010,1448,29974,29976,29987,29974,29984,1452,29974,29978,29987,29980,29986,100,1440,29979,29974,29978,29987,29980,29981,29974,29985,29978,29975,29988,1012,100,1448,29980,29974,29979,29974,29987,29981,100,100,100,1441,29974,1439,29981,29975,29983,102],"ids_no_specials":[100,100,100,100,100,1441,29974,100,100,1443,29988,29974,29990,29975,29987,29986,29975,29980,1010,1448,29974,29976,29987,29974,29984,1452,29974,29978,29987,29980,29986,100,1440,29979,29974,29978,29987,29980,29981,29974,29985,29978,29975,29988,1012,100,1448,29980,29974,29979,29974,29987,29981,100,100,100,1441,29974,1439,29981,29975,29983],"tokens":["[CLS]","[UNK]","[UNK]","[UNK]","[UNK]","[UNK]","დ","##ა","[UNK]","[UNK]","ვ","##ს","##ა","##უ","##ბ","##რ","##ო","##ბ","##თ",",","მ","##ა","##გ","##რ","##ა","##მ","ს","##ა","##ე","##რ","##თ","##ო","[UNK]","გ","##ვ","##ა","##ე","##რ","##თ","##ი","##ა","##ნ","##ე","##ბ","##ს",".","[UNK]","მ","##თ","##ა","##ვ","##ა","##რ","##ი","[UNK]","[UNK]","[UNK]","დ","##ა","ბ","##ი","##ბ","##ლ","[SEP]"],"offsets":[[0,0],[0,4],[5,13],[14,24],[25,34],[35,44],[45,46],[46,47],[48,58],[59,64],[65,66],[66,67],[67,68],[68,69],[69,70],[70,71],[71,72],[72,73],[73,74],[74,75],[76,77],[77,78],[78,79],[79,80],[80,81],[81,82],[83,84],[84,85],[85,86],[86,87],[87,88],[88,89],[90,96],[97,98],[98,99],[99,100],[100,101],[101,102],[102,103],[103,104],[104,105],[105,106],[106,107],[107,108],[108,109],[109,110],[111,116],[117,118],[118,119],[119,120],[120,121],[121,122],[122,123],[123,124],[125,131],[132,140],[141,152],[153,154],[154,155],[156,157],[157,158],[158,159],[159,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[null,0,1,2,3,4,5,5,6,7,8,8,8,8,8,8,8,8,8,9,10,10,10,10,10,10,11,11,11,11,11,11,12,13,13,13,13,13,13,13,13,13,13,13,13,14,15,16,16,16,16,16,16,16,17,18,19,20,20,21,21,21,21,null],"decoded":"და ვსაუბრობთ, მაგრამ საერთო გვაერთიანებს. მთავარი და ბიბლ","decoded_with_specials":"[CLS] [UNK] [UNK] [UNK] [UNK] [UNK] და [UNK] [UNK] ვსაუბრობთ, მაგრამ საერთო [UNK] გვაერთიანებს. [UNK] მთავარი [UNK] [UNK] [UNK] და ბიბლ [SEP]"} +{"kind":"sample","source":"fixtures/lang/kor_Hang.txt[:160]","text":"전화번호:\n위치: 뉴질랜드 > 남섬 > 말버러 > 블레넘\n가격대 (1박): 117,886 원 - 140,244 원\n4.5성급 — Lugano Motor Lodge 4.5*\n- 예약 옵션:\n- 트립어드바이저는 호텔스닷컴, Expedia, Agoda, Asia Web Direct 및 Boo","ids":[101,1464,30008,30021,30005,30012,29996,30008,30021,30005,30011,1024,100,1024,1456,30016,30000,30019,30022,29994,30007,30021,29993,30017,1028,1456,30006,30023,29997,30008,30023,1028,1459,30006,30022,29996,30008,29994,30008,1028,1460,30017,30022,29994,30009,29992,30008,30023,1455,30006,29991,30010,30020,29993,30007,1006,1015,29996,30006,30020,1007,1024,12567,1010,6070,2575,1463,30015,30021,1011,8574,1010,24194,1463,30015,30021,1018,1012,1019,29997,30008,30025,29991,30017,30024,1517,11320,29451,5013,7410,1018,1012,1019,1008,1011,100,1463,30011,30024,29997,30010,30021,1024,1011,1467,30017,29994,30019,30024,29999,30008,29993,30017,29996,30006,29999,30019,30000,30008,29992,30017,30021,100,1010,4654,5669,2401,1010,3283,2850,1010,4021,4773,3622,100,22017,102],"ids_no_specials":[1464,30008,30021,30005,30012,29996,30008,30021,30005,30011,1024,100,1024,1456,30016,30000,30019,30022,29994,30007,30021,29993,30017,1028,1456,30006,30023,29997,30008,30023,1028,1459,30006,30022,29996,30008,29994,30008,1028,1460,30017,30022,29994,30009,29992,30008,30023,1455,30006,29991,30010,30020,29993,30007,1006,1015,29996,30006,30020,1007,1024,12567,1010,6070,2575,1463,30015,30021,1011,8574,1010,24194,1463,30015,30021,1018,1012,1019,29997,30008,30025,29991,30017,30024,1517,11320,29451,5013,7410,1018,1012,1019,1008,1011,100,1463,30011,30024,29997,30010,30021,1024,1011,1467,30017,29994,30019,30024,29999,30008,29993,30017,29996,30006,29999,30019,30000,30008,29992,30017,30021,100,1010,4654,5669,2401,1010,3283,2850,1010,4021,4773,3622,100,22017],"tokens":["[CLS]","ᄌ","##ᅥ","##ᆫ","##ᄒ","##ᅪ","##ᄇ","##ᅥ","##ᆫ","##ᄒ","##ᅩ",":","[UNK]",":","ᄂ","##ᅲ","##ᄌ","##ᅵ","##ᆯ","##ᄅ","##ᅢ","##ᆫ","##ᄃ","##ᅳ",">","ᄂ","##ᅡ","##ᆷ","##ᄉ","##ᅥ","##ᆷ",">","ᄆ","##ᅡ","##ᆯ","##ᄇ","##ᅥ","##ᄅ","##ᅥ",">","ᄇ","##ᅳ","##ᆯ","##ᄅ","##ᅦ","##ᄂ","##ᅥ","##ᆷ","ᄀ","##ᅡ","##ᄀ","##ᅧ","##ᆨ","##ᄃ","##ᅢ","(","1","##ᄇ","##ᅡ","##ᆨ",")",":","117",",","88","##6","ᄋ","##ᅯ","##ᆫ","-","140",",","244","ᄋ","##ᅯ","##ᆫ","4",".","5","##ᄉ","##ᅥ","##ᆼ","##ᄀ","##ᅳ","##ᆸ","—","lu","##gano","motor","lodge","4",".","5","*","-","[UNK]","ᄋ","##ᅩ","##ᆸ","##ᄉ","##ᅧ","##ᆫ",":","-","ᄐ","##ᅳ","##ᄅ","##ᅵ","##ᆸ","##ᄋ","##ᅥ","##ᄃ","##ᅳ","##ᄇ","##ᅡ","##ᄋ","##ᅵ","##ᄌ","##ᅥ","##ᄂ","##ᅳ","##ᆫ","[UNK]",",","ex","##ped","##ia",",","ago","##da",",","asia","web","direct","[UNK]","boo","[SEP]"],"offsets":[[0,0],[0,1],[0,1],[0,1],[1,2],[1,2],[2,3],[2,3],[2,3],[3,4],[3,4],[4,5],[6,8],[8,9],[10,11],[10,11],[11,12],[11,12],[11,12],[12,13],[12,13],[12,13],[13,14],[13,14],[15,16],[17,18],[17,18],[17,18],[18,19],[18,19],[18,19],[20,21],[22,23],[22,23],[22,23],[23,24],[23,24],[24,25],[24,25],[26,27],[28,29],[28,29],[28,29],[29,30],[29,30],[30,31],[30,31],[30,31],[32,33],[32,33],[33,34],[33,34],[33,34],[34,35],[34,35],[36,37],[37,38],[38,39],[38,39],[38,39],[39,40],[40,41],[42,45],[45,46],[46,48],[48,49],[50,51],[50,51],[50,51],[52,53],[54,57],[57,58],[58,61],[62,63],[62,63],[62,63],[64,65],[65,66],[66,67],[67,68],[67,68],[67,68],[68,69],[68,69],[68,69],[70,71],[72,74],[74,78],[79,84],[85,90],[91,92],[92,93],[93,94],[94,95],[96,97],[98,100],[101,102],[101,102],[101,102],[102,103],[102,103],[102,103],[103,104],[105,106],[107,108],[107,108],[108,109],[108,109],[108,109],[109,110],[109,110],[110,111],[110,111],[111,112],[111,112],[112,113],[112,113],[113,114],[113,114],[114,115],[114,115],[114,115],[116,121],[121,122],[123,125],[125,128],[128,130],[130,131],[132,135],[135,137],[137,138],[139,143],[144,147],[148,154],[155,156],[157,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,1,2,3,4,4,4,4,4,4,4,4,4,4,5,6,6,6,6,6,6,7,8,8,8,8,8,8,8,9,10,10,10,10,10,10,10,10,11,11,11,11,11,11,11,12,13,13,13,13,14,15,16,17,18,18,19,19,19,20,21,22,23,24,24,24,25,26,27,27,27,27,27,27,27,28,29,29,30,31,32,33,34,35,36,37,38,38,38,38,38,38,39,40,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,42,43,44,44,44,45,46,46,47,48,49,50,51,52,null],"decoded":"전화번호 : : 뉴질랜드 > 남섬 > 말버러 > 블레넘 가격대 ( 1박 ) : 117, 886 원 - 140, 244 원 4. 5성급 — lugano motor lodge 4. 5 * - 옵션 : - 트립어드바이저는, expedia, agoda, asia web direct boo","decoded_with_specials":"[CLS] 전화번호 : [UNK] : 뉴질랜드 > 남섬 > 말버러 > 블레넘 가격대 ( 1박 ) : 117, 886 원 - 140, 244 원 4. 5성급 — lugano motor lodge 4. 5 * - [UNK] 옵션 : - 트립어드바이저는 [UNK], expedia, agoda, asia web direct [UNK] boo [SEP]"} +{"kind":"sample","source":"fixtures/lang/rus_Cyrl.txt[:160]","text":"Покупая продукты в супермаркетах, сегодня уже мало кто верит в их качество. Как не сделать свое меню экстремальным и на что следует обращать внимание – читайте!","ids":[101,1194,14150,23925,29748,29746,10260,17432,1194,16856,14150,29742,29748,23925,22919,29113,1182,1196,29748,29746,15290,16856,29745,10260,16856,23925,15290,22919,10260,29750,1010,1196,15290,29741,14150,29742,18947,17432,1198,29743,15290,1191,10260,29436,14150,1189,22919,14150,1182,15290,16856,10325,22919,1182,1188,29750,1189,10260,29752,15290,29747,22919,25529,14150,1012,1189,10260,23925,1192,15290,1196,29742,15290,29436,10260,22919,23742,1196,25529,14150,15290,1191,15290,18947,29757,1208,23925,29747,22919,16856,15290,29745,10260,29436,23742,18947,29113,29745,1188,1192,10260,1202,22919,14150,1196,29436,15290,29742,29748,15290,22919,1193,29740,16856,10260,29754,10260,22919,23742,1182,18947,10325,29745,28995,10325,15290,1516,1202,10325,22919,10260,10325,22919,15290,999,102],"ids_no_specials":[1194,14150,23925,29748,29746,10260,17432,1194,16856,14150,29742,29748,23925,22919,29113,1182,1196,29748,29746,15290,16856,29745,10260,16856,23925,15290,22919,10260,29750,1010,1196,15290,29741,14150,29742,18947,17432,1198,29743,15290,1191,10260,29436,14150,1189,22919,14150,1182,15290,16856,10325,22919,1182,1188,29750,1189,10260,29752,15290,29747,22919,25529,14150,1012,1189,10260,23925,1192,15290,1196,29742,15290,29436,10260,22919,23742,1196,25529,14150,15290,1191,15290,18947,29757,1208,23925,29747,22919,16856,15290,29745,10260,29436,23742,18947,29113,29745,1188,1192,10260,1202,22919,14150,1196,29436,15290,29742,29748,15290,22919,1193,29740,16856,10260,29754,10260,22919,23742,1182,18947,10325,29745,28995,10325,15290,1516,1202,10325,22919,10260,10325,22919,15290,999],"tokens":["[CLS]","п","##о","##к","##у","##п","##а","##я","п","##р","##о","##д","##у","##к","##т","##ы","в","с","##у","##п","##е","##р","##м","##а","##р","##к","##е","##т","##а","##х",",","с","##е","##г","##о","##д","##н","##я","у","##ж","##е","м","##а","##л","##о","к","##т","##о","в","##е","##р","##и","##т","в","и","##х","к","##а","##ч","##е","##с","##т","##в","##о",".","к","##а","##к","н","##е","с","##д","##е","##л","##а","##т","##ь","с","##в","##о","##е","м","##е","##н","##ю","э","##к","##с","##т","##р","##е","##м","##а","##л","##ь","##н","##ы","##м","и","н","##а","ч","##т","##о","с","##л","##е","##д","##у","##е","##т","о","##б","##р","##а","##щ","##а","##т","##ь","в","##н","##и","##м","##ан","##и","##е","–","ч","##и","##т","##а","##и","##т","##е","!","[SEP]"],"offsets":[[0,0],[0,1],[1,2],[2,3],[3,4],[4,5],[5,6],[6,7],[8,9],[9,10],[10,11],[11,12],[12,13],[13,14],[14,15],[15,16],[17,18],[19,20],[20,21],[21,22],[22,23],[23,24],[24,25],[25,26],[26,27],[27,28],[28,29],[29,30],[30,31],[31,32],[32,33],[34,35],[35,36],[36,37],[37,38],[38,39],[39,40],[40,41],[42,43],[43,44],[44,45],[46,47],[47,48],[48,49],[49,50],[51,52],[52,53],[53,54],[55,56],[56,57],[57,58],[58,59],[59,60],[61,62],[63,64],[64,65],[66,67],[67,68],[68,69],[69,70],[70,71],[71,72],[72,73],[73,74],[74,75],[76,77],[77,78],[78,79],[80,81],[81,82],[83,84],[84,85],[85,86],[86,87],[87,88],[88,89],[89,90],[91,92],[92,93],[93,94],[94,95],[96,97],[97,98],[98,99],[99,100],[101,102],[102,103],[103,104],[104,105],[105,106],[106,107],[107,108],[108,109],[109,110],[110,111],[111,112],[112,113],[113,114],[115,116],[117,118],[118,119],[120,121],[121,122],[122,123],[124,125],[125,126],[126,127],[127,128],[128,129],[129,130],[130,131],[132,133],[133,134],[134,135],[135,136],[136,137],[137,138],[138,139],[139,140],[141,142],[142,143],[143,144],[144,145],[145,147],[147,148],[148,149],[150,151],[152,153],[153,154],[154,155],[155,156],[156,157],[157,158],[158,159],[159,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[null,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,2,3,3,3,3,3,3,3,3,3,3,3,3,3,4,5,5,5,5,5,5,5,6,6,6,7,7,7,7,8,8,8,9,9,9,9,9,10,11,11,12,12,12,12,12,12,12,12,13,14,14,14,15,15,16,16,16,16,16,16,16,17,17,17,17,18,18,18,18,19,19,19,19,19,19,19,19,19,19,19,19,19,20,21,21,22,22,22,23,23,23,23,23,23,23,24,24,24,24,24,24,24,24,25,25,25,25,25,25,25,26,27,27,27,27,27,27,27,28,null],"decoded":"покупая продукты в супермаркетах, сегодня уже мало кто верит в их качество. как не сделать свое меню экстремальным и на что следует обращать внимание – читаите!","decoded_with_specials":"[CLS] покупая продукты в супермаркетах, сегодня уже мало кто верит в их качество. как не сделать свое меню экстремальным и на что следует обращать внимание – читаите! [SEP]"} +{"kind":"sample","source":"fixtures/lang/tam_Taml.txt[:160]","text":"‘ஹலோ கலெக்டர் சாரா… சரக்கு வேணும்… கடை எப்ப திறப்பீங்க?’\nஒரு படத்தில் ஒயின்ஷாப்புக்குள் திருடப் போன வடிவேலு நன்றாக சரக்கடித்து விட்டு, ஒயின்ஷாப் ஓனருக்கு போன் ப","ids":[101,1520,100,100,1383,29931,29927,29931,1529,1383,29927,29918,29918,29933,100,1529,1382,29920,29935,100,100,1029,1521,100,1388,29920,29921,29921,29932,29928,100,1385,29932,29927,29933,29920,29924,1388,29934,29931,29923,1394,29920,29932,29930,29934,29928,29933,100,1383,29927,29918,29918,29920,29932,29921,29921,29933,1394,29932,29920,29920,29933,1010,100,100,1388,29934,29931,29923,1388,102],"ids_no_specials":[1520,100,100,1383,29931,29927,29931,1529,1383,29927,29918,29918,29933,100,1529,1382,29920,29935,100,100,1029,1521,100,1388,29920,29921,29921,29932,29928,100,1385,29932,29927,29933,29920,29924,1388,29934,29931,29923,1394,29920,29932,29930,29934,29928,29933,100,1383,29927,29918,29918,29920,29932,29921,29921,29933,1394,29932,29920,29920,29933,1010,100,100,1388,29934,29931,29923,1388],"tokens":["[CLS]","‘","[UNK]","[UNK]","ச","##ா","##ர","##ா","…","ச","##ர","##க","##க","##ு","[UNK]","…","க","##ட","##ை","[UNK]","[UNK]","?","’","[UNK]","ப","##ட","##த","##த","##ி","##ல","[UNK]","த","##ி","##ர","##ு","##ட","##ப","ப","##ே","##ா","##ன","வ","##ட","##ி","##வ","##ே","##ல","##ு","[UNK]","ச","##ர","##க","##க","##ட","##ி","##த","##த","##ு","வ","##ி","##ட","##ட","##ு",",","[UNK]","[UNK]","ப","##ே","##ா","##ன","ப","[SEP]"],"offsets":[[0,0],[0,1],[1,4],[5,12],[14,15],[15,16],[16,17],[17,18],[18,19],[20,21],[21,22],[22,23],[24,25],[25,26],[27,32],[33,34],[35,36],[36,37],[37,38],[39,43],[44,54],[54,55],[55,56],[57,60],[61,62],[62,63],[63,64],[65,66],[66,67],[67,68],[70,86],[88,89],[89,90],[90,91],[91,92],[92,93],[93,94],[96,97],[97,98],[97,98],[98,99],[100,101],[101,102],[102,103],[103,104],[104,105],[105,106],[106,107],[108,114],[115,116],[116,117],[117,118],[119,120],[120,121],[121,122],[122,123],[124,125],[125,126],[127,128],[128,129],[129,130],[131,132],[132,133],[133,134],[135,143],[145,153],[154,155],[155,156],[155,156],[156,157],[159,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[null,0,1,2,3,3,3,3,4,5,5,5,5,5,6,7,8,8,8,9,10,11,12,13,14,14,14,14,14,14,15,16,16,16,16,16,16,17,17,17,17,18,18,18,18,18,18,18,19,20,20,20,20,20,20,20,20,20,21,21,21,21,21,22,23,24,25,25,25,25,26,null],"decoded":"‘ சாரா … சரககு … கடை? ’ படததில திருடப போன வடிவேலு சரககடிதது விடடு, போன ப","decoded_with_specials":"[CLS] ‘ [UNK] [UNK] சாரா … சரககு [UNK] … கடை [UNK] [UNK]? ’ [UNK] படததில [UNK] திருடப போன வடிவேலு [UNK] சரககடிதது விடடு, [UNK] [UNK] போன ப [SEP]"} +{"kind":"sample","source":"fixtures/lang/tha_Thai.txt[:160]","text":"นับเจดีย์ดูนะครับว่าครบยี่สิบองค์รึเปล่า คำว่าซาว ทางเหนือแปลว่่ายี่สิบครับ ผมนับดูแล้วก็ครบนะ ลองดู เสร็จแล้วก็เข้าเยี่ยมชมพิพธภัณฑ์ เดินเข้าไปด้านในหน่อยก็จะม","ids":[101,100,100,100,100,100,100,100,102],"ids_no_specials":[100,100,100,100,100,100,100],"tokens":["[CLS]","[UNK]","[UNK]","[UNK]","[UNK]","[UNK]","[UNK]","[UNK]","[SEP]"],"offsets":[[0,0],[0,40],[41,49],[50,75],[76,94],[95,99],[101,132],[134,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,1],"word_ids":[null,0,1,2,3,4,5,6,null],"decoded":"","decoded_with_specials":"[CLS] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [SEP]"} +{"kind":"sample","source":"fixtures/modalities/added_normalized_dense.txt[:160]","text":"FLIBBERJAST CRUNGLEDORF FLIBBERJAST SNORLAXIAN fast BLORPTRONIC ZORPTASTIC split WIDGETRON split stage SNORLAXIAN BLORPTRONIC BLORPTRONIC \n QUIBBLENAUT CRUNGLED","ids":[101,13109,12322,5677,17386,2102,13675,5575,3709,16347,13109,12322,5677,17386,2102,1055,12131,2721,14787,2078,3435,1038,10626,13876,4948,2594,1062,2953,22799,10074,3975,15536,24291,4948,3975,2754,1055,12131,2721,14787,2078,1038,10626,13876,4948,2594,1038,10626,13876,4948,2594,21864,11362,24619,13675,5575,3709,102],"ids_no_specials":[13109,12322,5677,17386,2102,13675,5575,3709,16347,13109,12322,5677,17386,2102,1055,12131,2721,14787,2078,3435,1038,10626,13876,4948,2594,1062,2953,22799,10074,3975,15536,24291,4948,3975,2754,1055,12131,2721,14787,2078,1038,10626,13876,4948,2594,1038,10626,13876,4948,2594,21864,11362,24619,13675,5575,3709],"tokens":["[CLS]","fl","##ib","##ber","##jas","##t","cr","##ung","##led","##orf","fl","##ib","##ber","##jas","##t","s","##nor","##la","##xia","##n","fast","b","##lor","##pt","##ron","##ic","z","##or","##pta","##stic","split","wi","##dget","##ron","split","stage","s","##nor","##la","##xia","##n","b","##lor","##pt","##ron","##ic","b","##lor","##pt","##ron","##ic","qui","##bble","##naut","cr","##ung","##led","[SEP]"],"offsets":[[0,0],[0,2],[2,4],[4,7],[7,10],[10,11],[12,14],[14,17],[17,20],[20,23],[24,26],[26,28],[28,31],[31,34],[34,35],[36,37],[37,40],[40,42],[42,45],[45,46],[47,51],[52,53],[53,56],[56,58],[58,61],[61,63],[64,65],[65,67],[67,70],[70,74],[75,80],[81,83],[83,87],[87,90],[91,96],[97,102],[103,104],[104,107],[107,109],[109,112],[112,113],[114,115],[115,118],[118,120],[120,123],[123,125],[126,127],[127,130],[130,132],[132,135],[135,137],[140,143],[143,147],[147,151],[152,154],[154,157],[157,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[null,0,0,0,0,0,1,1,1,1,2,2,2,2,2,3,3,3,3,3,4,5,5,5,5,5,6,6,6,6,7,8,8,8,9,10,11,11,11,11,11,12,12,12,12,12,13,13,13,13,13,14,14,14,15,15,15,null],"decoded":"flibberjast crungledorf flibberjast snorlaxian fast blorptronic zorptastic split widgetron split stage snorlaxian blorptronic blorptronic quibblenaut crungled","decoded_with_specials":"[CLS] flibberjast crungledorf flibberjast snorlaxian fast blorptronic zorptastic split widgetron split stage snorlaxian blorptronic blorptronic quibblenaut crungled [SEP]"} +{"kind":"sample","source":"fixtures/modalities/added_normalized_sparse.txt[:160]","text":"ZORPTASTIC for we for decode CRUNGLEDORF here through ZORPTASTIC and the we WIDGETRON model \n normalize bytes and back flows text CRUNGLEDORF WUZZLEFANG chunk b","ids":[101,1062,2953,22799,10074,2005,2057,2005,21933,3207,13675,5575,3709,16347,2182,2083,1062,2953,22799,10074,1998,1996,2057,15536,24291,4948,2944,3671,4697,27507,1998,2067,6223,3793,13675,5575,3709,16347,8814,17644,15143,2290,20000,1038,102],"ids_no_specials":[1062,2953,22799,10074,2005,2057,2005,21933,3207,13675,5575,3709,16347,2182,2083,1062,2953,22799,10074,1998,1996,2057,15536,24291,4948,2944,3671,4697,27507,1998,2067,6223,3793,13675,5575,3709,16347,8814,17644,15143,2290,20000,1038],"tokens":["[CLS]","z","##or","##pta","##stic","for","we","for","deco","##de","cr","##ung","##led","##orf","here","through","z","##or","##pta","##stic","and","the","we","wi","##dget","##ron","model","normal","##ize","bytes","and","back","flows","text","cr","##ung","##led","##orf","wu","##zzle","##fan","##g","chunk","b","[SEP]"],"offsets":[[0,0],[0,1],[1,3],[3,6],[6,10],[11,14],[15,17],[18,21],[22,26],[26,28],[29,31],[31,34],[34,37],[37,40],[41,45],[46,53],[54,55],[55,57],[57,60],[60,64],[65,68],[69,72],[73,75],[76,78],[78,82],[82,85],[86,91],[94,100],[100,103],[104,109],[110,113],[114,118],[119,124],[125,129],[130,132],[132,135],[135,138],[138,141],[142,144],[144,148],[148,151],[151,152],[153,158],[159,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[null,0,0,0,0,1,2,3,4,4,5,5,5,5,6,7,8,8,8,8,9,10,11,12,12,12,13,14,14,15,16,17,18,19,20,20,20,20,21,21,21,21,22,23,null],"decoded":"zorptastic for we for decode crungledorf here through zorptastic and the we widgetron model normalize bytes and back flows text crungledorf wuzzlefang chunk b","decoded_with_specials":"[CLS] zorptastic for we for decode crungledorf here through zorptastic and the we widgetron model normalize bytes and back flows text crungledorf wuzzlefang chunk b [SEP]"} +{"kind":"sample","source":"fixtures/modalities/added_special_dense.txt[:160]","text":"fast <|xs2|> chunk <|xs1|> <|xs4|> <|xs3|> <|xs4|> modality language <|xs0|> reads <|xs2|> and and \n <|xs1|> <|xs0|> <|xs1|> <|xs4|> <|xs4|> <|xs1|> back and re","ids":[101,3435,1026,1064,1060,2015,2475,1064,1028,20000,1026,1064,1060,2015,2487,1064,1028,1026,1064,1060,2015,2549,1064,1028,1026,1064,1060,2015,2509,1064,1028,1026,1064,1060,2015,2549,1064,1028,16913,23732,2653,1026,1064,1060,2015,2692,1064,1028,9631,1026,1064,1060,2015,2475,1064,1028,1998,1998,1026,1064,1060,2015,2487,1064,1028,1026,1064,1060,2015,2692,1064,1028,1026,1064,1060,2015,2487,1064,1028,1026,1064,1060,2015,2549,1064,1028,1026,1064,1060,2015,2549,1064,1028,1026,1064,1060,2015,2487,1064,1028,2067,1998,2128,102],"ids_no_specials":[3435,1026,1064,1060,2015,2475,1064,1028,20000,1026,1064,1060,2015,2487,1064,1028,1026,1064,1060,2015,2549,1064,1028,1026,1064,1060,2015,2509,1064,1028,1026,1064,1060,2015,2549,1064,1028,16913,23732,2653,1026,1064,1060,2015,2692,1064,1028,9631,1026,1064,1060,2015,2475,1064,1028,1998,1998,1026,1064,1060,2015,2487,1064,1028,1026,1064,1060,2015,2692,1064,1028,1026,1064,1060,2015,2487,1064,1028,1026,1064,1060,2015,2549,1064,1028,1026,1064,1060,2015,2549,1064,1028,1026,1064,1060,2015,2487,1064,1028,2067,1998,2128],"tokens":["[CLS]","fast","<","|","x","##s","##2","|",">","chunk","<","|","x","##s","##1","|",">","<","|","x","##s","##4","|",">","<","|","x","##s","##3","|",">","<","|","x","##s","##4","|",">","mod","##ality","language","<","|","x","##s","##0","|",">","reads","<","|","x","##s","##2","|",">","and","and","<","|","x","##s","##1","|",">","<","|","x","##s","##0","|",">","<","|","x","##s","##1","|",">","<","|","x","##s","##4","|",">","<","|","x","##s","##4","|",">","<","|","x","##s","##1","|",">","back","and","re","[SEP]"],"offsets":[[0,0],[0,4],[5,6],[6,7],[7,8],[8,9],[9,10],[10,11],[11,12],[13,18],[19,20],[20,21],[21,22],[22,23],[23,24],[24,25],[25,26],[27,28],[28,29],[29,30],[30,31],[31,32],[32,33],[33,34],[35,36],[36,37],[37,38],[38,39],[39,40],[40,41],[41,42],[43,44],[44,45],[45,46],[46,47],[47,48],[48,49],[49,50],[51,54],[54,59],[60,68],[69,70],[70,71],[71,72],[72,73],[73,74],[74,75],[75,76],[77,82],[83,84],[84,85],[85,86],[86,87],[87,88],[88,89],[89,90],[91,94],[95,98],[101,102],[102,103],[103,104],[104,105],[105,106],[106,107],[107,108],[109,110],[110,111],[111,112],[112,113],[113,114],[114,115],[115,116],[117,118],[118,119],[119,120],[120,121],[121,122],[122,123],[123,124],[125,126],[126,127],[127,128],[128,129],[129,130],[130,131],[131,132],[133,134],[134,135],[135,136],[136,137],[137,138],[138,139],[139,140],[141,142],[142,143],[143,144],[144,145],[145,146],[146,147],[147,148],[149,153],[154,157],[158,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[null,0,1,2,3,3,3,4,5,6,7,8,9,9,9,10,11,12,13,14,14,14,15,16,17,18,19,19,19,20,21,22,23,24,24,24,25,26,27,27,28,29,30,31,31,31,32,33,34,35,36,37,37,37,38,39,40,41,42,43,44,44,44,45,46,47,48,49,49,49,50,51,52,53,54,54,54,55,56,57,58,59,59,59,60,61,62,63,64,64,64,65,66,67,68,69,69,69,70,71,72,73,74,null],"decoded":"fast < | xs2 | > chunk < | xs1 | > < | xs4 | > < | xs3 | > < | xs4 | > modality language < | xs0 | > reads < | xs2 | > and and < | xs1 | > < | xs0 | > < | xs1 | > < | xs4 | > < | xs4 | > < | xs1 | > back and re","decoded_with_specials":"[CLS] fast < | xs2 | > chunk < | xs1 | > < | xs4 | > < | xs3 | > < | xs4 | > modality language < | xs0 | > reads < | xs2 | > and and < | xs1 | > < | xs0 | > < | xs1 | > < | xs4 | > < | xs4 | > < | xs1 | > back and re [SEP]"} +{"kind":"sample","source":"fixtures/modalities/added_special_sparse.txt[:160]","text":"<|xs0|> every for encode <|xs0|> bytes the normalize decode text model <|xs4|> <|xs3|> and \n and <|xs3|> here <|xs1|> again model here text <|xs2|> <|xs2|> lang","ids":[101,1026,1064,1060,2015,2692,1064,1028,2296,2005,4372,16044,1026,1064,1060,2015,2692,1064,1028,27507,1996,3671,4697,21933,3207,3793,2944,1026,1064,1060,2015,2549,1064,1028,1026,1064,1060,2015,2509,1064,1028,1998,1998,1026,1064,1060,2015,2509,1064,1028,2182,1026,1064,1060,2015,2487,1064,1028,2153,2944,2182,3793,1026,1064,1060,2015,2475,1064,1028,1026,1064,1060,2015,2475,1064,1028,11374,102],"ids_no_specials":[1026,1064,1060,2015,2692,1064,1028,2296,2005,4372,16044,1026,1064,1060,2015,2692,1064,1028,27507,1996,3671,4697,21933,3207,3793,2944,1026,1064,1060,2015,2549,1064,1028,1026,1064,1060,2015,2509,1064,1028,1998,1998,1026,1064,1060,2015,2509,1064,1028,2182,1026,1064,1060,2015,2487,1064,1028,2153,2944,2182,3793,1026,1064,1060,2015,2475,1064,1028,1026,1064,1060,2015,2475,1064,1028,11374],"tokens":["[CLS]","<","|","x","##s","##0","|",">","every","for","en","##code","<","|","x","##s","##0","|",">","bytes","the","normal","##ize","deco","##de","text","model","<","|","x","##s","##4","|",">","<","|","x","##s","##3","|",">","and","and","<","|","x","##s","##3","|",">","here","<","|","x","##s","##1","|",">","again","model","here","text","<","|","x","##s","##2","|",">","<","|","x","##s","##2","|",">","lang","[SEP]"],"offsets":[[0,0],[0,1],[1,2],[2,3],[3,4],[4,5],[5,6],[6,7],[8,13],[14,17],[18,20],[20,24],[25,26],[26,27],[27,28],[28,29],[29,30],[30,31],[31,32],[33,38],[39,42],[43,49],[49,52],[53,57],[57,59],[60,64],[65,70],[71,72],[72,73],[73,74],[74,75],[75,76],[76,77],[77,78],[79,80],[80,81],[81,82],[82,83],[83,84],[84,85],[85,86],[87,90],[93,96],[97,98],[98,99],[99,100],[100,101],[101,102],[102,103],[103,104],[105,109],[110,111],[111,112],[112,113],[113,114],[114,115],[115,116],[116,117],[118,123],[124,129],[130,134],[135,139],[140,141],[141,142],[142,143],[143,144],[144,145],[145,146],[146,147],[148,149],[149,150],[150,151],[151,152],[152,153],[153,154],[154,155],[156,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[null,0,1,2,2,2,3,4,5,6,7,7,8,9,10,10,10,11,12,13,14,15,15,16,16,17,18,19,20,21,21,21,22,23,24,25,26,26,26,27,28,29,30,31,32,33,33,33,34,35,36,37,38,39,39,39,40,41,42,43,44,45,46,47,48,48,48,49,50,51,52,53,53,53,54,55,56,null],"decoded":"< | xs0 | > every for encode < | xs0 | > bytes the normalize decode text model < | xs4 | > < | xs3 | > and and < | xs3 | > here < | xs1 | > again model here text < | xs2 | > < | xs2 | > lang","decoded_with_specials":"[CLS] < | xs0 | > every for encode < | xs0 | > bytes the normalize decode text model < | xs4 | > < | xs3 | > and and < | xs3 | > here < | xs1 | > again model here text < | xs2 | > < | xs2 | > lang [SEP]"} +{"kind":"sample","source":"fixtures/modalities/agentic-traces.txt[:160]","text":"Analyze this codebase and summarize what it is and how it works.\nI need to explore the codebase structure by reading the main entry points and configuration fil","ids":[101,17908,2023,3642,15058,1998,7680,7849,4697,2054,2009,2003,1998,2129,2009,2573,1012,1045,2342,2000,8849,1996,3642,15058,3252,2011,3752,1996,2364,4443,2685,1998,9563,10882,2140,102],"ids_no_specials":[17908,2023,3642,15058,1998,7680,7849,4697,2054,2009,2003,1998,2129,2009,2573,1012,1045,2342,2000,8849,1996,3642,15058,3252,2011,3752,1996,2364,4443,2685,1998,9563,10882,2140],"tokens":["[CLS]","analyze","this","code","##base","and","sum","##mar","##ize","what","it","is","and","how","it","works",".","i","need","to","explore","the","code","##base","structure","by","reading","the","main","entry","points","and","configuration","fi","##l","[SEP]"],"offsets":[[0,0],[0,7],[8,12],[13,17],[17,21],[22,25],[26,29],[29,32],[32,35],[36,40],[41,43],[44,46],[47,50],[51,54],[55,57],[58,63],[63,64],[65,66],[67,71],[72,74],[75,82],[83,86],[87,91],[91,95],[96,105],[106,108],[109,116],[117,120],[121,125],[126,131],[132,138],[139,142],[143,156],[157,159],[159,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[null,0,1,2,2,3,4,4,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,18,19,20,21,22,23,24,25,26,27,28,28,null],"decoded":"analyze this codebase and summarize what it is and how it works. i need to explore the codebase structure by reading the main entry points and configuration fil","decoded_with_specials":"[CLS] analyze this codebase and summarize what it is and how it works. i need to explore the codebase structure by reading the main entry points and configuration fil [SEP]"} +{"kind":"sample","source":"fixtures/modalities/agentic_swe.txt[:160]","text":"[system]\nYou are a helpful assistant that can interact with a computer to solve tasks.\n\n[user]\n\n/testbed\n\nI've uploaded a pytho","ids":[101,1031,2291,1033,2017,2024,1037,14044,3353,2008,2064,11835,2007,1037,3274,2000,9611,8518,1012,1031,5310,1033,1026,21345,1035,6764,1028,1013,3231,8270,1026,1013,21345,1035,6764,1028,1045,1005,2310,21345,1037,1052,22123,6806,102],"ids_no_specials":[1031,2291,1033,2017,2024,1037,14044,3353,2008,2064,11835,2007,1037,3274,2000,9611,8518,1012,1031,5310,1033,1026,21345,1035,6764,1028,1013,3231,8270,1026,1013,21345,1035,6764,1028,1045,1005,2310,21345,1037,1052,22123,6806],"tokens":["[CLS]","[","system","]","you","are","a","helpful","assistant","that","can","interact","with","a","computer","to","solve","tasks",".","[","user","]","<","uploaded","_","files",">","/","test","##bed","<","/","uploaded","_","files",">","i","'","ve","uploaded","a","p","##yt","##ho","[SEP]"],"offsets":[[0,0],[0,1],[1,7],[7,8],[9,12],[13,16],[17,18],[19,26],[27,36],[37,41],[42,45],[46,54],[55,59],[60,61],[62,70],[71,73],[74,79],[80,85],[85,86],[88,89],[89,93],[93,94],[95,96],[96,104],[104,105],[105,110],[110,111],[112,113],[113,117],[117,120],[121,122],[122,123],[123,131],[131,132],[132,137],[137,138],[139,140],[140,141],[141,143],[144,152],[153,154],[155,156],[156,158],[158,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[null,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,27,28,29,30,31,32,33,34,35,36,37,38,39,39,39,null],"decoded":"[ system ] you are a helpful assistant that can interact with a computer to solve tasks. [ user ] < uploaded _ files > / testbed < / uploaded _ files > i ' ve uploaded a pytho","decoded_with_specials":"[CLS] [ system ] you are a helpful assistant that can interact with a computer to solve tasks. [ user ] < uploaded _ files > / testbed < / uploaded _ files > i ' ve uploaded a pytho [SEP]"} +{"kind":"sample","source":"fixtures/modalities/code_mixed.txt[:160]","text":"// django-f3f9601cff03b38942e70ce8042bdfdec6002449/django/__init__.py\nfrom django.utils.version import get_version\n\nVERSION = (6, 2, 0, \"alpha\", 0)\n\n__version__","ids":[101,1013,1013,6520,23422,1011,1042,2509,2546,2683,16086,2487,2278,4246,2692,2509,2497,22025,2683,20958,2063,19841,3401,17914,20958,2497,20952,3207,2278,16086,2692,18827,26224,1013,6520,23422,1013,1035,1035,1999,4183,1035,1035,1012,1052,2100,2013,6520,23422,1012,21183,12146,1012,2544,12324,2131,1035,2544,2544,1027,1006,1020,1010,1016,1010,1014,1010,1000,6541,1000,1010,1014,1007,1035,1035,2544,1035,1035,102],"ids_no_specials":[1013,1013,6520,23422,1011,1042,2509,2546,2683,16086,2487,2278,4246,2692,2509,2497,22025,2683,20958,2063,19841,3401,17914,20958,2497,20952,3207,2278,16086,2692,18827,26224,1013,6520,23422,1013,1035,1035,1999,4183,1035,1035,1012,1052,2100,2013,6520,23422,1012,21183,12146,1012,2544,12324,2131,1035,2544,2544,1027,1006,1020,1010,1016,1010,1014,1010,1000,6541,1000,1010,1014,1007,1035,1035,2544,1035,1035],"tokens":["[CLS]","/","/","dj","##ango","-","f","##3","##f","##9","##60","##1","##c","##ff","##0","##3","##b","##38","##9","##42","##e","##70","##ce","##80","##42","##b","##df","##de","##c","##60","##0","##24","##49","/","dj","##ango","/","_","_","in","##it","_","_",".","p","##y","from","dj","##ango",".","ut","##ils",".","version","import","get","_","version","version","=","(","6",",","2",",","0",",","\"","alpha","\"",",","0",")","_","_","version","_","_","[SEP]"],"offsets":[[0,0],[0,1],[1,2],[3,5],[5,9],[9,10],[10,11],[11,12],[12,13],[13,14],[14,16],[16,17],[17,18],[18,20],[20,21],[21,22],[22,23],[23,25],[25,26],[26,28],[28,29],[29,31],[31,33],[33,35],[35,37],[37,38],[38,40],[40,42],[42,43],[43,45],[45,46],[46,48],[48,50],[50,51],[51,53],[53,57],[57,58],[58,59],[59,60],[60,62],[62,64],[64,65],[65,66],[66,67],[67,68],[68,69],[70,74],[75,77],[77,81],[81,82],[82,84],[84,87],[87,88],[88,95],[96,102],[103,106],[106,107],[107,114],[116,123],[124,125],[126,127],[127,128],[128,129],[130,131],[131,132],[133,134],[134,135],[136,137],[137,142],[142,143],[143,144],[145,146],[146,147],[149,150],[150,151],[151,158],[158,159],[159,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[null,0,1,2,2,3,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,5,6,6,7,8,9,10,10,11,12,13,14,14,15,16,16,17,18,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,null],"decoded":"/ / django - f3f9601cff03b38942e70ce8042bdfdec6002449 / django / _ _ init _ _. py from django. utils. version import get _ version version = ( 6, 2, 0, \" alpha \", 0 ) _ _ version _ _","decoded_with_specials":"[CLS] / / django - f3f9601cff03b38942e70ce8042bdfdec6002449 / django / _ _ init _ _. py from django. utils. version import get _ version version = ( 6, 2, 0, \" alpha \", 0 ) _ _ version _ _ [SEP]"} +{"kind":"sample","source":"fixtures/modalities/math_latex.txt[:160]","text":"Bayes and his Theorem\n\nMy earlier post on Bayesian probability seems to have generated quite a lot of readers, so this lunchtime I thought I’d add a little bit ","ids":[101,3016,2229,1998,2010,9872,2026,3041,2695,2006,3016,25253,9723,3849,2000,2031,7013,3243,1037,2843,1997,8141,1010,2061,2023,6265,7292,1045,2245,1045,1521,1040,5587,1037,2210,2978,102],"ids_no_specials":[3016,2229,1998,2010,9872,2026,3041,2695,2006,3016,25253,9723,3849,2000,2031,7013,3243,1037,2843,1997,8141,1010,2061,2023,6265,7292,1045,2245,1045,1521,1040,5587,1037,2210,2978],"tokens":["[CLS]","bay","##es","and","his","theorem","my","earlier","post","on","bay","##esian","probability","seems","to","have","generated","quite","a","lot","of","readers",",","so","this","lunch","##time","i","thought","i","’","d","add","a","little","bit","[SEP]"],"offsets":[[0,0],[0,3],[3,5],[6,9],[10,13],[14,21],[23,25],[26,33],[34,38],[39,41],[42,45],[45,50],[51,62],[63,68],[69,71],[72,76],[77,86],[87,92],[93,94],[95,98],[99,101],[102,109],[109,110],[111,113],[114,118],[119,124],[124,128],[129,130],[131,138],[139,140],[140,141],[141,142],[143,146],[147,148],[149,155],[156,159],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[null,0,0,1,2,3,4,5,6,7,8,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,22,23,24,25,26,27,28,29,30,31,null],"decoded":"bayes and his theorem my earlier post on bayesian probability seems to have generated quite a lot of readers, so this lunchtime i thought i ’ d add a little bit","decoded_with_specials":"[CLS] bayes and his theorem my earlier post on bayesian probability seems to have generated quite a lot of readers, so this lunchtime i thought i ’ d add a little bit [SEP]"} +{"kind":"pair","text":"What is the capital of France?","pair":"Paris is the capital of France.","ids":[101,2054,2003,1996,3007,1997,2605,1029,102,3000,2003,1996,3007,1997,2605,1012,102],"tokens":["[CLS]","what","is","the","capital","of","france","?","[SEP]","paris","is","the","capital","of","france",".","[SEP]"],"type_ids":[0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1],"sequence_ids":[null,0,0,0,0,0,0,0,null,1,1,1,1,1,1,1,null],"special_tokens_mask":[1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1],"offsets":[[0,0],[0,4],[5,7],[8,11],[12,19],[20,22],[23,29],[29,30],[0,0],[0,5],[6,8],[9,12],[13,20],[21,23],[24,30],[30,31],[0,0]]} +{"kind":"pair","text":"Question in English?","pair":"Ответ на русском языке, с цифрами 123.","ids":[101,3160,1999,2394,1029,102,1193,22919,25529,15290,22919,1192,10260,1195,29748,29747,29747,23925,14150,29745,1210,29744,29113,23925,15290,1010,1196,1201,10325,29749,16856,10260,29745,10325,13138,1012,102],"tokens":["[CLS]","question","in","english","?","[SEP]","о","##т","##в","##е","##т","н","##а","р","##у","##с","##с","##к","##о","##м","я","##з","##ы","##к","##е",",","с","ц","##и","##ф","##р","##а","##м","##и","123",".","[SEP]"],"type_ids":[0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],"sequence_ids":[null,0,0,0,0,null,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,null],"special_tokens_mask":[1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"offsets":[[0,0],[0,8],[9,11],[12,19],[19,20],[0,0],[0,1],[1,2],[2,3],[3,4],[4,5],[6,7],[7,8],[9,10],[10,11],[11,12],[12,13],[13,14],[14,15],[15,16],[17,18],[18,19],[19,20],[20,21],[21,22],[22,23],[24,25],[26,27],[27,28],[28,29],[29,30],[30,31],[31,32],[32,33],[34,37],[37,38],[0,0]]} +{"kind":"digest","file":"fixtures/lang/amh_Ethi.txt","cap_chars":200000,"text_sha256":"e353d32f45b449b76fb65ee48f1522086c02a6bffa607e73ed8c875c03c84908","n_tokens":49697,"ids_sha256":"e82dd84f0e35688753a13c2a2a1cf2f0e6d6c848ca1678b269a8c56a5be784f4"} +{"kind":"digest","file":"fixtures/lang/arb_Arab.txt","cap_chars":200000,"text_sha256":"529587117db0a7abaa2c8f31ab05d7b2f77e84a11068b46e26b2b37774d836c7","n_tokens":159424,"ids_sha256":"124bc1fcb54855dbba167d412c29d2718221f1e65e3dde002d09989efd3a483b"} +{"kind":"digest","file":"fixtures/lang/ben_Beng.txt","cap_chars":200000,"text_sha256":"915afd7dbbea8256bdb6112b16424c83aa47dae63c5cca0ebadfc6c227d2b89f","n_tokens":142342,"ids_sha256":"2eb60f95c05bb82a4bdba5318c6166f9ac604d9d9a66084a4476f62647a2373c"} +{"kind":"digest","file":"fixtures/lang/cmn_Hani.txt","cap_chars":200000,"text_sha256":"1273c0c093a00739be9efea7804b10a295f902359d11663306d097211934c6df","n_tokens":181784,"ids_sha256":"74c025d02369e4eb2540ec0c7cf09b8dcf5a21a75d7110625bc83f03b26f196d"} +{"kind":"digest","file":"fixtures/lang/ell_Grek.txt","cap_chars":200000,"text_sha256":"158c8d3e4933091099bf12a1e21f57c7b7ff3f5381d4b25901eb1fa7ca0926f4","n_tokens":158345,"ids_sha256":"6addc3754b3f2eb0bbda5764cd48a1ed4feace79d663df203854f7f93bbb2652"} +{"kind":"digest","file":"fixtures/lang/eng_Latn.txt","cap_chars":200000,"text_sha256":"d998cb3153e3a56c2ecdf49293062e8bc6ca43365b6c91cfed147d73b84c19fb","n_tokens":44840,"ids_sha256":"8c0e902dff051c86b82ad31e20c46f1b1a016172f3056ee7f808905fe7758455"} +{"kind":"digest","file":"fixtures/lang/heb_Hebr.txt","cap_chars":200000,"text_sha256":"386da8c0e0d416bffbe84b51baf895c67dbce6e0fd153be597933261794300fd","n_tokens":161549,"ids_sha256":"38bfa7f98470ea5394e71b82559670fecb5ef821865ccdb64fc0998538682a4f"} +{"kind":"digest","file":"fixtures/lang/hin_Deva.txt","cap_chars":200000,"text_sha256":"2d114148c81035b1b63c94c7e9e805933c215d8aa9500d916923aec0a124ddde","n_tokens":116091,"ids_sha256":"028f1888d0d29e04309e10a37f7d78c88c0909ea675a7fc9d3972d2153587bd3"} +{"kind":"digest","file":"fixtures/lang/jpn_Jpan.txt","cap_chars":200000,"text_sha256":"2e55c036f500d7720d020be2db45c87a49c17d3b7e63b30bada9cef4cb6aa158","n_tokens":172849,"ids_sha256":"8f1e39de8ac94fcb359923ebb34ae83178256156436677ad19b10839394762fe"} +{"kind":"digest","file":"fixtures/lang/kat_Geor.txt","cap_chars":200000,"text_sha256":"c69a1aa8fb80b41459c98c15c756eb8f39a9151468b1d5b3f9bbffe4d82eb87a","n_tokens":79718,"ids_sha256":"8c3c941a56e1e9443527237cab92d5d09413b8e7a1eca554a32777c068a5493d"} +{"kind":"digest","file":"fixtures/lang/kor_Hang.txt","cap_chars":200000,"text_sha256":"68659781c288136c1caffcb10eec9130406ae8a7b003c1e327b7e3601eba041d","n_tokens":261063,"ids_sha256":"e322f50af838fd94cbaaf8e66b12edaff3fcbb69d8bb958e4f0359f2a415b003"} +{"kind":"digest","file":"fixtures/lang/rus_Cyrl.txt","cap_chars":200000,"text_sha256":"5f967a82627afbd72c2f21c6e56183dfa5efc986ba34350c1cf9eea6c27a6132","n_tokens":163862,"ids_sha256":"7885d7089831296bfc608588bb9c4415784a67a9ec9181fccd2ac3647974faa4"} +{"kind":"digest","file":"fixtures/lang/tam_Taml.txt","cap_chars":200000,"text_sha256":"bf09e1bca55aeab34162d5dc61afec7ac976b4e66eb5833ac5a037305911d165","n_tokens":68077,"ids_sha256":"b764ba1ff411931463efa13e43662199e075c306f4a39e93c6a8e21671c7d38c"} +{"kind":"digest","file":"fixtures/lang/tha_Thai.txt","cap_chars":200000,"text_sha256":"25209f4750b3d3f04440f98aeff50316a118cb982462a4763106e8747172b79c","n_tokens":13448,"ids_sha256":"1b6f019d369e3321eae366bbca31a6b0fd0a695d9f5bf023794562915f73293d"} +{"kind":"digest","file":"fixtures/modalities/added_normalized_dense.txt","cap_chars":200000,"text_sha256":"9d49b3cacf40962c051a09f1350a0f5dc2fc210556889ed0f341cfe0cf0af4f9","n_tokens":27161,"ids_sha256":"37bb0bf83d747153db72a533e78640e950211465c68f2aee2f59ada84a1d56b0"} +{"kind":"digest","file":"fixtures/modalities/added_normalized_sparse.txt","cap_chars":200000,"text_sha256":"55383f52b02a5d9686f9e4347729e5c08e61b9cff77913bd94ceb65c2fdf7fe4","n_tokens":21243,"ids_sha256":"3f8e1515735a65937d12314b8e085203a4a7d44a369b9208307ac986988433eb"} +{"kind":"digest","file":"fixtures/modalities/added_special_dense.txt","cap_chars":200000,"text_sha256":"472f90f6f2f5803626df5d7088f8ef12984de6e88fedf67723887511d35b64b2","n_tokens":59382,"ids_sha256":"467eac54894793c40465246a7597e8119b04505977a679ba888ac561f308bc88"} +{"kind":"digest","file":"fixtures/modalities/added_special_sparse.txt","cap_chars":200000,"text_sha256":"9c52602f7c8d009b11377743f95e5bfe9053694fc6b48c3dd6c1c6dc0681ab5a","n_tokens":33144,"ids_sha256":"e09ec9ee9c60e36a3dec3503a3b00eaf802605066371895ad004860f60887783"} +{"kind":"digest","file":"fixtures/modalities/agentic-traces.txt","cap_chars":200000,"text_sha256":"ae9d63c1adc49f572d9af635ac1b0421461e4d1b92c5f35ea572980ff1e2480c","n_tokens":61319,"ids_sha256":"c50b0d03cc597166dd0a235978d3a76fd41fddd0f9bd337fd07c899a85c265c0"} +{"kind":"digest","file":"fixtures/modalities/agentic_swe.txt","cap_chars":200000,"text_sha256":"b2d31e91ff91bb6c9a3aec3832d25acbecf9b58d5c0674e364d3b287182e7253","n_tokens":71916,"ids_sha256":"4006e406c5411a9544c3295748014afa7c39b032e966d6e729e7ce4d11c228c7"} +{"kind":"digest","file":"fixtures/modalities/code_mixed.txt","cap_chars":200000,"text_sha256":"0fa7314727726db056433d36bc1bda021803be7f1d150cae83e362a3ff41821d","n_tokens":77654,"ids_sha256":"8cb4c94f1a91fb29b860b4a6611cefaf62b2c13fcb5a2f64c52e18941d1f58a5"} +{"kind":"digest","file":"fixtures/modalities/math_latex.txt","cap_chars":200000,"text_sha256":"2d17be8704f1f7b4f2174a054c0b85d6d22039e00be8e4a487927d27f3852e90","n_tokens":54032,"ids_sha256":"b120c46d3230ee3a7dcab804454dab50907af5717bcc92cf18192fd18605f144"} diff --git a/bindings/python/tests/golden/goldens/deepseek-v4.jsonl b/bindings/python/tests/golden/goldens/deepseek-v4.jsonl new file mode 100644 index 000000000..5983e062b --- /dev/null +++ b/bindings/python/tests/golden/goldens/deepseek-v4.jsonl @@ -0,0 +1,64 @@ +{"kind":"meta","model":"deepseek-v4","tokenizer_file":"deepseek-v4.json","tokenizers_version":"0.23.1"} +{"kind":"sample","source":"edge:empty","text":"","ids":[],"ids_no_specials":[],"tokens":[],"offsets":[],"type_ids":[],"special_tokens_mask":[],"word_ids":[],"decoded":"","decoded_with_specials":""} +{"kind":"sample","source":"edge:spaces","text":" ","ids":[361],"ids_no_specials":[361],"tokens":["ĠĠĠ"],"offsets":[[0,3]],"type_ids":[0],"special_tokens_mask":[0],"word_ids":[0],"decoded":" ","decoded_with_specials":" "} +{"kind":"sample","source":"edge:hello","text":"Hello world","ids":[19923,2058],"ids_no_specials":[19923,2058],"tokens":["Hello","Ġworld"],"offsets":[[0,5],[5,11]],"type_ids":[0,0],"special_tokens_mask":[0,0],"word_ids":[0,1],"decoded":"Hello world","decoded_with_specials":"Hello world"} +{"kind":"sample","source":"edge:punctuation","text":"Hello, world!! How's it going? (fine; thanks...)","ids":[19923,14,2058,6909,1730,734,436,2887,33,343,86453,29,13482,87958],"ids_no_specials":[19923,14,2058,6909,1730,734,436,2887,33,343,86453,29,13482,87958],"tokens":["Hello",",","Ġworld","!!","ĠHow","'s","Ġit","Ġgoing","?","Ġ(","fine",";","Ġthanks","...)"],"offsets":[[0,5],[5,6],[6,12],[12,14],[14,18],[18,20],[20,23],[23,29],[29,30],[30,32],[32,36],[36,37],[37,44],[44,48]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,1,2,3,4,5,6,7,8,9,10,11,12,13],"decoded":"Hello, world!! How's it going? (fine; thanks...)","decoded_with_specials":"Hello, world!! How's it going? (fine; thanks...)"} +{"kind":"sample","source":"edge:whitespace-mix","text":"line one\nline two\r\n\tindented\n trailing ","ids":[1836,834,201,1836,1234,204,201,200,655,19686,201,223,52964,262],"ids_no_specials":[1836,834,201,1836,1234,204,201,200,655,19686,201,223,52964,262],"tokens":["line","Ġone","Ċ","line","Ġtwo","č","Ċ","ĉ","ind","ented","Ċ","Ġ","Ġtrailing","ĠĠ"],"offsets":[[0,4],[4,8],[8,9],[9,13],[13,17],[17,18],[18,19],[19,20],[20,23],[23,28],[28,29],[29,30],[30,39],[39,41]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,1,2,3,4,5,5,6,6,6,7,8,9,10],"decoded":"line one\nline two\r\n\tindented\n trailing ","decoded_with_specials":"line one\nline two\r\n\tindented\n trailing "} +{"kind":"sample","source":"edge:accents","text":"café naïve résumé — déjà vu","ids":[69,2797,619,112752,21538,90902,2136,54292,22695],"ids_no_specials":[69,2797,619,112752,21538,90902,2136,54292,22695],"tokens":["c","af","é","Ġnaïve","Ġrés","umé","ĠâĢĶ","ĠdéjÃł","Ġvu"],"offsets":[[0,1],[1,3],[3,4],[4,10],[10,14],[14,17],[17,19],[19,24],[24,27]],"type_ids":[0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,1,2,2,3,4,5],"decoded":"café naïve résumé — déjà vu","decoded_with_specials":"café naïve résumé — déjà vu"} +{"kind":"sample","source":"edge:accents-decomposed","text":"café naïve résumé","ids":[69,15702,17793,313,2238,66383,427,322,17793,85,3041,17793],"ids_no_specials":[69,15702,17793,313,2238,66383,427,322,17793,85,3041,17793],"tokens":["c","afe","Ìģ","Ġn","ai","ÌĪ","ve","Ġre","Ìģ","s","ume","Ìģ"],"offsets":[[0,1],[1,4],[4,5],[5,7],[7,9],[9,10],[10,12],[12,15],[15,16],[16,17],[17,20],[20,21]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,1,1,1,1,2,2,2,2,2],"decoded":"café naïve résumé","decoded_with_specials":"café naïve résumé"} +{"kind":"sample","source":"edge:emoji","text":"🤗 emoji, families 👨‍👩‍👧‍👦, flags 🇫🇷 and skin tones 👍🏽","ids":[62377,248,980,82644,14,8348,52780,104,46088,25730,105,46088,25730,103,46088,25730,102,14,30698,77579,107,43754,118,305,7147,40684,102527,40965,124],"ids_no_specials":[62377,248,980,82644,14,8348,52780,104,46088,25730,105,46088,25730,103,46088,25730,102,14,30698,77579,107,43754,118,305,7147,40684,102527,40965,124],"tokens":["ðŁ¤","Ĺ","Ġem","oji",",","Ġfamilies","ĠðŁij","¨","âĢį","ðŁij","©","âĢį","ðŁij","§","âĢį","ðŁij","¦",",","Ġflags","ĠðŁĩ","«","ðŁĩ","·","Ġand","Ġskin","Ġtones","ĠðŁijį","ðŁı","½"],"offsets":[[0,1],[0,1],[1,4],[4,7],[7,8],[8,17],[17,19],[18,19],[19,20],[20,21],[20,21],[21,22],[22,23],[22,23],[23,24],[24,25],[24,25],[25,26],[26,32],[32,34],[33,34],[34,35],[34,35],[35,39],[39,44],[44,50],[50,52],[52,53],[52,53]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,1,1,2,3,4,4,5,6,6,7,8,8,9,10,10,10,11,12,12,12,12,13,14,15,16,16,16],"decoded":"🤗 emoji, families 👨‍👩‍👧‍👦, flags 🇫🇷 and skin tones 👍🏽","decoded_with_specials":"🤗 emoji, families 👨‍👩‍👧‍👦, flags 🇫🇷 and skin tones 👍🏽"} +{"kind":"sample","source":"edge:cjk","text":"漢字とひらがなとカタカナが混ざった文章です。","ids":[29069,2024,2495,40259,4970,2936,2942,2495,15961,11767,15961,27071,2936,5764,51060,11033,7790,8262,320],"ids_no_specials":[29069,2024,2495,40259,4970,2936,2942,2495,15961,11767,15961,27071,2936,5764,51060,11033,7790,8262,320],"tokens":["æ¼¢","åŃĹ","ãģ¨","ãģ²","ãĤī","ãģĮ","ãģª","ãģ¨","ãĤ«","ãĤ¿","ãĤ«","ãĥĬ","ãģĮ","æ··","ãģĸ","ãģ£ãģŁ","æĸĩç«ł","ãģ§ãģĻ","ãĢĤ"],"offsets":[[0,1],[1,2],[2,3],[3,4],[4,5],[5,6],[6,7],[7,8],[8,9],[9,10],[10,11],[11,12],[12,13],[13,14],[14,15],[15,17],[17,19],[19,21],[21,22]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"decoded":"漢字とひらがなとカタカナが混ざった文章です。","decoded_with_specials":"漢字とひらがなとカタカナが混ざった文章です。"} +{"kind":"sample","source":"edge:korean","text":"한국어 텍스트 조각","ids":[5634,17564,9227,61291,238,45652,24837,32318],"ids_no_specials":[5634,17564,9227,61291,238,45652,24837,32318],"tokens":["íķľ","êµŃ","ìĸ´","Ġíħ","į","ìĬ¤íĬ¸","Ġì¡°","ê°ģ"],"offsets":[[0,1],[1,2],[2,3],[3,5],[4,5],[5,7],[7,9],[9,10]],"type_ids":[0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0],"word_ids":[0,0,0,1,1,1,2,2],"decoded":"한국어 텍스트 조각","decoded_with_specials":"한국어 텍스트 조각"} +{"kind":"sample","source":"edge:rtl","text":"مرحبا بالعالم — שלום עולם","ids":[10393,2212,53067,9254,1183,14059,2136,9500,16467,6166,41141],"ids_no_specials":[10393,2212,53067,9254,1183,14059,2136,9500,16467,6166,41141],"tokens":["Ùħر","ØŃ","با","ĠباÙĦ","ع","اÙĦÙħ","ĠâĢĶ","Ġש׾","×ķ×Ŀ","Ġ×¢","×ķ׾×Ŀ"],"offsets":[[0,2],[2,3],[3,5],[5,9],[9,10],[10,13],[13,15],[15,18],[18,20],[20,22],[22,25]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,1,1,1,2,3,3,4,4],"decoded":"مرحبا بالعالم — שלום עולם","decoded_with_specials":"مرحبا بالعالم — שלום עולם"} +{"kind":"sample","source":"edge:numbers","text":"1234567890, 3.14159, 1,000,000th","ids":[6895,18009,25744,18,14,223,21,16,9926,3318,14,223,19,14,1320,14,1320,463],"ids_no_specials":[6895,18009,25744,18,14,223,21,16,9926,3318,14,223,19,14,1320,14,1320,463],"tokens":["123","456","789","0",",","Ġ","3",".","141","59",",","Ġ","1",",","000",",","000","th"],"offsets":[[0,3],[3,6],[6,9],[9,10],[10,11],[11,12],[12,13],[13,14],[14,17],[17,19],[19,20],[20,21],[21,22],[22,23],[23,26],[26,27],[27,30],[30,32]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17],"decoded":"1234567890, 3.14159, 1,000,000th","decoded_with_specials":"1234567890, 3.14159, 1,000,000th"} +{"kind":"sample","source":"edge:code","text":"def f(x):\n return x**2 # squared\nprint(f'{f(3)=}')","ids":[3465,285,4042,3395,361,1354,1527,666,20,223,1823,11768,201,3098,5123,79523,72,10,21,6912,95,6528],"ids_no_specials":[3465,285,4042,3395,361,1354,1527,666,20,223,1823,11768,201,3098,5123,79523,72,10,21,6912,95,6528],"tokens":["def","Ġf","(x","):Ċ","ĠĠĠ","Ġreturn","Ġx","**","2","Ġ","Ġ#","Ġsquared","Ċ","print","(f","'{","f","(","3",")=","}","')"],"offsets":[[0,3],[3,5],[5,7],[7,10],[10,13],[13,20],[20,22],[22,24],[24,25],[25,26],[26,28],[28,36],[36,37],[37,42],[42,44],[44,46],[46,47],[47,48],[48,49],[49,51],[51,52],[52,54]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,19,19],"decoded":"def f(x):\n return x**2 # squared\nprint(f'{f(3)=}')","decoded_with_specials":"def f(x):\n return x**2 # squared\nprint(f'{f(3)=}')"} +{"kind":"sample","source":"edge:long-word","text":"Donaudampfschifffahrtsgesellschaftskapitänsmützenabzeichen","ids":[38,6880,519,2030,72,7753,394,617,13695,1648,4179,42477,4056,19091,8243,11352,85180,425,102623],"ids_no_specials":[38,6880,519,2030,72,7753,394,617,13695,1648,4179,42477,4056,19091,8243,11352,85180,425,102623],"tokens":["D","ona","ud","amp","f","sch","if","ff","ahr","ts","ges","ellschaft","sk","apit","än","sm","ützen","ab","zeichen"],"offsets":[[0,1],[1,4],[4,6],[6,9],[9,10],[10,13],[13,15],[15,17],[17,20],[20,22],[22,25],[25,34],[34,36],[36,40],[40,42],[42,44],[44,49],[49,51],[51,58]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":"Donaudampfschifffahrtsgesellschaftskapitänsmützenabzeichen","decoded_with_specials":"Donaudampfschifffahrtsgesellschaftskapitänsmützenabzeichen"} +{"kind":"sample","source":"edge:url-email","text":"https://example.com/a/b?q=1&r=2#frag user.name+tag@example.co.uk","ids":[5395,2272,30357,2193,20922,9928,33,83,31,19,8,84,31,20,5,72,3174,3967,15236,13,24164,34,30357,21592,23014],"ids_no_specials":[5395,2272,30357,2193,20922,9928,33,83,31,19,8,84,31,20,5,72,3174,3967,15236,13,24164,34,30357,21592,23014],"tokens":["https","://","example",".com","/a","/b","?","q","=","1","&","r","=","2","#","f","rag","Ġuser",".name","+","tag","@","example",".co",".uk"],"offsets":[[0,5],[5,8],[8,15],[15,19],[19,21],[21,23],[23,24],[24,25],[25,26],[26,27],[27,28],[28,29],[29,30],[30,31],[31,32],[32,33],[33,36],[36,41],[41,46],[46,47],[47,50],[50,51],[51,58],[58,61],[61,64]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,1,2,3,4,5,6,6,7,8,9,9,10,11,12,12,12,13,14,15,15,16,16,17,18],"decoded":"https://example.com/a/b?q=1&r=2#frag user.name+tag@example.co.uk","decoded_with_specials":"https://example.com/a/b?q=1&r=2#frag user.name+tag@example.co.uk"} +{"kind":"sample","source":"edge:unicode-spaces","text":"a b c　d","ids":[67,2162,68,98362,69,18524,70],"ids_no_specials":[67,2162,68,98362,69,18524,70],"tokens":["a","Âł","b","âĢī","c","ãĢĢ","d"],"offsets":[[0,1],[1,2],[2,3],[3,4],[4,5],[5,6],[6,7]],"type_ids":[0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0],"word_ids":[0,1,1,2,2,3,3],"decoded":"a b c　d","decoded_with_specials":"a b c　d"} +{"kind":"sample","source":"edge:math","text":"∑ᵢ xᵢ² ≤ ∫₀^∞ e⁻ˣ dx ≈ 1","ids":[25951,160,116,98,1527,160,116,98,1628,19212,71126,26558,225,64,24219,312,123520,138,99,27707,35015,223,19],"ids_no_specials":[25951,160,116,98,1527,160,116,98,1628,19212,71126,26558,225,64,24219,312,123520,138,99,27707,35015,223,19],"tokens":["âĪij","á","µ","¢","Ġx","á","µ","¢","²","Ġâī¤","ĠâĪ«","âĤ","Ģ","^","âĪŀ","Ġe","âģ»","Ë","£","Ġdx","ĠâīĪ","Ġ","1"],"offsets":[[0,1],[1,2],[1,2],[1,2],[2,4],[4,5],[4,5],[4,5],[5,6],[6,8],[8,10],[10,11],[10,11],[11,12],[12,13],[13,15],[15,16],[16,17],[16,17],[17,20],[20,22],[22,23],[23,24]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,1,1,1,2,2,2,2,3,4,5,6,6,7,7,8,9,10,10,11,12,13,14],"decoded":"∑ᵢ xᵢ² ≤ ∫₀^∞ e⁻ˣ dx ≈ 1","decoded_with_specials":"∑ᵢ xᵢ² ≤ ∫₀^∞ e⁻ˣ dx ≈ 1"} +{"kind":"sample","source":"fixtures/lang/amh_Ethi.txt[:160]","text":"- ፍርድ ቤቱ በአቶ ዮናታን ተስፋዬ ላይ የ6 አመት ከ6 ወር ጽኑ የእስር ቅጣት አስተላለፈ\n- አንድነት ዶክተር ቴድሮስ ለዓለም ጤና ድርጅት ዋና ዳይሬክተርነት በመመረጣቸው ደስታውን ገለጸ\n- ህንድ መጠነ ሰፊ የድንጋይ ከሰል ሃይል ጣብያዎች የመገንባት እ","ids":[15,8751,238,238,54585,258,88017,116,8751,234,100,83721,112,8751,234,257,88991,257,83721,117,8751,236,109,88991,244,83721,114,88991,246,8751,234,111,54585,116,160,238,236,88017,108,8751,233,236,88017,258,8751,236,104,24,8751,235,257,160,6950,83721,116,8751,235,104,24,8751,236,233,54585,258,8751,237,124,88991,242,8751,236,104,88991,101,54585,116,54585,258,8751,234,230,160,237,99,83721,116,8751,235,257,54585,116,83721,111,54585,236,54585,233,160,238,233,201,15,8751,235,257,88991,246,88017,116,88991,241,83721,116,8751,236,117,88991,258,83721,111,54585,258,8751,91720,88017,116,54585,109,54585,116,8751,233,233,88017,244,54585,233,54585,254,8751,237,100,88991,244,8751,236,116,54585,258,160,237,230,83721,116,8751,236,236,88991,244,8751,236,114,88017,258,54585,108,88991,258,83721,111,54585,258,88991,241,83721,116,8751,234,257,160,6950,160,6950,54585,104,160,237,99,83721,119,88017,238,8751,236,111,54585,116,83721,114,88017,238,88991,246,8751,237,233,54585,233,160,237,119,201,15,8751,233,230,88991,246,88017,116,8751,6950,160,237,257,88991,241,8751,233,111,160,238,235,8751,236,104,88017,116,88991,246,160,237,236,88017,258,8751,235,104,54585,111,54585,238,8751,233,228,88017,258,54585,238,8751,237,99,83721,101,88017,107,88017,239,83721,124,8751,236,104,160,6950,160,237,233,88991,246,83721,99,83721,116,8751,235,101],"ids_no_specials":[15,8751,238,238,54585,258,88017,116,8751,234,100,83721,112,8751,234,257,88991,257,83721,117,8751,236,109,88991,244,83721,114,88991,246,8751,234,111,54585,116,160,238,236,88017,108,8751,233,236,88017,258,8751,236,104,24,8751,235,257,160,6950,83721,116,8751,235,104,24,8751,236,233,54585,258,8751,237,124,88991,242,8751,236,104,88991,101,54585,116,54585,258,8751,234,230,160,237,99,83721,116,8751,235,257,54585,116,83721,111,54585,236,54585,233,160,238,233,201,15,8751,235,257,88991,246,88017,116,88991,241,83721,116,8751,236,117,88991,258,83721,111,54585,258,8751,91720,88017,116,54585,109,54585,116,8751,233,233,88017,244,54585,233,54585,254,8751,237,100,88991,244,8751,236,116,54585,258,160,237,230,83721,116,8751,236,236,88991,244,8751,236,114,88017,258,54585,108,88991,258,83721,111,54585,258,88991,241,83721,116,8751,234,257,160,6950,160,6950,54585,104,160,237,99,83721,119,88017,238,8751,236,111,54585,116,83721,114,88017,238,88991,246,8751,237,233,54585,233,160,237,119,201,15,8751,233,230,88991,246,88017,116,8751,6950,160,237,257,88991,241,8751,233,111,160,238,235,8751,236,104,88017,116,88991,246,160,237,236,88017,258,8751,235,104,54585,111,54585,238,8751,233,228,88017,258,54585,238,8751,237,99,83721,101,88017,107,88017,239,83721,124,8751,236,104,160,6950,160,237,233,88991,246,83721,99,83721,116,8751,235,101],"tokens":["-","Ġá","į","į","áĪ","Ń","áĭ","µ","Ġá","ī","¤","áī","±","Ġá","ī","ł","áĬ","ł","áī","¶","Ġá","ĭ","®","áĬ","ĵ","áī","³","áĬ","ķ","Ġá","ī","°","áĪ","µ","á","į","ĭ","áĭ","¬","Ġá","Ī","ĭ","áĭ","Ń","Ġá","ĭ","¨","6","Ġá","Ĭ","ł","á","Īĺ","áī","µ","Ġá","Ĭ","¨","6","Ġá","ĭ","Ī","áĪ","Ń","Ġá","Į","½","áĬ","ij","Ġá","ĭ","¨","áĬ","¥","áĪ","µ","áĪ","Ń","Ġá","ī","ħ","á","Į","£","áī","µ","Ġá","Ĭ","ł","áĪ","µ","áī","°","áĪ","ĭ","áĪ","Ī","á","į","Ī","Ċ","-","Ġá","Ĭ","ł","áĬ","ķ","áĭ","µ","áĬ","IJ","áī","µ","Ġá","ĭ","¶","áĬ","Ń","áī","°","áĪ","Ń","Ġá","ī´","áĭ","µ","áĪ","®","áĪ","µ","Ġá","Ī","Ī","áĭ","ĵ","áĪ","Ī","áĪ","Ŀ","Ġá","Į","¤","áĬ","ĵ","Ġá","ĭ","µ","áĪ","Ń","á","Į","ħ","áī","µ","Ġá","ĭ","ĭ","áĬ","ĵ","Ġá","ĭ","³","áĭ","Ń","áĪ","¬","áĬ","Ń","áī","°","áĪ","Ń","áĬ","IJ","áī","µ","Ġá","ī","ł","á","Īĺ","á","Īĺ","áĪ","¨","á","Į","£","áī","¸","áĭ","į","Ġá","ĭ","°","áĪ","µ","áī","³","áĭ","į","áĬ","ķ","Ġá","Į","Ī","áĪ","Ī","á","Į","¸","Ċ","-","Ġá","Ī","ħ","áĬ","ķ","áĭ","µ","Ġá","Īĺ","á","Į","ł","áĬ","IJ","Ġá","Ī","°","á","į","Ĭ","Ġá","ĭ","¨","áĭ","µ","áĬ","ķ","á","Į","ĭ","áĭ","Ń","Ġá","Ĭ","¨","áĪ","°","áĪ","į","Ġá","Ī","ĥ","áĭ","Ń","áĪ","į","Ġá","Į","£","áī","¥","áĭ","«","áĭ","İ","áī","½","Ġá","ĭ","¨","á","Īĺ","á","Į","Ī","áĬ","ķ","áī","£","áī","µ","Ġá","Ĭ","¥"],"offsets":[[0,1],[1,3],[2,3],[2,3],[3,4],[3,4],[4,5],[4,5],[5,7],[6,7],[6,7],[7,8],[7,8],[8,10],[9,10],[9,10],[10,11],[10,11],[11,12],[11,12],[12,14],[13,14],[13,14],[14,15],[14,15],[15,16],[15,16],[16,17],[16,17],[17,19],[18,19],[18,19],[19,20],[19,20],[20,21],[20,21],[20,21],[21,22],[21,22],[22,24],[23,24],[23,24],[24,25],[24,25],[25,27],[26,27],[26,27],[27,28],[28,30],[29,30],[29,30],[30,31],[30,31],[31,32],[31,32],[32,34],[33,34],[33,34],[34,35],[35,37],[36,37],[36,37],[37,38],[37,38],[38,40],[39,40],[39,40],[40,41],[40,41],[41,43],[42,43],[42,43],[43,44],[43,44],[44,45],[44,45],[45,46],[45,46],[46,48],[47,48],[47,48],[48,49],[48,49],[48,49],[49,50],[49,50],[50,52],[51,52],[51,52],[52,53],[52,53],[53,54],[53,54],[54,55],[54,55],[55,56],[55,56],[56,57],[56,57],[56,57],[57,58],[58,59],[59,61],[60,61],[60,61],[61,62],[61,62],[62,63],[62,63],[63,64],[63,64],[64,65],[64,65],[65,67],[66,67],[66,67],[67,68],[67,68],[68,69],[68,69],[69,70],[69,70],[70,72],[71,72],[72,73],[72,73],[73,74],[73,74],[74,75],[74,75],[75,77],[76,77],[76,77],[77,78],[77,78],[78,79],[78,79],[79,80],[79,80],[80,82],[81,82],[81,82],[82,83],[82,83],[83,85],[84,85],[84,85],[85,86],[85,86],[86,87],[86,87],[86,87],[87,88],[87,88],[88,90],[89,90],[89,90],[90,91],[90,91],[91,93],[92,93],[92,93],[93,94],[93,94],[94,95],[94,95],[95,96],[95,96],[96,97],[96,97],[97,98],[97,98],[98,99],[98,99],[99,100],[99,100],[100,102],[101,102],[101,102],[102,103],[102,103],[103,104],[103,104],[104,105],[104,105],[105,106],[105,106],[105,106],[106,107],[106,107],[107,108],[107,108],[108,110],[109,110],[109,110],[110,111],[110,111],[111,112],[111,112],[112,113],[112,113],[113,114],[113,114],[114,116],[115,116],[115,116],[116,117],[116,117],[117,118],[117,118],[117,118],[118,119],[119,120],[120,122],[121,122],[121,122],[122,123],[122,123],[123,124],[123,124],[124,126],[125,126],[126,127],[126,127],[126,127],[127,128],[127,128],[128,130],[129,130],[129,130],[130,131],[130,131],[130,131],[131,133],[132,133],[132,133],[133,134],[133,134],[134,135],[134,135],[135,136],[135,136],[135,136],[136,137],[136,137],[137,139],[138,139],[138,139],[139,140],[139,140],[140,141],[140,141],[141,143],[142,143],[142,143],[143,144],[143,144],[144,145],[144,145],[145,147],[146,147],[146,147],[147,148],[147,148],[148,149],[148,149],[149,150],[149,150],[150,151],[150,151],[151,153],[152,153],[152,153],[153,154],[153,154],[154,155],[154,155],[154,155],[155,156],[155,156],[156,157],[156,157],[157,158],[157,158],[158,160],[159,160],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,1,1,1,1,1,1,1,2,2,2,2,2,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,7,7,7,8,9,9,9,9,9,9,9,10,10,10,11,12,12,12,12,12,13,13,13,13,13,14,14,14,14,14,14,14,14,14,15,15,15,15,15,15,15,15,16,16,16,16,16,16,16,16,16,16,16,16,16,16,17,18,19,19,19,19,19,19,19,19,19,19,19,20,20,20,20,20,20,20,20,20,21,21,21,21,21,21,21,21,22,22,22,22,22,22,22,22,22,23,23,23,23,23,24,24,24,24,24,24,24,24,24,24,25,25,25,25,25,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,28,28,28,28,28,28,28,28,28,28,28,29,29,29,29,29,29,29,29,30,31,32,32,32,32,32,32,32,33,33,33,33,33,33,33,34,34,34,34,34,34,35,35,35,35,35,35,35,35,35,35,35,35,36,36,36,36,36,36,36,37,37,37,37,37,37,37,38,38,38,38,38,38,38,38,38,38,38,39,39,39,39,39,39,39,39,39,39,39,39,39,39,40,40,40],"decoded":"- ፍርድ ቤቱ በአቶ ዮናታን ተስፋዬ ላይ የ6 አመት ከ6 ወር ጽኑ የእስር ቅጣት አስተላለፈ\n- አንድነት ዶክተር ቴድሮስ ለዓለም ጤና ድርጅት ዋና ዳይሬክተርነት በመመረጣቸው ደስታውን ገለጸ\n- ህንድ መጠነ ሰፊ የድንጋይ ከሰል ሃይል ጣብያዎች የመገንባት እ","decoded_with_specials":"- ፍርድ ቤቱ በአቶ ዮናታን ተስፋዬ ላይ የ6 አመት ከ6 ወር ጽኑ የእስር ቅጣት አስተላለፈ\n- አንድነት ዶክተር ቴድሮስ ለዓለም ጤና ድርጅት ዋና ዳይሬክተርነት በመመረጣቸው ደስታውን ገለጸ\n- ህንድ መጠነ ሰፊ የድንጋይ ከሰል ሃይል ጣብያዎች የመገንባት እ"} +{"kind":"sample","source":"fixtures/lang/arb_Arab.txt[:160]","text":"[بالصور] ترقوميا : الموت يطفىء باقة ورد قيد التفتح !!\nالخليل – دوت كوم – من غسان عبد الحميد - لم يخطر في بال أهالي \"ترقوميا\" التي انخلع قلبها وهي تودع التراب جث","ids":[61,1145,27874,3769,63,19090,1563,6725,9371,1313,3017,18797,3781,37203,2277,22678,1372,44840,1371,8247,4833,7844,5435,20467,2212,4050,8567,1712,3087,43247,1256,2183,18797,97594,1256,3560,12929,37472,44058,8014,745,7844,565,24253,3781,3087,23712,3787,9254,2789,815,30708,582,7215,1563,6725,9371,4,14895,9050,28569,1183,111717,3141,52624,2026,4817,1183,124245,4727,4784,4581],"ids_no_specials":[61,1145,27874,3769,63,19090,1563,6725,9371,1313,3017,18797,3781,37203,2277,22678,1372,44840,1371,8247,4833,7844,5435,20467,2212,4050,8567,1712,3087,43247,1256,2183,18797,97594,1256,3560,12929,37472,44058,8014,745,7844,565,24253,3781,3087,23712,3787,9254,2789,815,30708,582,7215,1563,6725,9371,4,14895,9050,28569,1183,111717,3141,52624,2026,4817,1183,124245,4727,4784,4581],"tokens":["[","ب","اÙĦص","ÙĪر","]","Ġتر","ÙĤ","ÙĪÙħ","ÙĬا","Ġ:","ĠاÙĦÙħ","ÙĪت","ĠÙĬ","Ø·Ùģ","Ùī","Ø¡","Ġب","اÙĤØ©","ĠÙĪ","رد","ĠÙĤ","ÙĬد","ĠاÙĦت","Ùģت","ØŃ","Ġ!","!Ċ","اÙĦ","Ø®","ÙĦÙĬÙĦ","ĠâĢĵ","Ġد","ÙĪت","ĠÙĥÙĪÙħ","ĠâĢĵ","ĠÙħÙĨ","Ġغ","ساÙĨ","Ġعبد","ĠاÙĦØŃ","Ùħ","ÙĬد","Ġ-","ĠÙĦÙħ","ĠÙĬ","Ø®","طر","ĠÙģÙĬ","ĠباÙĦ","ĠØ£","Ùĩ","اÙĦÙĬ","Ġ\"","تر","ÙĤ","ÙĪÙħ","ÙĬا","\"","ĠاÙĦتÙĬ","ĠاÙĨ","Ø®ÙĦ","ع","ĠÙĤÙĦب","Ùĩا","ĠÙĪÙĩÙĬ","Ġت","ÙĪد","ع","ĠاÙĦتر","اب","Ġج","Ø«"],"offsets":[[0,1],[1,2],[2,5],[5,7],[7,8],[8,11],[11,12],[12,14],[14,16],[16,18],[18,22],[22,24],[24,26],[26,28],[28,29],[29,30],[30,32],[32,35],[35,37],[37,39],[39,41],[41,43],[43,47],[47,49],[49,50],[50,52],[52,54],[54,56],[56,57],[57,60],[60,62],[62,64],[64,66],[66,70],[70,72],[72,75],[75,77],[77,80],[80,84],[84,88],[88,89],[89,91],[91,93],[93,96],[96,98],[98,99],[99,101],[101,104],[104,108],[108,110],[110,111],[111,114],[114,116],[116,118],[118,119],[119,121],[121,123],[123,124],[124,129],[129,132],[132,134],[134,135],[135,139],[139,141],[141,145],[145,147],[147,149],[149,150],[150,155],[155,157],[157,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,1,1,1,2,3,3,3,3,4,5,5,6,6,6,6,7,7,8,8,9,9,10,10,10,11,11,12,12,12,13,14,14,15,16,17,18,18,19,20,20,20,21,22,23,23,23,24,25,26,26,26,27,28,28,28,28,29,30,31,31,31,32,32,33,34,34,34,35,35,36,36],"decoded":"[بالصور] ترقوميا : الموت يطفىء باقة ورد قيد التفتح !!\nالخليل – دوت كوم – من غسان عبد الحميد - لم يخطر في بال أهالي \"ترقوميا\" التي انخلع قلبها وهي تودع التراب جث","decoded_with_specials":"[بالصور] ترقوميا : الموت يطفىء باقة ورد قيد التفتح !!\nالخليل – دوت كوم – من غسان عبد الحميد - لم يخطر في بال أهالي \"ترقوميا\" التي انخلع قلبها وهي تودع التراب جث"} +{"kind":"sample","source":"fixtures/lang/ben_Beng.txt[:160]","text":"গ্রহ নীহারিকা\nগ্রহ নীহারিকা (ইংরেজি ভাষায়: Planetary nebula) এক বিশেষ ধরনের গ্যাসীয় নীহারিকা। যেসব তারার ভর কম, নির্দিষ্টভাবে বলতে গেলে যেসব তারার ভর সূর্যের ","ids":[65609,8154,6404,55629,88073,201,65609,8154,6404,55629,88073,343,12042,119526,1794,85291,19683,28,103671,69279,4171,11,17859,95417,97268,36429,13980,93178,44697,8154,6404,55629,88073,4361,12117,50882,5205,27270,4384,16241,1442,58683,14,93640,102,86662,58386,34994,18262,13980,100509,12117,50882,5205,27270,4384,16241,1442,4783,58184,77214,223],"ids_no_specials":[65609,8154,6404,55629,88073,201,65609,8154,6404,55629,88073,343,12042,119526,1794,85291,19683,28,103671,69279,4171,11,17859,95417,97268,36429,13980,93178,44697,8154,6404,55629,88073,4361,12117,50882,5205,27270,4384,16241,1442,58683,14,93640,102,86662,58386,34994,18262,13980,100509,12117,50882,5205,27270,4384,16241,1442,4783,58184,77214,223],"tokens":["à¦Ĺà§įরহ","Ġন","à§Ģ","হার","িà¦ķা","Ċ","à¦Ĺà§įরহ","Ġন","à§Ģ","হার","িà¦ķা","Ġ(","à¦ĩ","à¦Ĥরà§ĩà¦ľ","ি","Ġà¦Ńাষ","ায়",":","ĠPlanetary","Ġneb","ula",")","Ġà¦ıà¦ķ","Ġবিশà§ĩষ","Ġধর","নà§ĩর","Ġà¦Ĺ","à§įযাস","à§Ģয়","Ġন","à§Ģ","হার","িà¦ķা","।","Ġয","à§ĩস","ব","Ġতার","ার","Ġà¦Ń","র","Ġà¦ķম",",","Ġনিরà§įà¦","¦","িষà§įà¦Ł","à¦Ńাবà§ĩ","Ġবল","তà§ĩ","Ġà¦Ĺ","à§ĩলà§ĩ","Ġয","à§ĩস","ব","Ġতার","ার","Ġà¦Ń","র","Ġস","à§Ĥর","à§įযà§ĩর","Ġ"],"offsets":[[0,4],[4,6],[6,7],[7,10],[10,13],[13,14],[14,18],[18,20],[20,21],[21,24],[24,27],[27,29],[29,30],[30,34],[34,35],[35,39],[39,42],[42,43],[43,53],[53,57],[57,60],[60,61],[61,64],[64,70],[70,73],[73,76],[76,78],[78,82],[82,85],[85,87],[87,88],[88,91],[91,94],[94,95],[95,97],[97,99],[99,100],[100,104],[104,106],[106,108],[108,109],[109,112],[112,113],[113,119],[118,119],[119,123],[123,127],[127,130],[130,132],[132,134],[134,137],[137,139],[139,141],[141,142],[142,146],[146,148],[148,150],[150,151],[151,153],[153,155],[155,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,1,1,1,1,2,3,4,4,4,4,5,6,6,6,7,7,8,9,10,10,11,12,13,14,14,15,15,15,16,16,16,16,17,18,18,18,19,19,20,20,21,22,23,23,23,23,24,24,25,25,26,26,26,27,27,28,28,29,29,29,30],"decoded":"গ্রহ নীহারিকা\nগ্রহ নীহারিকা (ইংরেজি ভাষায়: Planetary nebula) এক বিশেষ ধরনের গ্যাসীয় নীহারিকা। যেসব তারার ভর কম, নির্দিষ্টভাবে বলতে গেলে যেসব তারার ভর সূর্যের ","decoded_with_specials":"গ্রহ নীহারিকা\nগ্রহ নীহারিকা (ইংরেজি ভাষায়: Planetary nebula) এক বিশেষ ধরনের গ্যাসীয় নীহারিকা। যেসব তারার ভর কম, নির্দিষ্টভাবে বলতে গেলে যেসব তারার ভর সূর্যের "} +{"kind":"sample","source":"fixtures/lang/cmn_Hani.txt[:160]","text":"強力建議大家未來盡量避免英航阿~~~\n班機原訂航程\n6/9 台北→香港\n6/9 香港→倫敦 (原訂23:15起飛，4:50am抵達)\n6/10 倫敦→斯德哥爾摩 (原訂7:40am起飛，11:05am抵達)\n就在第二段，英航no.25班機在香港機場兩度離開閘口又兩度返航\n第一次因有旅客身體不適 (好，不怪他，消耗時間也","ids":[10336,910,47905,3627,48512,27120,1121,9209,3218,6372,4487,30125,3677,7521,1532,44428,6372,1251,201,24,17,27,223,43480,8674,13212,201,24,17,27,223,13212,8674,50769,20778,343,1532,44428,1349,28,856,1078,17019,303,22,28,1328,356,8887,14693,682,24,17,553,223,50769,20778,8674,2683,2773,4900,20117,9084,343,1532,44428,25,28,1484,356,1078,17019,303,779,28,2642,356,8887,14693,682,9164,3854,2484,303,3218,6372,3567,16,1069,3677,7521,88507,117298,8053,1029,34084,2388,249,1573,1680,8053,1029,7868,6372,201,11010,1139,450,37929,34974,422,19256,343,853,303,422,6691,626,303,19530,11554,728],"ids_no_specials":[10336,910,47905,3627,48512,27120,1121,9209,3218,6372,4487,30125,3677,7521,1532,44428,6372,1251,201,24,17,27,223,43480,8674,13212,201,24,17,27,223,13212,8674,50769,20778,343,1532,44428,1349,28,856,1078,17019,303,22,28,1328,356,8887,14693,682,24,17,553,223,50769,20778,8674,2683,2773,4900,20117,9084,343,1532,44428,25,28,1484,356,1078,17019,303,779,28,2642,356,8887,14693,682,9164,3854,2484,303,3218,6372,3567,16,1069,3677,7521,88507,117298,8053,1029,34084,2388,249,1573,1680,8053,1029,7868,6372,201,11010,1139,450,37929,34974,422,19256,343,853,303,422,6691,626,303,19530,11554,728],"tokens":["å¼·","åĬĽ","建èŃ°","大家","æľªä¾Ĩ","çĽ¡","éĩı","éģ¿åħį","èĭ±","èĪª","éĺ¿","~~~Ċ","çıŃ","æ©Ł","åİŁ","è¨Ĥ","èĪª","ç¨ĭ","Ċ","6","/","9","Ġ","åı°åĮĹ","âĨĴ","é¦Ļ港","Ċ","6","/","9","Ġ","é¦Ļ港","âĨĴ","åĢ«","æķ¦","Ġ(","åİŁ","è¨Ĥ","23",":","15","èµ·","é£Ľ","ï¼Į","4",":","50","am","æĬµ","éģĶ",")Ċ","6","/","10","Ġ","åĢ«","æķ¦","âĨĴ","æĸ¯","å¾·","åĵ¥","çĪ¾","æij©","Ġ(","åİŁ","è¨Ĥ","7",":","40","am","èµ·","é£Ľ","ï¼Į","11",":","05","am","æĬµ","éģĶ",")Ċ","å°±åľ¨","第äºĮ","段","ï¼Į","èĭ±","èĪª","no",".","25","çıŃ","æ©Ł","åľ¨é¦Ļ港","æ©Łåł´","åħ©","度","éĽ¢éĸĭ","éĸ","ĺ","åı£","åıĪ","åħ©","度","è¿Ķ","èĪª","Ċ","第ä¸Ģ次","åĽł","æľī","æĹħ客","身é«Ķ","ä¸į","éģ©","Ġ(","好","ï¼Į","ä¸į","æĢª","ä»ĸ","ï¼Į","æ¶ĪèĢĹ","æĻĤéĸĵ","ä¹Ł"],"offsets":[[0,1],[1,2],[2,4],[4,6],[6,8],[8,9],[9,10],[10,12],[12,13],[13,14],[14,15],[15,19],[19,20],[20,21],[21,22],[22,23],[23,24],[24,25],[25,26],[26,27],[27,28],[28,29],[29,30],[30,32],[32,33],[33,35],[35,36],[36,37],[37,38],[38,39],[39,40],[40,42],[42,43],[43,44],[44,45],[45,47],[47,48],[48,49],[49,51],[51,52],[52,54],[54,55],[55,56],[56,57],[57,58],[58,59],[59,61],[61,63],[63,64],[64,65],[65,67],[67,68],[68,69],[69,71],[71,72],[72,73],[73,74],[74,75],[75,76],[76,77],[77,78],[78,79],[79,80],[80,82],[82,83],[83,84],[84,85],[85,86],[86,88],[88,90],[90,91],[91,92],[92,93],[93,95],[95,96],[96,98],[98,100],[100,101],[101,102],[102,104],[104,106],[106,108],[108,109],[109,110],[110,111],[111,112],[112,114],[114,115],[115,117],[117,118],[118,119],[119,122],[122,124],[124,125],[125,126],[126,128],[128,129],[128,129],[129,130],[130,131],[131,132],[132,133],[133,134],[134,135],[135,136],[136,139],[139,140],[140,141],[141,143],[143,145],[145,146],[146,147],[147,149],[149,150],[150,151],[151,152],[152,153],[153,154],[154,155],[155,157],[157,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,0,0,0,0,1,2,2,2,2,2,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,18,19,20,20,21,22,23,24,24,25,26,27,28,29,30,30,31,32,33,34,35,36,36,37,38,38,38,38,38,39,40,40,41,42,43,44,45,45,46,47,48,49,50,51,51,52,53,53,53,54,55,55,56,57,58,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,60,61,61,61,61,61,61,61,62,63,64,65,65,65,66,67,67,67],"decoded":"強力建議大家未來盡量避免英航阿~~~\n班機原訂航程\n6/9 台北→香港\n6/9 香港→倫敦 (原訂23:15起飛，4:50am抵達)\n6/10 倫敦→斯德哥爾摩 (原訂7:40am起飛，11:05am抵達)\n就在第二段，英航no.25班機在香港機場兩度離開閘口又兩度返航\n第一次因有旅客身體不適 (好，不怪他，消耗時間也","decoded_with_specials":"強力建議大家未來盡量避免英航阿~~~\n班機原訂航程\n6/9 台北→香港\n6/9 香港→倫敦 (原訂23:15起飛，4:50am抵達)\n6/10 倫敦→斯德哥爾摩 (原訂7:40am起飛，11:05am抵達)\n就在第二段，英航no.25班機在香港機場兩度離開閘口又兩度返航\n第一次因有旅客身體不適 (好，不怪他，消耗時間也"} +{"kind":"sample","source":"fixtures/lang/ell_Grek.txt[:160]","text":"Θυμήσου με\nΗ πόλη προσφέρει πολλές ευκαιρίες - πήγαινε στο εμπορικό κέντρο, στο σαλόνι ομορφιάς , σε κλάμπ και διασκέδασε με την ψυχή σου.\nΔημιούργησε μαγευτικά","ids":[67718,3484,88905,2781,5700,21283,201,52228,97481,25696,107030,7482,63193,10976,40119,19624,23769,7972,3484,34576,2106,21472,24171,565,7014,7015,68865,31790,2520,35164,7972,3936,14226,2649,16736,8055,6711,27230,13343,14,35164,6931,1753,3422,63604,2106,15107,14597,2649,7482,49494,3299,1537,37213,8055,60325,40028,16700,52175,96678,6711,44822,30632,21283,29472,52871,3484,83330,6931,5700,603,17805,78041,2106,21996,38390,3511,30632,86049,5591,99621,95633],"ids_no_specials":[67718,3484,88905,2781,5700,21283,201,52228,97481,25696,107030,7482,63193,10976,40119,19624,23769,7972,3484,34576,2106,21472,24171,565,7014,7015,68865,31790,2520,35164,7972,3936,14226,2649,16736,8055,6711,27230,13343,14,35164,6931,1753,3422,63604,2106,15107,14597,2649,7482,49494,3299,1537,37213,8055,60325,40028,16700,52175,96678,6711,44822,30632,21283,29472,52871,3484,83330,6931,5700,603,17805,78041,2106,21996,38390,3511,30632,86049,5591,99621,95633],"tokens":["Îĺ","Ïħ","μή","Ïĥ","οÏħ","Ġμε","Ċ","ÎĹ","ĠÏĢÏĮ","λη","ĠÏĢÏģοÏĥ","ÏĨ","ÎŃÏģ","ει","ĠÏĢο","λλ","ÎŃÏĤ","Ġε","Ïħ","κα","ι","Ïģί","εÏĤ","Ġ-","ĠÏĢ","ή","γα","ιν","ε","ĠÏĥÏĦο","Ġε","μ","ÏĢο","Ïģ","ικÏĮ","Ġκ","ÎŃ","νÏĦ","Ïģο",",","ĠÏĥÏĦο","ĠÏĥ","α","λ","ÏĮν","ι","Ġο","μο","Ïģ","ÏĨ","ιά","ÏĤ","Ġ,","ĠÏĥε","Ġκ","λά","μÏĢ","Ġκαι","Ġδια","Ïĥκ","ÎŃ","δα","Ïĥε","Ġμε","ĠÏĦην","ĠÏĪ","Ïħ","Ïĩή","ĠÏĥ","οÏħ",".Ċ","ÎĶ","ημ","ι","οÏį","Ïģγ","η","Ïĥε","Ġμα","γ","εÏħ","ÏĦικά"],"offsets":[[0,1],[1,2],[2,4],[4,5],[5,7],[7,10],[10,11],[11,12],[12,15],[15,17],[17,22],[22,23],[23,25],[25,27],[27,30],[30,32],[32,34],[34,36],[36,37],[37,39],[39,40],[40,42],[42,44],[44,46],[46,48],[48,49],[49,51],[51,53],[53,54],[54,58],[58,60],[60,61],[61,63],[63,64],[64,67],[67,69],[69,70],[70,72],[72,74],[74,75],[75,79],[79,81],[81,82],[82,83],[83,85],[85,86],[86,88],[88,90],[90,91],[91,92],[92,94],[94,95],[95,97],[97,100],[100,102],[102,104],[104,106],[106,110],[110,114],[114,116],[116,117],[117,119],[119,121],[121,124],[124,128],[128,130],[130,131],[131,133],[133,135],[135,137],[137,139],[139,140],[140,142],[142,143],[143,145],[145,147],[147,148],[148,150],[150,153],[153,154],[154,156],[156,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,1,2,3,4,4,5,5,5,5,6,6,6,7,7,7,7,7,7,8,9,9,9,9,9,10,11,11,11,11,11,12,12,12,12,13,14,15,15,15,15,15,16,16,16,16,16,16,17,18,19,19,19,20,21,21,21,21,21,22,23,24,24,24,25,25,26,27,27,27,27,27,27,27,28,28,28,28],"decoded":"Θυμήσου με\nΗ πόλη προσφέρει πολλές ευκαιρίες - πήγαινε στο εμπορικό κέντρο, στο σαλόνι ομορφιάς , σε κλάμπ και διασκέδασε με την ψυχή σου.\nΔημιούργησε μαγευτικά","decoded_with_specials":"Θυμήσου με\nΗ πόλη προσφέρει πολλές ευκαιρίες - πήγαινε στο εμπορικό κέντρο, στο σαλόνι ομορφιάς , σε κλάμπ και διασκέδασε με την ψυχή σου.\nΔημιούργησε μαγευτικά"} +{"kind":"sample","source":"fixtures/lang/eng_Latn.txt[:160]","text":"|Viewing Single Post From: Spoilers for the Week of February 11th|\n|Lil||Feb 1 2013, 09:58 AM|\nDon't care about Chloe/Taniel/Jen-Jen. Don't care about Sami, rea","ids":[94,4149,288,24266,5245,5795,28,117010,54934,362,270,14159,294,6396,223,779,463,22301,94,46,321,18826,21924,223,19,223,643,21,14,223,3130,28,3175,7167,22301,13222,1664,2312,943,84996,15058,276,1434,37250,267,13471,267,16,7740,1664,2312,943,7789,75,14,322,67],"ids_no_specials":[94,4149,288,24266,5245,5795,28,117010,54934,362,270,14159,294,6396,223,779,463,22301,94,46,321,18826,21924,223,19,223,643,21,14,223,3130,28,3175,7167,22301,13222,1664,2312,943,84996,15058,276,1434,37250,267,13471,267,16,7740,1664,2312,943,7789,75,14,322,67],"tokens":["|","View","ing","ĠSingle","ĠPost","ĠFrom",":","ĠSpo","ilers","Ġfor","Ġthe","ĠWeek","Ġof","ĠFebruary","Ġ","11","th","|Ċ","|","L","il","||","Feb","Ġ","1","Ġ","201","3",",","Ġ","09",":","58","ĠAM","|Ċ","Don","'t","Ġcare","Ġabout","ĠChloe","/T","an","iel","/J","en","-J","en",".","ĠDon","'t","Ġcare","Ġabout","ĠSam","i",",","Ġre","a"],"offsets":[[0,1],[1,5],[5,8],[8,15],[15,20],[20,25],[25,26],[26,30],[30,35],[35,39],[39,43],[43,48],[48,51],[51,60],[60,61],[61,63],[63,65],[65,67],[67,68],[68,69],[69,71],[71,73],[73,76],[76,77],[77,78],[78,79],[79,82],[82,83],[83,84],[84,85],[85,87],[87,88],[88,90],[90,93],[93,95],[95,98],[98,100],[100,105],[105,111],[111,117],[117,119],[119,121],[121,124],[124,126],[126,128],[128,130],[130,132],[132,133],[133,137],[137,139],[139,144],[144,150],[150,154],[154,155],[155,156],[156,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,1,2,3,4,5,5,6,7,8,9,10,11,12,13,14,15,15,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,35,35,36,36,37,37,38,39,40,41,42,43,43,44,45,45],"decoded":"|Viewing Single Post From: Spoilers for the Week of February 11th|\n|Lil||Feb 1 2013, 09:58 AM|\nDon't care about Chloe/Taniel/Jen-Jen. Don't care about Sami, rea","decoded_with_specials":"|Viewing Single Post From: Spoilers for the Week of February 11th|\n|Lil||Feb 1 2013, 09:58 AM|\nDon't care about Chloe/Taniel/Jen-Jen. Don't care about Sami, rea"} +{"kind":"sample","source":"fixtures/lang/heb_Hebr.txt[:160]","text":"סיכונים – אלמנט מרכזי, חיוני לפעילות מסחרית. להבין מה הסיכון הוא מאוד חשוב. החוויה האנושית עולה כי מי יודע איך לקחת סיכונים בזמן, היא לנצח גדולים. לזכור פוליטיק","ids":[44039,24225,40979,1256,57890,2100,62926,56915,74336,1006,14,70125,44304,4168,45277,71627,80946,20204,8170,16,16960,242,12536,42026,41620,24225,11783,25250,99047,80906,15018,16,24489,52619,10657,22902,257,25086,8170,6166,47333,37712,63198,7385,109406,6958,34898,4168,11176,37254,21565,24225,40979,105939,54780,14,38603,101127,102,22353,69155,56094,16,4168,89151,6524,17060,11122,21867,17696],"ids_no_specials":[44039,24225,40979,1256,57890,2100,62926,56915,74336,1006,14,70125,44304,4168,45277,71627,80946,20204,8170,16,16960,242,12536,42026,41620,24225,11783,25250,99047,80906,15018,16,24489,52619,10657,22902,257,25086,8170,6166,47333,37712,63198,7385,109406,6958,34898,4168,11176,37254,21565,24225,40979,105939,54780,14,38603,101127,102,22353,69155,56094,16,4168,89151,6524,17060,11122,21867,17696],"tokens":["ס","×Ļ׼","×ķ׳×Ļ×Ŀ","ĠâĢĵ","Ġ×IJ׾×","ŀ×","ł×ĺ","Ġ×ŀר×","Ľ×ĸ","×Ļ",",","Ġ×Ĺ×Ļ","×ķ׳×Ļ","Ġ׾×","¤×¢","×Ļ׾×ķת","Ġ×ŀס×","Ĺר","×Ļת",".","Ġ׾×Ķ×","ij","×Ļף","Ġ×ŀ×Ķ","Ġ×Ķס","×Ļ׼","×ķף","Ġ×Ķ×ķ×IJ","Ġ×ŀ×IJ×ķ×ĵ","Ġ×Ĺש","×ķ×ij",".","Ġ×Ķ×Ĺ","×ķ×ķ","×Ļ×Ķ","Ġ×Ķ×IJ×","ł","×ķש","×Ļת","Ġ×¢","×ķ׾×Ķ","Ġ׼×Ļ","Ġ×ŀ×Ļ","Ġ×Ļ","×ķ×ĵ×¢","Ġ×IJ","×Ļ×ļ","Ġ׾×","§×","Ĺת","Ġס","×Ļ׼","×ķ׳×Ļ×Ŀ","Ġ×ij×ĸ","×ŀף",",","Ġ×Ķ×Ļ×IJ","Ġ׾׳×","¦","×Ĺ","Ġ×Ĵ×ĵ","×ķ׾×Ļ×Ŀ",".","Ġ׾×","ĸ׼","×ķר","Ġפ","×ķ׾","×Ļ×ĺ","×Ļק"],"offsets":[[0,1],[1,3],[3,7],[7,9],[9,13],[12,14],[13,15],[15,19],[18,20],[20,21],[21,22],[22,25],[25,28],[28,31],[30,32],[32,36],[36,40],[39,41],[41,43],[43,44],[44,48],[47,48],[48,50],[50,53],[53,56],[56,58],[58,60],[60,64],[64,69],[69,72],[72,74],[74,75],[75,78],[78,80],[80,82],[82,86],[85,86],[86,88],[88,90],[90,92],[92,95],[95,98],[98,101],[101,103],[103,106],[106,108],[108,110],[110,113],[112,114],[113,115],[115,117],[117,119],[119,123],[123,126],[126,128],[128,129],[129,133],[133,137],[136,137],[137,138],[138,141],[141,145],[145,146],[146,149],[148,150],[150,152],[152,154],[154,156],[156,158],[158,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,1,2,2,2,3,3,3,4,5,5,6,6,6,7,7,7,8,9,9,9,10,11,11,11,12,13,14,14,15,16,16,16,17,17,17,17,18,18,19,20,21,21,22,22,23,23,23,24,24,24,25,25,26,27,28,28,28,29,29,30,31,31,31,32,32,32,32],"decoded":"סיכונים – אלמנט מרכזי, חיוני לפעילות מסחרית. להבין מה הסיכון הוא מאוד חשוב. החוויה האנושית עולה כי מי יודע איך לקחת סיכונים בזמן, היא לנצח גדולים. לזכור פוליטיק","decoded_with_specials":"סיכונים – אלמנט מרכזי, חיוני לפעילות מסחרית. להבין מה הסיכון הוא מאוד חשוב. החוויה האנושית עולה כי מי יודע איך לקחת סיכונים בזמן, היא לנצח גדולים. לזכור פוליטיק"} +{"kind":"sample","source":"fixtures/lang/hin_Deva.txt[:160]","text":"PHOTOS: न्यूज पढ़ते-पढ़ते अचानक ये क्या करने लगी एंकर!\nकुछ समय पहले एक टीवी चैनल पर एंकर खबर पढ़ रही थी और पीछे की स्क्रीन पर 10 मिनिट का पोर्न वीडियो चलता रहा,","ids":[20050,4894,4687,28,23597,11072,19725,26265,14410,739,98,48932,47563,15,16472,739,98,48932,47563,21215,25096,22336,11018,29648,6011,10169,38738,48949,52769,33772,22270,8714,45988,9902,11018,7672,8567,11018,82684,252,71730,18246,14410,18359,68228,66862,115140,8714,15243,8714,36929,20846,9094,14598,49933,45988,9902,11018,7672,72525,30456,7672,14410,739,98,48932,32441,18359,8714,69316,8714,72134,14410,8714,739,252,6011,58531,12624,61515,95800,9094,49933,223,553,16199,42180,6581,256,56005,14410,10263,26064,104,19799,8714,35874,42190,10263,36929,14598,58447,32441,18359,3350,14],"ids_no_specials":[20050,4894,4687,28,23597,11072,19725,26265,14410,739,98,48932,47563,15,16472,739,98,48932,47563,21215,25096,22336,11018,29648,6011,10169,38738,48949,52769,33772,22270,8714,45988,9902,11018,7672,8567,11018,82684,252,71730,18246,14410,18359,68228,66862,115140,8714,15243,8714,36929,20846,9094,14598,49933,45988,9902,11018,7672,72525,30456,7672,14410,739,98,48932,32441,18359,8714,69316,8714,72134,14410,8714,739,252,6011,58531,12624,61515,95800,9094,49933,223,553,16199,42180,6581,256,56005,14410,10263,26064,104,19799,8714,35874,42190,10263,36929,14598,58447,32441,18359,3350,14],"tokens":["PH","OT","OS",":","Ġन","à¥įय","à¥Ĥ","à¤ľ","Ġप","à¤","¢","़","तà¥ĩ","-","प","à¤","¢","़","तà¥ĩ","Ġà¤ħ","à¤ļ","ान","à¤ķ","Ġय","à¥ĩ","Ġà¤ķ","à¥įया","Ġà¤ķर","नà¥ĩ","Ġल","à¤Ĺ","à¥Ģ","Ġà¤ı","à¤Ĥ","à¤ķ","र","!Ċ","à¤ķ","à¥ģà¤","Ľ","Ġसम","य","Ġप","ह","लà¥ĩ","Ġà¤ıà¤ķ","Ġà¤Ł","à¥Ģ","व","à¥Ģ","Ġà¤ļ","à¥Ī","न","ल","Ġपर","Ġà¤ı","à¤Ĥ","à¤ķ","र","Ġà¤ĸ","ब","र","Ġप","à¤","¢","़","Ġर","ह","à¥Ģ","Ġथ","à¥Ģ","Ġà¤Ķर","Ġप","à¥Ģ","à¤","Ľ","à¥ĩ","Ġà¤ķà¥Ģ","Ġस","à¥įà¤ķ","à¥įरà¥Ģ","न","Ġपर","Ġ","10","Ġम","िन","िà¤","Ł","Ġà¤ķा","Ġप","à¥ĭ","रà¥įà¤","¨","Ġव","à¥Ģ","ड","िय","à¥ĭ","Ġà¤ļ","ल","ता","Ġर","ह","ा",","],"offsets":[[0,2],[2,4],[4,6],[6,7],[7,9],[9,11],[11,12],[12,13],[13,15],[15,16],[15,16],[16,17],[17,19],[19,20],[20,21],[21,22],[21,22],[22,23],[23,25],[25,27],[27,28],[28,30],[30,31],[31,33],[33,34],[34,36],[36,39],[39,42],[42,44],[44,46],[46,47],[47,48],[48,50],[50,51],[51,52],[52,53],[53,55],[55,56],[56,58],[57,58],[58,61],[61,62],[62,64],[64,65],[65,67],[67,70],[70,72],[72,73],[73,74],[74,75],[75,77],[77,78],[78,79],[79,80],[80,83],[83,85],[85,86],[86,87],[87,88],[88,90],[90,91],[91,92],[92,94],[94,95],[94,95],[95,96],[96,98],[98,99],[99,100],[100,102],[102,103],[103,106],[106,108],[108,109],[109,110],[109,110],[110,111],[111,114],[114,116],[116,118],[118,121],[121,122],[122,125],[125,126],[126,128],[128,130],[130,132],[132,134],[133,134],[134,137],[137,139],[139,140],[140,143],[142,143],[143,145],[145,146],[146,147],[147,149],[149,150],[150,152],[152,153],[153,155],[155,157],[157,158],[158,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,1,2,2,2,2,3,3,3,3,3,4,5,5,5,5,5,6,6,6,6,7,7,8,8,9,9,10,10,10,11,11,11,11,12,13,13,13,14,14,15,15,15,16,17,17,17,17,18,18,18,18,19,20,20,20,20,21,21,21,22,22,22,22,23,23,23,24,24,25,26,26,26,26,26,27,28,28,28,28,29,30,31,32,32,32,32,33,34,34,34,34,35,35,35,35,35,36,36,36,37,37,37,38],"decoded":"PHOTOS: न्यूज पढ़ते-पढ़ते अचानक ये क्या करने लगी एंकर!\nकुछ समय पहले एक टीवी चैनल पर एंकर खबर पढ़ रही थी और पीछे की स्क्रीन पर 10 मिनिट का पोर्न वीडियो चलता रहा,","decoded_with_specials":"PHOTOS: न्यूज पढ़ते-पढ़ते अचानक ये क्या करने लगी एंकर!\nकुछ समय पहले एक टीवी चैनल पर एंकर खबर पढ़ रही थी और पीछे की स्क्रीन पर 10 मिनिट का पोर्न वीडियो चलता रहा,"} +{"kind":"sample","source":"fixtures/lang/jpn_Jpan.txt[:160]","text":"Omni Dallas Parkwest Hotelでは4ツ星ホテルで、アイアンホース・ゴルフコース、Love Field Airport とテキサス・スタジアムから7.8kmかかるのところにあります。 優れたホテルにオープンし、ダラスにある古代の建築の象徴です。\n部屋\n快適なゲストルームには、モダンな設備を備えたプレ","ids":[20556,5787,35752,6938,14795,25198,13761,22,41799,3326,47424,17383,7429,2539,410,96610,10194,14145,252,37046,4825,48896,7429,15587,14925,37046,410,36896,14233,23629,223,2495,17383,20367,18884,6421,4825,55554,15124,10194,22325,10115,25,16,26,12492,4196,76436,1576,58509,2298,26550,320,223,22228,48075,47424,17383,7429,2298,21721,64779,4919,2593,410,34755,9690,6421,67063,16783,1576,52484,1576,2421,77166,8262,876,1001,6647,201,2163,19256,2942,59952,24552,7429,44090,16658,410,31076,34755,4919,2942,56345,2747,14872,78616,100650],"ids_no_specials":[20556,5787,35752,6938,14795,25198,13761,22,41799,3326,47424,17383,7429,2539,410,96610,10194,14145,252,37046,4825,48896,7429,15587,14925,37046,410,36896,14233,23629,223,2495,17383,20367,18884,6421,4825,55554,15124,10194,22325,10115,25,16,26,12492,4196,76436,1576,58509,2298,26550,320,223,22228,48075,47424,17383,7429,2298,21721,64779,4919,2593,410,34755,9690,6421,67063,16783,1576,52484,1576,2421,77166,8262,876,1001,6647,201,2163,19256,2942,59952,24552,7429,44090,16658,410,31076,34755,4919,2942,56345,2747,14872,78616,100650],"tokens":["Om","ni","ĠDallas","ĠPark","west","ĠHotel","ãģ§ãģ¯","4","ãĥĦ","æĺŁ","ãĥĽ","ãĥĨ","ãĥ«","ãģ§","ãĢģ","ãĤ¢ãĤ¤","ãĤ¢","ãĥ³ãĥ","Ľ","ãĥ¼ãĤ¹","ãĥ»","ãĤ´","ãĥ«","ãĥķ","ãĤ³","ãĥ¼ãĤ¹","ãĢģ","Love","ĠField","ĠAirport","Ġ","ãģ¨","ãĥĨ","ãĤŃ","ãĤµ","ãĤ¹","ãĥ»","ãĤ¹ãĤ¿","ãĤ¸","ãĤ¢","ãĥł","ãģĭãĤī","7",".","8","km","ãģĭ","ãģĭãĤĭ","ãģ®","ãģ¨ãģĵãĤį","ãģ«","ãģĤãĤĬãģ¾ãģĻ","ãĢĤ","Ġ","åĦª","ãĤĮãģŁ","ãĥĽ","ãĥĨ","ãĥ«","ãģ«","ãĤª","ãĥ¼ãĥĹ","ãĥ³","ãģĹ","ãĢģ","ãĥĢ","ãĥ©","ãĤ¹","ãģ«ãģĤãĤĭ","åı¤ä»£","ãģ®","建ç¯ī","ãģ®","象","å¾´","ãģ§ãģĻ","ãĢĤĊ","éĥ¨","å±ĭ","Ċ","å¿«","éģ©","ãģª","ãĤ²","ãĤ¹ãĥĪ","ãĥ«","ãĥ¼ãĥł","ãģ«ãģ¯","ãĢģ","ãĥ¢","ãĥĢ","ãĥ³","ãģª","è¨ŃåĤĻ","ãĤĴ","åĤĻ","ãģĪãģŁ","ãĥĹãĥ¬"],"offsets":[[0,2],[2,4],[4,11],[11,16],[16,20],[20,26],[26,28],[28,29],[29,30],[30,31],[31,32],[32,33],[33,34],[34,35],[35,36],[36,38],[38,39],[39,41],[40,41],[41,43],[43,44],[44,45],[45,46],[46,47],[47,48],[48,50],[50,51],[51,55],[55,61],[61,69],[69,70],[70,71],[71,72],[72,73],[73,74],[74,75],[75,76],[76,78],[78,79],[79,80],[80,81],[81,83],[83,84],[84,85],[85,86],[86,88],[88,89],[89,91],[91,92],[92,95],[95,96],[96,100],[100,101],[101,102],[102,103],[103,105],[105,106],[106,107],[107,108],[108,109],[109,110],[110,112],[112,113],[113,114],[114,115],[115,116],[116,117],[117,118],[118,121],[121,123],[123,124],[124,126],[126,127],[127,128],[128,129],[129,131],[131,133],[133,134],[134,135],[135,136],[136,137],[137,138],[138,139],[139,140],[140,142],[142,143],[143,145],[145,147],[147,148],[148,149],[149,150],[150,151],[151,152],[152,154],[154,155],[155,156],[156,158],[158,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,1,2,2,3,4,5,6,6,6,6,6,6,7,8,8,8,8,8,9,10,10,10,10,10,11,12,13,14,15,16,16,16,16,16,17,18,18,18,18,18,19,20,21,22,23,23,23,23,23,23,24,25,26,26,26,26,26,26,26,26,26,26,27,28,28,28,28,28,28,28,28,28,28,28,29,30,30,31,32,32,32,32,32,32,32,32,33,34,34,34,34,34,34,34,34,34],"decoded":"Omni Dallas Parkwest Hotelでは4ツ星ホテルで、アイアンホース・ゴルフコース、Love Field Airport とテキサス・スタジアムから7.8kmかかるのところにあります。 優れたホテルにオープンし、ダラスにある古代の建築の象徴です。\n部屋\n快適なゲストルームには、モダンな設備を備えたプレ","decoded_with_specials":"Omni Dallas Parkwest Hotelでは4ツ星ホテルで、アイアンホース・ゴルフコース、Love Field Airport とテキサス・スタジアムから7.8kmかかるのところにあります。 優れたホテルにオープンし、ダラスにある古代の建築の象徴です。\n部屋\n快適なゲストルームには、モダンな設備を備えたプレ"} +{"kind":"sample","source":"fixtures/lang/kat_Geor.txt[:160]","text":"ჩვენ მსოფლიოს სხვადასხვა ქვეყანაში ვცხოვრობთ და სხვადასხვა ენაზე ვსაუბრობთ, მაგრამ საერთო მიზანი გვაერთიანებს. ჩვენი მთავარი მიზანი სამყაროს შემოქმედისა და ბიბლ","ids":[2017,105,45141,75222,41746,51342,21985,100,48078,13043,254,51342,49235,92931,45141,77330,82986,92931,45141,8891,10416,101,45141,13762,103,44243,10798,88400,10416,246,2017,106,92931,21985,246,44525,96584,108936,91805,49235,92931,45141,77330,82986,92931,45141,8891,108376,72949,10798,247,12618,10416,246,51342,10798,28910,242,44525,96584,108936,14,41746,10798,243,44525,66313,102694,245,96965,248,19063,41746,13043,247,44243,9066,65775,45141,10798,245,96965,248,9066,44243,35599,51342,16,10416,105,45141,75222,9066,41746,108936,84247,74684,9066,41746,13043,247,44243,9066,49235,66313,2017,103,74684,113381,93372,111331,21985,101,62051,13762,244,28837,8891,91805,10416,242,13043,242,48078],"ids_no_specials":[2017,105,45141,75222,41746,51342,21985,100,48078,13043,254,51342,49235,92931,45141,77330,82986,92931,45141,8891,10416,101,45141,13762,103,44243,10798,88400,10416,246,2017,106,92931,21985,246,44525,96584,108936,91805,49235,92931,45141,77330,82986,92931,45141,8891,108376,72949,10798,247,12618,10416,246,51342,10798,28910,242,44525,96584,108936,14,41746,10798,243,44525,66313,102694,245,96965,248,19063,41746,13043,247,44243,9066,65775,45141,10798,245,96965,248,9066,44243,35599,51342,16,10416,105,45141,75222,9066,41746,108936,84247,74684,9066,41746,13043,247,44243,9066,49235,66313,2017,103,74684,113381,93372,111331,21985,101,62051,13762,244,28837,8891,91805,10416,242,13043,242,48078],"tokens":["áĥ","©","áĥķ","áĥĶáĥľ","ĠáĥĽ","áĥ¡","áĥĿáĥ","¤","áĥļ","áĥĺáĥ","Ŀ","áĥ¡","Ġáĥ¡","áĥ®","áĥķ","áĥIJáĥĵ","áĥIJáĥ¡","áĥ®","áĥķ","áĥIJ","Ġáĥ","¥","áĥķ","áĥĶáĥ","§","áĥIJáĥľ","áĥIJáĥ","¨áĥĺ","Ġáĥ","ķ","áĥ","ª","áĥ®","áĥĿáĥ","ķ","áĥł","áĥĿáĥij","áĥĹ","ĠáĥĵáĥIJ","Ġáĥ¡","áĥ®","áĥķ","áĥIJáĥĵ","áĥIJáĥ¡","áĥ®","áĥķ","áĥIJ","ĠáĥĶ","áĥľ","áĥIJáĥ","ĸ","áĥĶ","Ġáĥ","ķ","áĥ¡","áĥIJáĥ","£áĥ","ij","áĥł","áĥĿáĥij","áĥĹ",",","ĠáĥĽ","áĥIJáĥ","Ĵ","áĥł","áĥIJáĥĽ","Ġáĥ¡áĥIJáĥ","Ķ","áĥłáĥ","Ĺ","áĥĿ","ĠáĥĽ","áĥĺáĥ","ĸ","áĥIJáĥľ","áĥĺ","ĠáĥĴ","áĥķ","áĥIJáĥ","Ķ","áĥłáĥ","Ĺ","áĥĺ","áĥIJáĥľ","áĥĶáĥij","áĥ¡",".","Ġáĥ","©","áĥķ","áĥĶáĥľ","áĥĺ","ĠáĥĽ","áĥĹ","áĥIJáĥķ","áĥIJáĥł","áĥĺ","ĠáĥĽ","áĥĺáĥ","ĸ","áĥIJáĥľ","áĥĺ","Ġáĥ¡","áĥIJáĥĽ","áĥ","§","áĥIJáĥł","áĥĿáĥ¡","Ġáĥ¨","áĥĶáĥĽ","áĥĿáĥ","¥","áĥĽ","áĥĶáĥ","ĵ","áĥĺáĥ¡","áĥIJ","ĠáĥĵáĥIJ","Ġáĥ","ij","áĥĺáĥ","ij","áĥļ"],"offsets":[[0,1],[0,1],[1,2],[2,4],[4,6],[6,7],[7,9],[8,9],[9,10],[10,12],[11,12],[12,13],[13,15],[15,16],[16,17],[17,19],[19,21],[21,22],[22,23],[23,24],[24,26],[25,26],[26,27],[27,29],[28,29],[29,31],[31,33],[32,34],[34,36],[35,36],[36,37],[36,37],[37,38],[38,40],[39,40],[40,41],[41,43],[43,44],[44,47],[47,49],[49,50],[50,51],[51,53],[53,55],[55,56],[56,57],[57,58],[58,60],[60,61],[61,63],[62,63],[63,64],[64,66],[65,66],[66,67],[67,69],[68,70],[69,70],[70,71],[71,73],[73,74],[74,75],[75,77],[77,79],[78,79],[79,80],[80,82],[82,86],[85,86],[86,88],[87,88],[88,89],[89,91],[91,93],[92,93],[93,95],[95,96],[96,98],[98,99],[99,101],[100,101],[101,103],[102,103],[103,104],[104,106],[106,108],[108,109],[109,110],[110,112],[111,112],[112,113],[113,115],[115,116],[116,118],[118,119],[119,121],[121,123],[123,124],[124,126],[126,128],[127,128],[128,130],[130,131],[131,133],[133,135],[135,136],[135,136],[136,138],[138,140],[140,142],[142,144],[144,146],[145,146],[146,147],[147,149],[148,149],[149,151],[151,152],[152,155],[155,157],[156,157],[157,159],[158,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,5,6,6,6,6,6,6,6,6,7,7,7,7,7,8,8,8,8,8,8,8,8,8,9,10,10,10,10,10,11,11,11,11,11,12,12,12,12,12,13,13,13,13,13,13,13,13,13,13,14,15,15,15,15,15,16,16,16,16,16,17,17,17,17,17,18,18,18,18,18,18,19,19,19,19,19,19,19,19,19,20,21,21,21,21,21],"decoded":"ჩვენ მსოფლიოს სხვადასხვა ქვეყანაში ვცხოვრობთ და სხვადასხვა ენაზე ვსაუბრობთ, მაგრამ საერთო მიზანი გვაერთიანებს. ჩვენი მთავარი მიზანი სამყაროს შემოქმედისა და ბიბლ","decoded_with_specials":"ჩვენ მსოფლიოს სხვადასხვა ქვეყანაში ვცხოვრობთ და სხვადასხვა ენაზე ვსაუბრობთ, მაგრამ საერთო მიზანი გვაერთიანებს. ჩვენი მთავარი მიზანი სამყაროს შემოქმედისა და ბიბლ"} +{"kind":"sample","source":"fixtures/lang/kor_Hang.txt[:160]","text":"전화번호:\n위치: 뉴질랜드 > 남섬 > 말버러 > 블레넘\n가격대 (1박): 117,886 원 - 140,244 원\n4.5성급 — Lugano Motor Lodge 4.5*\n- 예약 옵션:\n- 트립어드바이저는 호텔스닷컴, Expedia, Agoda, Asia Web Direct 및 Boo","ids":[17815,15775,78106,1137,25486,19807,28,1525,91720,40705,106760,17939,1955,53008,4199,108,1955,35342,46484,20499,1955,121377,36450,29934,249,201,6751,45046,14341,343,19,84062,2605,223,8717,14,31315,36249,565,223,7331,14,15676,36249,201,22,16,23,12169,50867,2136,75112,4728,27511,61153,223,22,16,23,32850,15,34870,48708,12132,116,58388,1137,15,107534,45505,9227,17939,46904,3761,46520,4065,58521,36966,245,10714,2714,118,67532,115,14,8699,6507,14,4396,13078,14,11607,5575,7851,29192,4983,81],"ids_no_specials":[17815,15775,78106,1137,25486,19807,28,1525,91720,40705,106760,17939,1955,53008,4199,108,1955,35342,46484,20499,1955,121377,36450,29934,249,201,6751,45046,14341,343,19,84062,2605,223,8717,14,31315,36249,565,223,7331,14,15676,36249,201,22,16,23,12169,50867,2136,75112,4728,27511,61153,223,22,16,23,32850,15,34870,48708,12132,116,58388,1137,15,107534,45505,9227,17939,46904,3761,46520,4065,58521,36966,245,10714,2714,118,67532,115,14,8699,6507,14,4396,13078,14,11607,5575,7851,29192,4983,81],"tokens":["ìłĦ","íĻĶ","ë²Īíĺ¸",":Ċ","ìľĦ","ì¹ĺ",":","Ġë","ī´","ì§Ī","ëŀľ","ëĵľ","Ġ>","ĠëĤ¨","ìĦ","¬","Ġ>","Ġë§IJ","ë²Ħ","룬","Ġ>","Ġë¸Ķ","ëłĪ","ëĦ","ĺ","Ċ","ê°Ģ","격","ëĮĢ","Ġ(","1","ë°ķ","):","Ġ","117",",","886","ĠìĽIJ","Ġ-","Ġ","140",",","244","ĠìĽIJ","Ċ","4",".","5","ìĦ±","ê¸ī","ĠâĢĶ","ĠLug","ano","ĠMotor","ĠLodge","Ġ","4",".","5","*Ċ","-","ĠìĺĪ","ìķ½","Ġìĺ","µ","ìħĺ",":Ċ","-","ĠíĬ¸","립","ìĸ´","ëĵľ","ë°Ķ","ìĿ´","ìłĢ","ëĬĶ","Ġíĺ¸","íħ","Ķ","ìĬ¤","ëĭ","·","ì»","´",",","ĠExp","edia",",","ĠAg","oda",",","ĠAsia","ĠWeb","ĠDirect","Ġë°ı","ĠBo","o"],"offsets":[[0,1],[1,2],[2,4],[4,6],[6,7],[7,8],[8,9],[9,11],[10,11],[11,12],[12,13],[13,14],[14,16],[16,18],[18,19],[18,19],[19,21],[21,23],[23,24],[24,25],[25,27],[27,29],[29,30],[30,31],[30,31],[31,32],[32,33],[33,34],[34,35],[35,37],[37,38],[38,39],[39,41],[41,42],[42,45],[45,46],[46,49],[49,51],[51,53],[53,54],[54,57],[57,58],[58,61],[61,63],[63,64],[64,65],[65,66],[66,67],[67,68],[68,69],[69,71],[71,75],[75,78],[78,84],[84,90],[90,91],[91,92],[92,93],[93,94],[94,96],[96,97],[97,99],[99,100],[100,102],[101,102],[102,103],[103,105],[105,106],[106,108],[108,109],[109,110],[110,111],[111,112],[112,113],[113,114],[114,115],[115,117],[117,118],[117,118],[118,119],[119,120],[119,120],[120,121],[120,121],[121,122],[122,126],[126,130],[130,131],[131,134],[134,137],[137,138],[138,143],[143,147],[147,154],[154,156],[156,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,1,2,2,3,4,4,4,4,4,5,6,6,6,7,8,8,8,9,10,10,10,10,11,12,12,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,32,33,34,34,35,36,37,38,39,40,41,42,43,43,44,44,44,45,46,47,47,47,47,47,47,47,47,48,48,48,48,48,48,48,48,49,50,50,51,52,52,53,54,55,56,57,58,58],"decoded":"전화번호:\n위치: 뉴질랜드 > 남섬 > 말버러 > 블레넘\n가격대 (1박): 117,886 원 - 140,244 원\n4.5성급 — Lugano Motor Lodge 4.5*\n- 예약 옵션:\n- 트립어드바이저는 호텔스닷컴, Expedia, Agoda, Asia Web Direct 및 Boo","decoded_with_specials":"전화번호:\n위치: 뉴질랜드 > 남섬 > 말버러 > 블레넘\n가격대 (1박): 117,886 원 - 140,244 원\n4.5성급 — Lugano Motor Lodge 4.5*\n- 예약 옵션:\n- 트립어드바이저는 호텔스닷컴, Expedia, Agoda, Asia Web Direct 및 Boo"} +{"kind":"sample","source":"fixtures/lang/rus_Cyrl.txt[:160]","text":"Покупая продукты в супермаркетах, сегодня уже мало кто верит в их качество. Как не сделать свое меню экстремальным и на что следует обращать внимание – читайте!","ids":[5890,25916,6836,738,111395,1000,9560,27515,89534,658,31543,1152,14,73245,31244,92007,54803,9837,53719,1000,15419,30134,10197,16,32029,3000,51541,42568,35673,1678,19757,33583,3556,54957,1130,1857,7069,38535,24047,103805,59602,1256,51563,22763,3],"ids_no_specials":[5890,25916,6836,738,111395,1000,9560,27515,89534,658,31543,1152,14,73245,31244,92007,54803,9837,53719,1000,15419,30134,10197,16,32029,3000,51541,42568,35673,1678,19757,33583,3556,54957,1130,1857,7069,38535,24047,103805,59602,1256,51563,22763,3],"tokens":["ÐŁ","окÑĥ","па","Ñı","ĠпÑĢодÑĥкÑĤÑĭ","Ġв","ĠÑģÑĥ","пеÑĢ","маÑĢ","к","еÑĤа","Ñħ",",","ĠÑģегоднÑı","ĠÑĥже","Ġмало","ĠкÑĤо","Ġве","ÑĢиÑĤ","Ġв","ĠиÑħ","ĠкаÑĩе","ÑģÑĤво",".","ĠÐļак","Ġне","ĠÑģделаÑĤÑĮ","ĠÑģвое","Ġмен","Ñİ","ĠÑįк","ÑģÑĤÑĢе","ма","лÑĮнÑĭм","Ġи","Ġна","ĠÑĩÑĤо","ĠÑģледÑĥеÑĤ","ĠобÑĢа","ÑīаÑĤÑĮ","Ġвнимание","ĠâĢĵ","ĠÑĩиÑĤа","йÑĤе","!"],"offsets":[[0,1],[1,4],[4,6],[6,7],[7,16],[16,18],[18,21],[21,24],[24,27],[27,28],[28,31],[31,32],[32,33],[33,41],[41,45],[45,50],[50,54],[54,57],[57,60],[60,62],[62,65],[65,70],[70,74],[74,75],[75,79],[79,82],[82,90],[90,95],[95,99],[99,100],[100,103],[103,107],[107,109],[109,114],[114,116],[116,119],[119,123],[123,131],[131,136],[136,140],[140,149],[149,151],[151,156],[156,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,1,2,3,3,3,3,3,3,4,5,6,7,8,9,9,10,11,12,12,13,14,15,16,17,18,18,19,19,19,19,20,21,22,23,24,24,25,26,27,27,28],"decoded":"Покупая продукты в супермаркетах, сегодня уже мало кто верит в их качество. Как не сделать свое меню экстремальным и на что следует обращать внимание – читайте!","decoded_with_specials":"Покупая продукты в супермаркетах, сегодня уже мало кто верит в их качество. Как не сделать свое меню экстремальным и на что следует обращать внимание – читайте!"} +{"kind":"sample","source":"fixtures/lang/tam_Taml.txt[:160]","text":"‘ஹலோ கலெக்டர் சாரா… சரக்கு வேணும்… கடை எப்ப திறப்பீங்க?’\nஒரு படத்தில் ஒயின்ஷாப்புக்குள் திருடப் போன வடிவேலு நன்றாக சரக்கடித்து விட்டு, ஒயின்ஷாப் ஓனருக்கு போன் ப","ids":[3505,1136,120,17857,30659,19375,17857,25685,16723,21966,40521,27061,36973,9519,1248,27061,12102,37825,6181,117304,29022,25482,1248,19375,18177,13046,36157,36619,24662,47443,36619,51471,40819,49667,1136,243,109015,17486,18177,37139,42091,68023,20858,33380,4543,118,9962,58542,48412,8080,87466,24662,34955,73859,73670,94842,16045,23713,18177,83798,23421,17857,6181,31515,50738,45465,27061,12102,37825,18177,95090,6181,23713,124854,6181,14,68023,20858,33380,4543,118,9962,106,3149,4192,244,16045,12102,48412,6181,94842,46763,17486],"ids_no_specials":[3505,1136,120,17857,30659,19375,17857,25685,16723,21966,40521,27061,36973,9519,1248,27061,12102,37825,6181,117304,29022,25482,1248,19375,18177,13046,36157,36619,24662,47443,36619,51471,40819,49667,1136,243,109015,17486,18177,37139,42091,68023,20858,33380,4543,118,9962,58542,48412,8080,87466,24662,34955,73859,73670,94842,16045,23713,18177,83798,23421,17857,6181,31515,50738,45465,27061,12102,37825,18177,95090,6181,23713,124854,6181,14,68023,20858,33380,4543,118,9962,106,3149,4192,244,16045,12102,48412,6181,94842,46763,17486],"tokens":["âĢĺ","à®","¹","ல","à¯ĭ","Ġà®ķ","ல","à¯Ĩ","à®ķ","à¯įà®Ł","à®°à¯į","Ġà®ļ","ார","ா","âĢ¦","Ġà®ļ","à®°","à®ķà¯įà®ķ","à¯ģ","Ġவà¯ĩ","ண","à¯ģà®®à¯į","âĢ¦","Ġà®ķ","à®Ł","à¯Ī","Ġà®İ","பà¯įப","Ġத","ிற","பà¯įப","à¯Ģ","à®Ļà¯įà®ķ","?âĢĻĊ","à®","Ĵ","à®°à¯ģ","Ġப","à®Ł","தà¯įத","ிலà¯į","Ġà®Ĵ","ய","ின","à¯įà®","·","ாà®","ªà¯įப","à¯ģà®ķà¯įà®ķ","à¯ģà®","³à¯į","Ġத","ிர","à¯ģà®Ł","பà¯į","Ġபà¯ĭ","ன","Ġவ","à®Ł","ிவ","à¯ĩ","ல","à¯ģ","Ġந","னà¯įà®±","ாà®ķ","Ġà®ļ","à®°","à®ķà¯įà®ķ","à®Ł","ிதà¯įத","à¯ģ","Ġவ","à®¿à®Łà¯įà®Ł","à¯ģ",",","Ġà®Ĵ","ய","ின","à¯įà®","·","ாà®","ª","à¯į","Ġà®","ĵ","ன","à®°","à¯ģà®ķà¯įà®ķ","à¯ģ","Ġபà¯ĭ","னà¯į","Ġப"],"offsets":[[0,1],[1,2],[1,2],[2,3],[3,4],[4,6],[6,7],[7,8],[8,9],[9,11],[11,13],[13,15],[15,17],[17,18],[18,19],[19,21],[21,22],[22,25],[25,26],[26,29],[29,30],[30,33],[33,34],[34,36],[36,37],[37,38],[38,40],[40,43],[43,45],[45,47],[47,50],[50,51],[51,54],[54,57],[57,58],[57,58],[58,60],[60,62],[62,63],[63,66],[66,69],[69,71],[71,72],[72,74],[74,76],[75,76],[76,78],[77,80],[80,84],[84,86],[85,87],[87,89],[89,91],[91,93],[93,95],[95,98],[98,99],[99,101],[101,102],[102,104],[104,105],[105,106],[106,107],[107,109],[109,112],[112,114],[114,116],[116,117],[117,120],[120,121],[121,125],[125,126],[126,128],[128,132],[132,133],[133,134],[134,136],[136,137],[137,139],[139,141],[140,141],[141,143],[142,143],[143,144],[144,146],[145,146],[146,147],[147,148],[148,152],[152,153],[153,156],[156,158],[158,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,1,1,1,1,2,2,2,2,2,2,3,3,3,4,5,5,5,5,6,6,6,7,8,8,8,9,9,10,10,10,10,10,11,12,12,12,13,13,13,13,14,14,14,14,14,14,14,14,14,14,15,15,15,15,16,16,17,17,17,17,17,17,18,18,18,19,19,19,19,19,19,20,20,20,21,22,22,22,22,22,22,22,22,23,23,23,23,23,23,24,24,25],"decoded":"‘ஹலோ கலெக்டர் சாரா… சரக்கு வேணும்… கடை எப்ப திறப்பீங்க?’\nஒரு படத்தில் ஒயின்ஷாப்புக்குள் திருடப் போன வடிவேலு நன்றாக சரக்கடித்து விட்டு, ஒயின்ஷாப் ஓனருக்கு போன் ப","decoded_with_specials":"‘ஹலோ கலெக்டர் சாரா… சரக்கு வேணும்… கடை எப்ப திறப்பீங்க?’\nஒரு படத்தில் ஒயின்ஷாப்புக்குள் திருடப் போன வடிவேலு நன்றாக சரக்கடித்து விட்டு, ஒயின்ஷாப் ஓனருக்கு போன் ப"} +{"kind":"sample","source":"fixtures/lang/tha_Thai.txt[:160]","text":"นับเจดีย์ดูนะครับว่าครบยี่สิบองค์รึเปล่า คำว่าซาว ทางเหนือแปลว่่ายี่สิบครับ ผมนับดูแล้วก็ครบนะ ลองดู เสร็จแล้วก็เข้าเยี่ยมชมพิพธภัณฑ์ เดินเข้าไปด้านในหน่อยก็จะม","ids":[1378,7913,26015,2665,115737,40432,76694,79813,24546,229,1598,3086,2553,4695,87028,53082,1598,7325,48760,15316,127487,24546,236,20063,223,21715,109099,66039,2384,1568,34008,4695,87028,79813,82140,60614,7913,40432,28017,21253,14175,66650,2925,48963,4566,40432,107627,1598,77276,28017,21253,53806,51494,21969,2088,5504,2088,33317,5143,11563,119793,59618,12968,53806,15862,44613,7976,13609,48439,21253,13166,2088],"ids_no_specials":[1378,7913,26015,2665,115737,40432,76694,79813,24546,229,1598,3086,2553,4695,87028,53082,1598,7325,48760,15316,127487,24546,236,20063,223,21715,109099,66039,2384,1568,34008,4695,87028,79813,82140,60614,7913,40432,28017,21253,14175,66650,2925,48963,4566,40432,107627,1598,77276,28017,21253,53806,51494,21969,2088,5504,2088,33317,5143,11563,119793,59618,12968,53806,15862,44613,7976,13609,48439,21253,13166,2088],"tokens":["à¸Ļ","ัà¸ļ","à¹Ģà¸Ī","à¸Ķ","ียà¹Į","à¸Ķู","à¸Ļะ","à¸Ħรัà¸ļ","วà¹Īาà¸","Ħ","ร","à¸ļ","ย","ีà¹Ī","สิà¸ļ","à¸Ńà¸ĩà¸Ħà¹Į","ร","ึ","à¹Ģà¸Ľà¸¥","à¹Īา","Ġà¸Ħำ","วà¹Īาà¸","ĭ","าว","Ġ","à¸Ĺาà¸ĩ","à¹Ģหà¸Ļืà¸Ń","à¹ģà¸Ľà¸¥","ว","à¹Ī","à¹Īาย","ีà¹Ī","สิà¸ļ","à¸Ħรัà¸ļ","Ġà¸ľ","มà¸Ļ","ัà¸ļ","à¸Ķู","à¹ģลà¹īว","à¸ģà¹ĩ","à¸Ħร","à¸ļà¸Ļ","ะ","Ġล","à¸Ńà¸ĩ","à¸Ķู","Ġà¹Ģส","ร","à¹ĩà¸Ī","à¹ģลà¹īว","à¸ģà¹ĩ","à¹Ģà¸Ĥà¹īา","à¹Ģย","ีà¹Īย","ม","à¸Ĭ","ม","à¸ŀิ","à¸ŀ","à¸ĺ","à¸łà¸±à¸ĵà¸ijà¹Į","Ġà¹Ģà¸Ķ","ิà¸Ļ","à¹Ģà¸Ĥà¹īา","à¹Ħà¸Ľ","à¸Ķà¹īาà¸Ļ","à¹ĥà¸Ļ","หà¸Ļ","à¹Īà¸Ńย","à¸ģà¹ĩ","à¸Īะ","ม"],"offsets":[[0,1],[1,3],[3,5],[5,6],[6,9],[9,11],[11,13],[13,17],[17,21],[20,21],[21,22],[22,23],[23,24],[24,26],[26,29],[29,33],[33,34],[34,35],[35,38],[38,40],[40,43],[43,47],[46,47],[47,49],[49,50],[50,53],[53,58],[58,61],[61,62],[62,63],[63,66],[66,68],[68,71],[71,75],[75,77],[77,79],[79,81],[81,83],[83,87],[87,89],[89,91],[91,93],[93,94],[94,96],[96,98],[98,100],[100,103],[103,104],[104,106],[106,110],[110,112],[112,116],[116,118],[118,121],[121,122],[122,123],[123,124],[124,126],[126,127],[127,128],[128,133],[133,136],[136,138],[138,142],[142,144],[144,148],[148,150],[150,152],[152,155],[155,157],[157,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,4,4,4,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6,6],"decoded":"นับเจดีย์ดูนะครับว่าครบยี่สิบองค์รึเปล่า คำว่าซาว ทางเหนือแปลว่่ายี่สิบครับ ผมนับดูแล้วก็ครบนะ ลองดู เสร็จแล้วก็เข้าเยี่ยมชมพิพธภัณฑ์ เดินเข้าไปด้านในหน่อยก็จะม","decoded_with_specials":"นับเจดีย์ดูนะครับว่าครบยี่สิบองค์รึเปล่า คำว่าซาว ทางเหนือแปลว่่ายี่สิบครับ ผมนับดูแล้วก็ครบนะ ลองดู เสร็จแล้วก็เข้าเยี่ยมชมพิพธภัณฑ์ เดินเข้าไปด้านในหน่อยก็จะม"} +{"kind":"sample","source":"fixtures/modalities/added_normalized_dense.txt[:160]","text":"FLIBBERJAST CRUNGLEDORF FLIBBERJAST SNORLAXIAN fast BLORPTRONIC ZORPTASTIC split WIDGETRON split stage SNORLAXIAN BLORPTRONIC BLORPTRONIC \n QUIBBLENAUT CRUNGLED","ids":[12973,11910,24728,44,36118,12414,6901,19385,3316,2508,40,21696,11910,24728,44,36118,23235,2508,11318,58,39129,6292,27787,2508,10986,72422,2848,1653,2508,10986,36118,2848,14241,448,2518,25175,72422,14241,6632,23235,2508,11318,58,39129,27787,2508,10986,72422,2848,27787,2508,10986,72422,2848,539,28734,11910,20880,87777,4713,12414,6901,19385,3316],"ids_no_specials":[12973,11910,24728,44,36118,12414,6901,19385,3316,2508,40,21696,11910,24728,44,36118,23235,2508,11318,58,39129,6292,27787,2508,10986,72422,2848,1653,2508,10986,36118,2848,14241,448,2518,25175,72422,14241,6632,23235,2508,11318,58,39129,27787,2508,10986,72422,2848,27787,2508,10986,72422,2848,539,28734,11910,20880,87777,4713,12414,6901,19385,3316],"tokens":["FL","IB","BER","J","AST","ĠCR","UN","GL","ED","OR","F","ĠFL","IB","BER","J","AST","ĠSN","OR","LA","X","IAN","Ġfast","ĠBL","OR","PT","RON","IC","ĠZ","OR","PT","AST","IC","Ġsplit","ĠW","ID","GET","RON","Ġsplit","Ġstage","ĠSN","OR","LA","X","IAN","ĠBL","OR","PT","RON","IC","ĠBL","OR","PT","RON","IC","ĠĊ","ĠQU","IB","BL","ENA","UT","ĠCR","UN","GL","ED"],"offsets":[[0,2],[2,4],[4,7],[7,8],[8,11],[11,14],[14,16],[16,18],[18,20],[20,22],[22,23],[23,26],[26,28],[28,31],[31,32],[32,35],[35,38],[38,40],[40,42],[42,43],[43,46],[46,51],[51,54],[54,56],[56,58],[58,61],[61,63],[63,65],[65,67],[67,69],[69,72],[72,74],[74,80],[80,82],[82,84],[84,87],[87,90],[90,96],[96,102],[102,105],[105,107],[107,109],[109,110],[110,113],[113,116],[116,118],[118,120],[120,123],[123,125],[125,128],[128,130],[130,132],[132,135],[135,137],[137,139],[139,142],[142,144],[144,146],[146,149],[149,151],[151,154],[154,156],[156,158],[158,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,1,1,1,1,1,1,2,2,2,2,2,3,3,3,3,3,4,5,5,5,5,5,6,6,6,6,6,7,8,8,8,8,9,10,11,11,11,11,11,12,12,12,12,12,13,13,13,13,13,14,15,15,15,15,15,16,16,16,16],"decoded":"FLIBBERJAST CRUNGLEDORF FLIBBERJAST SNORLAXIAN fast BLORPTRONIC ZORPTASTIC split WIDGETRON split stage SNORLAXIAN BLORPTRONIC BLORPTRONIC \n QUIBBLENAUT CRUNGLED","decoded_with_specials":"FLIBBERJAST CRUNGLEDORF FLIBBERJAST SNORLAXIAN fast BLORPTRONIC ZORPTASTIC split WIDGETRON split stage SNORLAXIAN BLORPTRONIC BLORPTRONIC \n QUIBBLENAUT CRUNGLED"} +{"kind":"sample","source":"fixtures/modalities/added_normalized_sparse.txt[:160]","text":"ZORPTASTIC for we for decode CRUNGLEDORF here through ZORPTASTIC and the we WIDGETRON model \n normalize bytes and back flows text CRUNGLEDORF WUZZLEFANG chunk b","ids":[60,2508,10986,36118,2848,362,579,362,64285,12414,6901,19385,3316,2508,40,2155,1407,1653,2508,10986,36118,2848,305,270,579,448,2518,25175,72422,2645,539,82182,20711,305,1559,18290,3051,12414,6901,19385,3316,2508,40,448,55,47534,4392,40,21096,48319,291],"ids_no_specials":[60,2508,10986,36118,2848,362,579,362,64285,12414,6901,19385,3316,2508,40,2155,1407,1653,2508,10986,36118,2848,305,270,579,448,2518,25175,72422,2645,539,82182,20711,305,1559,18290,3051,12414,6901,19385,3316,2508,40,448,55,47534,4392,40,21096,48319,291],"tokens":["Z","OR","PT","AST","IC","Ġfor","Ġwe","Ġfor","Ġdecode","ĠCR","UN","GL","ED","OR","F","Ġhere","Ġthrough","ĠZ","OR","PT","AST","IC","Ġand","Ġthe","Ġwe","ĠW","ID","GET","RON","Ġmodel","ĠĊ","Ġnormalize","Ġbytes","Ġand","Ġback","Ġflows","Ġtext","ĠCR","UN","GL","ED","OR","F","ĠW","U","ZZ","LE","F","ANG","Ġchunk","Ġb"],"offsets":[[0,1],[1,3],[3,5],[5,8],[8,10],[10,14],[14,17],[17,21],[21,28],[28,31],[31,33],[33,35],[35,37],[37,39],[39,40],[40,45],[45,53],[53,55],[55,57],[57,59],[59,62],[62,64],[64,68],[68,72],[72,75],[75,77],[77,79],[79,82],[82,85],[85,91],[91,93],[93,103],[103,109],[109,113],[113,118],[118,124],[124,129],[129,132],[132,134],[134,136],[136,138],[138,140],[140,141],[141,143],[143,144],[144,146],[146,148],[148,149],[149,152],[152,158],[158,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,1,2,3,4,5,5,5,5,5,5,6,7,8,8,8,8,8,9,10,11,12,12,12,12,13,14,15,16,17,18,19,20,21,21,21,21,21,21,22,22,22,22,22,22,23,24],"decoded":"ZORPTASTIC for we for decode CRUNGLEDORF here through ZORPTASTIC and the we WIDGETRON model \n normalize bytes and back flows text CRUNGLEDORF WUZZLEFANG chunk b","decoded_with_specials":"ZORPTASTIC for we for decode CRUNGLEDORF here through ZORPTASTIC and the we WIDGETRON model \n normalize bytes and back flows text CRUNGLEDORF WUZZLEFANG chunk b"} +{"kind":"sample","source":"fixtures/modalities/added_special_dense.txt[:160]","text":"fast <|xs2|> chunk <|xs1|> <|xs4|> <|xs3|> <|xs4|> modality language <|xs0|> reads <|xs2|> and and \n <|xs1|> <|xs0|> <|xs1|> <|xs4|> <|xs4|> <|xs1|> back and re","ids":[14484,818,94,38753,20,94,32,48319,818,94,38753,19,94,32,818,94,38753,22,94,32,818,94,38753,21,94,32,818,94,38753,22,94,32,80277,4063,818,94,38753,18,94,32,23332,818,94,38753,20,94,32,305,305,539,818,94,38753,19,94,32,818,94,38753,18,94,32,818,94,38753,19,94,32,818,94,38753,22,94,32,818,94,38753,22,94,32,818,94,38753,19,94,32,1559,305,322],"ids_no_specials":[14484,818,94,38753,20,94,32,48319,818,94,38753,19,94,32,818,94,38753,22,94,32,818,94,38753,21,94,32,818,94,38753,22,94,32,80277,4063,818,94,38753,18,94,32,23332,818,94,38753,20,94,32,305,305,539,818,94,38753,19,94,32,818,94,38753,18,94,32,818,94,38753,19,94,32,818,94,38753,22,94,32,818,94,38753,22,94,32,818,94,38753,19,94,32,1559,305,322],"tokens":["fast","Ġ<","|","xs","2","|",">","Ġchunk","Ġ<","|","xs","1","|",">","Ġ<","|","xs","4","|",">","Ġ<","|","xs","3","|",">","Ġ<","|","xs","4","|",">","Ġmodality","Ġlanguage","Ġ<","|","xs","0","|",">","Ġreads","Ġ<","|","xs","2","|",">","Ġand","Ġand","ĠĊ","Ġ<","|","xs","1","|",">","Ġ<","|","xs","0","|",">","Ġ<","|","xs","1","|",">","Ġ<","|","xs","4","|",">","Ġ<","|","xs","4","|",">","Ġ<","|","xs","1","|",">","Ġback","Ġand","Ġre"],"offsets":[[0,4],[4,6],[6,7],[7,9],[9,10],[10,11],[11,12],[12,18],[18,20],[20,21],[21,23],[23,24],[24,25],[25,26],[26,28],[28,29],[29,31],[31,32],[32,33],[33,34],[34,36],[36,37],[37,39],[39,40],[40,41],[41,42],[42,44],[44,45],[45,47],[47,48],[48,49],[49,50],[50,59],[59,68],[68,70],[70,71],[71,73],[73,74],[74,75],[75,76],[76,82],[82,84],[84,85],[85,87],[87,88],[88,89],[89,90],[90,94],[94,98],[98,100],[100,102],[102,103],[103,105],[105,106],[106,107],[107,108],[108,110],[110,111],[111,113],[113,114],[114,115],[115,116],[116,118],[118,119],[119,121],[121,122],[122,123],[123,124],[124,126],[126,127],[127,129],[129,130],[130,131],[131,132],[132,134],[134,135],[135,137],[137,138],[138,139],[139,140],[140,142],[142,143],[143,145],[145,146],[146,147],[147,148],[148,153],[153,157],[157,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,1,1,2,3,4,4,5,6,6,7,8,9,9,10,10,11,12,13,13,14,14,15,16,17,17,18,18,19,20,21,21,22,23,24,24,25,26,27,27,28,29,29,30,31,32,32,33,34,35,36,36,37,38,39,39,40,40,41,42,43,43,44,44,45,46,47,47,48,48,49,50,51,51,52,52,53,54,55,55,56,56,57,58,59,59,60,61,62],"decoded":"fast <|xs2|> chunk <|xs1|> <|xs4|> <|xs3|> <|xs4|> modality language <|xs0|> reads <|xs2|> and and \n <|xs1|> <|xs0|> <|xs1|> <|xs4|> <|xs4|> <|xs1|> back and re","decoded_with_specials":"fast <|xs2|> chunk <|xs1|> <|xs4|> <|xs3|> <|xs4|> modality language <|xs0|> reads <|xs2|> and and \n <|xs1|> <|xs0|> <|xs1|> <|xs4|> <|xs4|> <|xs1|> back and re"} +{"kind":"sample","source":"fixtures/modalities/added_special_sparse.txt[:160]","text":"<|xs0|> every for encode <|xs0|> bytes the normalize decode text model <|xs4|> <|xs3|> and \n and <|xs3|> here <|xs1|> again model here text <|xs2|> <|xs2|> lang","ids":[30,94,38753,18,94,32,1750,362,57395,818,94,38753,18,94,32,20711,270,82182,64285,3051,2645,818,94,38753,22,94,32,818,94,38753,21,94,32,305,539,305,818,94,38753,21,94,32,2155,818,94,38753,19,94,32,1820,2645,2155,3051,818,94,38753,20,94,32,818,94,38753,20,94,32,15096],"ids_no_specials":[30,94,38753,18,94,32,1750,362,57395,818,94,38753,18,94,32,20711,270,82182,64285,3051,2645,818,94,38753,22,94,32,818,94,38753,21,94,32,305,539,305,818,94,38753,21,94,32,2155,818,94,38753,19,94,32,1820,2645,2155,3051,818,94,38753,20,94,32,818,94,38753,20,94,32,15096],"tokens":["<","|","xs","0","|",">","Ġevery","Ġfor","Ġencode","Ġ<","|","xs","0","|",">","Ġbytes","Ġthe","Ġnormalize","Ġdecode","Ġtext","Ġmodel","Ġ<","|","xs","4","|",">","Ġ<","|","xs","3","|",">","Ġand","ĠĊ","Ġand","Ġ<","|","xs","3","|",">","Ġhere","Ġ<","|","xs","1","|",">","Ġagain","Ġmodel","Ġhere","Ġtext","Ġ<","|","xs","2","|",">","Ġ<","|","xs","2","|",">","Ġlang"],"offsets":[[0,1],[1,2],[2,4],[4,5],[5,6],[6,7],[7,13],[13,17],[17,24],[24,26],[26,27],[27,29],[29,30],[30,31],[31,32],[32,38],[38,42],[42,52],[52,59],[59,64],[64,70],[70,72],[72,73],[73,75],[75,76],[76,77],[77,78],[78,80],[80,81],[81,83],[83,84],[84,85],[85,86],[86,90],[90,92],[92,96],[96,98],[98,99],[99,101],[101,102],[102,103],[103,104],[104,109],[109,111],[111,112],[112,114],[114,115],[115,116],[116,117],[117,123],[123,129],[129,134],[134,139],[139,141],[141,142],[142,144],[144,145],[145,146],[146,147],[147,149],[149,150],[150,152],[152,153],[153,154],[154,155],[155,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,1,2,3,3,4,5,6,7,7,8,9,10,10,11,12,13,14,15,16,17,17,18,19,20,20,21,21,22,23,24,24,25,26,27,28,28,29,30,31,31,32,33,33,34,35,36,36,37,38,39,40,41,41,42,43,44,44,45,45,46,47,48,48,49],"decoded":"<|xs0|> every for encode <|xs0|> bytes the normalize decode text model <|xs4|> <|xs3|> and \n and <|xs3|> here <|xs1|> again model here text <|xs2|> <|xs2|> lang","decoded_with_specials":"<|xs0|> every for encode <|xs0|> bytes the normalize decode text model <|xs4|> <|xs3|> and \n and <|xs3|> here <|xs1|> again model here text <|xs2|> <|xs2|> lang"} +{"kind":"sample","source":"fixtures/modalities/agentic-traces.txt[:160]","text":"Analyze this codebase and summarize what it is and how it works.\nI need to explore the codebase structure by reading the main entry points and configuration fil","ids":[79418,2317,566,4181,14707,305,45706,1205,436,344,305,1192,436,2984,603,43,1309,304,8497,270,4181,14707,4456,513,5081,270,1840,10451,4365,305,13055,2274],"ids_no_specials":[79418,2317,566,4181,14707,305,45706,1205,436,344,305,1192,436,2984,603,43,1309,304,8497,270,4181,14707,4456,513,5081,270,1840,10451,4365,305,13055,2274],"tokens":["Analy","ze","Ġthis","Ġcode","base","Ġand","Ġsummarize","Ġwhat","Ġit","Ġis","Ġand","Ġhow","Ġit","Ġworks",".Ċ","I","Ġneed","Ġto","Ġexplore","Ġthe","Ġcode","base","Ġstructure","Ġby","Ġreading","Ġthe","Ġmain","Ġentry","Ġpoints","Ġand","Ġconfiguration","Ġfil"],"offsets":[[0,5],[5,7],[7,12],[12,17],[17,21],[21,25],[25,35],[35,40],[40,43],[43,46],[46,50],[50,54],[54,57],[57,63],[63,65],[65,66],[66,71],[71,74],[74,82],[82,86],[86,91],[91,95],[95,105],[105,108],[108,116],[116,120],[120,125],[125,131],[131,138],[138,142],[142,156],[156,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,1,2,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,18,19,20,21,22,23,24,25,26,27,28],"decoded":"Analyze this codebase and summarize what it is and how it works.\nI need to explore the codebase structure by reading the main entry points and configuration fil","decoded_with_specials":"Analyze this codebase and summarize what it is and how it works.\nI need to explore the codebase structure by reading the main entry points and configuration fil"} +{"kind":"sample","source":"fixtures/modalities/agentic_swe.txt[:160]","text":"[system]\nYou are a helpful assistant that can interact with a computer to solve tasks.\n\n[user]\n\n/testbed\n\nI've uploaded a pytho","ids":[61,27824,2296,3476,477,260,11502,22896,396,588,12982,418,260,6341,304,9487,10017,339,61,5265,2296,30,34980,284,63888,1018,54145,15130,201,1718,34980,284,63888,1018,43,5270,51532,260,280,5800,81],"ids_no_specials":[61,27824,2296,3476,477,260,11502,22896,396,588,12982,418,260,6341,304,9487,10017,339,61,5265,2296,30,34980,284,63888,1018,54145,15130,201,1718,34980,284,63888,1018,43,5270,51532,260,280,5800,81],"tokens":["[","system","]Ċ","You","Ġare","Ġa","Ġhelpful","Ġassistant","Ġthat","Ġcan","Ġinteract","Ġwith","Ġa","Ġcomputer","Ġto","Ġsolve","Ġtasks",".ĊĊ","[","user","]Ċ","<","upload","ed","_files",">Ċ","/test","bed","Ċ","Ċ","I","'ve","Ġuploaded","Ġa","Ġp","yth","o"],"offsets":[[0,1],[1,7],[7,9],[9,12],[12,16],[16,18],[18,26],[26,36],[36,41],[41,45],[45,54],[54,59],[59,61],[61,70],[70,73],[73,79],[79,85],[85,88],[88,89],[89,93],[93,95],[95,96],[96,102],[102,104],[104,110],[110,112],[112,117],[117,120],[120,121],[121,123],[123,129],[129,131],[131,137],[137,139],[139,140],[140,143],[143,152],[152,154],[154,156],[156,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,17,18,19,19,19,20,21,22,22,23,24,25,25,26,27,28,29,30,31,32,32,32],"decoded":"[system]\nYou are a helpful assistant that can interact with a computer to solve tasks.\n\n[user]\n\n/testbed\n\nI've uploaded a pytho","decoded_with_specials":"[system]\nYou are a helpful assistant that can interact with a computer to solve tasks.\n\n[user]\n\n/testbed\n\nI've uploaded a pytho"} +{"kind":"sample","source":"fixtures/modalities/code_mixed.txt[:160]","text":"// django-f3f9601cff03b38942e70ce8042bdfdec6002449/django/__init__.py\nfrom django.utils.version import get_version\n\nVERSION = (6, 2, 0, \"alpha\", 0)\n\n__version__","ids":[835,37923,2410,21,72,22610,19,69,617,3600,68,22297,3180,71,2122,353,27643,20,68,5920,23077,6391,15676,27,6761,25515,17,848,7025,121183,12403,201,5356,37923,61868,121719,1662,1178,48484,271,56,49061,438,343,24,14,223,20,14,223,18,14,582,8339,1760,223,18,868,848,9713,848],"ids_no_specials":[835,37923,2410,21,72,22610,19,69,617,3600,68,22297,3180,71,2122,353,27643,20,68,5920,23077,6391,15676,27,6761,25515,17,848,7025,121183,12403,201,5356,37923,61868,121719,1662,1178,48484,271,56,49061,438,343,24,14,223,20,14,223,18,14,582,8339,1760,223,18,868,848,9713,848],"tokens":["//","Ġdjango","-f","3","f","960","1","c","ff","03","b","389","42","e","70","ce","804","2","b","df","dec","600","244","9","/d","jango","/","__","init","__.","py","Ċ","from","Ġdjango",".utils",".version","Ġimport","Ġget","_version","ĊĊ","V","ERSION","Ġ=","Ġ(","6",",","Ġ","2",",","Ġ","0",",","Ġ\"","alpha","\",","Ġ","0",")ĊĊ","__","version","__"],"offsets":[[0,2],[2,9],[9,11],[11,12],[12,13],[13,16],[16,17],[17,18],[18,20],[20,22],[22,23],[23,26],[26,28],[28,29],[29,31],[31,33],[33,36],[36,37],[37,38],[38,40],[40,43],[43,46],[46,49],[49,50],[50,52],[52,57],[57,58],[58,60],[60,64],[64,67],[67,69],[69,70],[70,74],[74,81],[81,87],[87,95],[95,102],[102,106],[106,114],[114,116],[116,117],[117,123],[123,125],[125,127],[127,128],[128,129],[129,130],[130,131],[131,132],[132,133],[133,134],[134,135],[135,137],[137,142],[142,144],[144,145],[145,146],[146,149],[149,151],[151,158],[158,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,1,2,3,4,5,6,7,7,8,9,10,11,12,13,14,15,16,17,17,17,18,19,20,21,21,22,22,23,24,25,26,27,28,29,30,31,32,33,34,35,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54],"decoded":"// django-f3f9601cff03b38942e70ce8042bdfdec6002449/django/__init__.py\nfrom django.utils.version import get_version\n\nVERSION = (6, 2, 0, \"alpha\", 0)\n\n__version__","decoded_with_specials":"// django-f3f9601cff03b38942e70ce8042bdfdec6002449/django/__init__.py\nfrom django.utils.version import get_version\n\nVERSION = (6, 2, 0, \"alpha\", 0)\n\n__version__"} +{"kind":"sample","source":"fixtures/modalities/math_latex.txt[:160]","text":"Bayes and his Theorem\n\nMy earlier post on Bayesian probability seems to have generated quite a lot of readers, so this lunchtime I thought I’d add a little bit ","ids":[47498,273,305,793,2162,54895,271,6759,7728,2411,377,50869,10928,6179,304,611,9846,5686,260,3929,294,12592,14,832,566,17980,8322,342,3241,342,442,70,1258,260,2961,4669,223],"ids_no_specials":[47498,273,305,793,2162,54895,271,6759,7728,2411,377,50869,10928,6179,304,611,9846,5686,260,3929,294,12592,14,832,566,17980,8322,342,3241,342,442,70,1258,260,2961,4669,223],"tokens":["Bay","es","Ġand","Ġhis","Âł","Theorem","ĊĊ","My","Ġearlier","Ġpost","Ġon","ĠBayesian","Ġprobability","Ġseems","Ġto","Ġhave","Ġgenerated","Ġquite","Ġa","Ġlot","Ġof","Ġreaders",",","Ġso","Ġthis","Ġlunch","time","ĠI","Ġthought","ĠI","âĢĻ","d","Ġadd","Ġa","Ġlittle","Ġbit","Ġ"],"offsets":[[0,3],[3,5],[5,9],[9,13],[13,14],[14,21],[21,23],[23,25],[25,33],[33,38],[38,41],[41,50],[50,62],[62,68],[68,71],[71,76],[76,86],[86,92],[92,94],[94,98],[98,101],[101,109],[109,110],[110,113],[113,118],[118,124],[124,128],[128,130],[130,138],[138,140],[140,141],[141,142],[142,146],[146,148],[148,155],[155,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,1,2,3,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,23,24,25,26,27,28,29,30,31,32,33],"decoded":"Bayes and his Theorem\n\nMy earlier post on Bayesian probability seems to have generated quite a lot of readers, so this lunchtime I thought I’d add a little bit ","decoded_with_specials":"Bayes and his Theorem\n\nMy earlier post on Bayesian probability seems to have generated quite a lot of readers, so this lunchtime I thought I’d add a little bit "} +{"kind":"pair","text":"What is the capital of France?","pair":"Paris is the capital of France.","ids":[3085,344,270,6102,294,8760,33,51119,344,270,6102,294,8760,16],"tokens":["What","Ġis","Ġthe","Ġcapital","Ġof","ĠFrance","?","Paris","Ġis","Ġthe","Ġcapital","Ġof","ĠFrance","."],"type_ids":[0,0,0,0,0,0,0,1,1,1,1,1,1,1],"sequence_ids":[0,0,0,0,0,0,0,1,1,1,1,1,1,1],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0],"offsets":[[0,4],[4,7],[7,11],[11,19],[19,22],[22,29],[29,30],[0,5],[5,8],[8,12],[12,20],[20,23],[23,30],[30,31]]} +{"kind":"pair","text":"Question in English?","pair":"Ответ на русском языке, с цифрами 123.","ids":[10375,295,3947,33,61901,1857,39641,30114,116070,14,1025,63396,67823,223,6895,16],"tokens":["Question","Ġin","ĠEnglish","?","ÐŀÑĤвеÑĤ","Ġна","ĠÑĢÑĥÑģ","Ñģком","ĠÑıзÑĭке",",","ĠÑģ","ĠÑĨиÑĦ","ÑĢами","Ġ","123","."],"type_ids":[0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1],"sequence_ids":[0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"offsets":[[0,8],[8,11],[11,19],[19,20],[0,5],[5,8],[8,12],[12,16],[16,22],[22,23],[23,25],[25,29],[29,33],[33,34],[34,37],[37,38]]} +{"kind":"digest","file":"fixtures/lang/amh_Ethi.txt","cap_chars":200000,"text_sha256":"e353d32f45b449b76fb65ee48f1522086c02a6bffa607e73ed8c875c03c84908","n_tokens":292349,"ids_sha256":"539aa9ade55507e6906c189468d115449a1484691d447f748764867414c2dee0"} +{"kind":"digest","file":"fixtures/lang/arb_Arab.txt","cap_chars":200000,"text_sha256":"529587117db0a7abaa2c8f31ab05d7b2f77e84a11068b46e26b2b37774d836c7","n_tokens":76554,"ids_sha256":"bc4661d48d1e4c15bf3ca519bce7a46e698cd52099f75b87a4a96e3e20c7e45e"} +{"kind":"digest","file":"fixtures/lang/ben_Beng.txt","cap_chars":200000,"text_sha256":"915afd7dbbea8256bdb6112b16424c83aa47dae63c5cca0ebadfc6c227d2b89f","n_tokens":85970,"ids_sha256":"c7c385a5ce7ce120bcef57d907ce98bdffc520fd917faa0ba6850fcc65db359d"} +{"kind":"digest","file":"fixtures/lang/cmn_Hani.txt","cap_chars":200000,"text_sha256":"1273c0c093a00739be9efea7804b10a295f902359d11663306d097211934c6df","n_tokens":127127,"ids_sha256":"99a82dfe19e34b936c72ea4f3207adbacb816081df244a77335e5fcdb39fd9de"} +{"kind":"digest","file":"fixtures/lang/ell_Grek.txt","cap_chars":200000,"text_sha256":"158c8d3e4933091099bf12a1e21f57c7b7ff3f5381d4b25901eb1fa7ca0926f4","n_tokens":94320,"ids_sha256":"13c671c748f684e32d2759ff5c286cded43354a4fe89a9c14e76f40c812dad94"} +{"kind":"digest","file":"fixtures/lang/eng_Latn.txt","cap_chars":200000,"text_sha256":"d998cb3153e3a56c2ecdf49293062e8bc6ca43365b6c91cfed147d73b84c19fb","n_tokens":46419,"ids_sha256":"b8d5b3693ae0d8fa19034e3c0ce69344fbbd3211fb73a4e6fc1b379a63a754f4"} +{"kind":"digest","file":"fixtures/lang/heb_Hebr.txt","cap_chars":200000,"text_sha256":"386da8c0e0d416bffbe84b51baf895c67dbce6e0fd153be597933261794300fd","n_tokens":93691,"ids_sha256":"cac5409f8ffe1bf34340b05508baa948aaf2456820cbe32530251d16017a4887"} +{"kind":"digest","file":"fixtures/lang/hin_Deva.txt","cap_chars":200000,"text_sha256":"2d114148c81035b1b63c94c7e9e805933c215d8aa9500d916923aec0a124ddde","n_tokens":114815,"ids_sha256":"bb637bafe8d3c8a08a16a90c80a6955e67c01b58548bd05718d2a175989fdcb4"} +{"kind":"digest","file":"fixtures/lang/jpn_Jpan.txt","cap_chars":200000,"text_sha256":"2e55c036f500d7720d020be2db45c87a49c17d3b7e63b30bada9cef4cb6aa158","n_tokens":136421,"ids_sha256":"8231b7c1110facfb69cf3c9705699057b05c83a84a7f01c78595ae1280789b11"} +{"kind":"digest","file":"fixtures/lang/kat_Geor.txt","cap_chars":200000,"text_sha256":"c69a1aa8fb80b41459c98c15c756eb8f39a9151468b1d5b3f9bbffe4d82eb87a","n_tokens":146504,"ids_sha256":"9f4d1c5e8a8c5ac6d2d56bfe46b02fe267b04c38575a23f4bf29e6be18c4cafa"} +{"kind":"digest","file":"fixtures/lang/kor_Hang.txt","cap_chars":200000,"text_sha256":"68659781c288136c1caffcb10eec9130406ae8a7b003c1e327b7e3601eba041d","n_tokens":139133,"ids_sha256":"fce6695be12fb1d4d91ca0197bae61832e8370159df486e5d02c293932633280"} +{"kind":"digest","file":"fixtures/lang/rus_Cyrl.txt","cap_chars":200000,"text_sha256":"5f967a82627afbd72c2f21c6e56183dfa5efc986ba34350c1cf9eea6c27a6132","n_tokens":63127,"ids_sha256":"7b8a58e8723fb83102fa0c495430c9fc5e89db9cb841a9c37046e6a6a69bfdb3"} +{"kind":"digest","file":"fixtures/lang/tam_Taml.txt","cap_chars":200000,"text_sha256":"bf09e1bca55aeab34162d5dc61afec7ac976b4e66eb5833ac5a037305911d165","n_tokens":107236,"ids_sha256":"d3c05ae74b86c08dd32dbb630a6f9d7536844a2f1f928cae9028788e8346a518"} +{"kind":"digest","file":"fixtures/lang/tha_Thai.txt","cap_chars":200000,"text_sha256":"25209f4750b3d3f04440f98aeff50316a118cb982462a4763106e8747172b79c","n_tokens":78180,"ids_sha256":"20755a762906deb5267d6ec86359d3342c9e06569e65952fc1fe802ececea308"} +{"kind":"digest","file":"fixtures/modalities/added_normalized_dense.txt","cap_chars":200000,"text_sha256":"9d49b3cacf40962c051a09f1350a0f5dc2fc210556889ed0f341cfe0cf0af4f9","n_tokens":33526,"ids_sha256":"1de964eaed6ae747c33a9f5dcc659f482a0beef3b41aa156dcd6d58bdff28772"} +{"kind":"digest","file":"fixtures/modalities/added_normalized_sparse.txt","cap_chars":200000,"text_sha256":"55383f52b02a5d9686f9e4347729e5c08e61b9cff77913bd94ceb65c2fdf7fe4","n_tokens":23644,"ids_sha256":"746170ee1e1f1f67c7dab3ccdaedcd0f642d9848486eda911b6fbf7ee69659de"} +{"kind":"digest","file":"fixtures/modalities/added_special_dense.txt","cap_chars":200000,"text_sha256":"472f90f6f2f5803626df5d7088f8ef12984de6e88fedf67723887511d35b64b2","n_tokens":51782,"ids_sha256":"a0ba95b420230eafee51d61ff5f36e0f56ea6680195a1c5ee40714438f6dc32a"} +{"kind":"digest","file":"fixtures/modalities/added_special_sparse.txt","cap_chars":200000,"text_sha256":"9c52602f7c8d009b11377743f95e5bfe9053694fc6b48c3dd6c1c6dc0681ab5a","n_tokens":29618,"ids_sha256":"ca60d4bbeb92907448ebcab545e903dac46b8b86aadbbb566001fb4ba0304a4d"} +{"kind":"digest","file":"fixtures/modalities/agentic-traces.txt","cap_chars":200000,"text_sha256":"ae9d63c1adc49f572d9af635ac1b0421461e4d1b92c5f35ea572980ff1e2480c","n_tokens":53163,"ids_sha256":"54ab0b774818244cbd5220e91d41722260a7a89a207ccd8f17a11b90500ced93"} +{"kind":"digest","file":"fixtures/modalities/agentic_swe.txt","cap_chars":200000,"text_sha256":"b2d31e91ff91bb6c9a3aec3832d25acbecf9b58d5c0674e364d3b287182e7253","n_tokens":60758,"ids_sha256":"287c348a101f0d902bd5f889fffbdd59bd965581d27427b7fa8b7fbb1fea221c"} +{"kind":"digest","file":"fixtures/modalities/code_mixed.txt","cap_chars":200000,"text_sha256":"0fa7314727726db056433d36bc1bda021803be7f1d150cae83e362a3ff41821d","n_tokens":70400,"ids_sha256":"a8cddc61a12fed39f9cdda359e833d49f8d895fa1e09413c63c25dc2c964e15f"} +{"kind":"digest","file":"fixtures/modalities/math_latex.txt","cap_chars":200000,"text_sha256":"2d17be8704f1f7b4f2174a054c0b85d6d22039e00be8e4a487927d27f3852e90","n_tokens":51285,"ids_sha256":"f6e91353f94528afc4a520e13b1e0073c327e3a371f87a16db8b8451951208f8"} diff --git a/bindings/python/tests/golden/goldens/glm-5.2.jsonl b/bindings/python/tests/golden/goldens/glm-5.2.jsonl new file mode 100644 index 000000000..3c135eb31 --- /dev/null +++ b/bindings/python/tests/golden/goldens/glm-5.2.jsonl @@ -0,0 +1,64 @@ +{"kind":"meta","model":"glm-5.2","tokenizer_file":"glm-5.2-slim.json","tokenizers_version":"0.23.1"} +{"kind":"sample","source":"edge:empty","text":"","ids":[],"ids_no_specials":[],"tokens":[],"offsets":[],"type_ids":[],"special_tokens_mask":[],"word_ids":[],"decoded":"","decoded_with_specials":""} +{"kind":"sample","source":"edge:spaces","text":" ","ids":[262],"ids_no_specials":[262],"tokens":["ĠĠĠ"],"offsets":[[0,3]],"type_ids":[0],"special_tokens_mask":[0],"word_ids":[0],"decoded":" ","decoded_with_specials":" "} +{"kind":"sample","source":"edge:hello","text":"Hello world","ids":[39,301,385,1879],"ids_no_specials":[39,301,385,1879],"tokens":["H","el","lo","Ġworld"],"offsets":[[0,1],[1,3],[3,5],[5,11]],"type_ids":[0,0,0,0],"special_tokens_mask":[0,0,0,0],"word_ids":[0,0,0,1],"decoded":"Hello world","decoded_with_specials":"Hello world"} +{"kind":"sample","source":"edge:punctuation","text":"Hello, world!! How's it going? (fine; thanks...)","ids":[39,301,385,11,1879,0,0,472,363,594,432,728,287,30,320,61622,26,1091,74,82,1112,8],"ids_no_specials":[39,301,385,11,1879,0,0,472,363,594,432,728,287,30,320,61622,26,1091,74,82,1112,8],"tokens":["H","el","lo",",","Ġworld","!","!","ĠH","ow","'s","Ġit","Ġgo","ing","?","Ġ(","fine",";","Ġthan","k","s","...",")"],"offsets":[[0,1],[1,3],[3,5],[5,6],[6,12],[12,13],[13,14],[14,16],[16,18],[18,20],[20,23],[23,26],[26,29],[29,30],[30,32],[32,36],[36,37],[37,42],[42,43],[43,44],[44,47],[47,48]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,1,2,3,3,4,4,5,6,7,7,8,9,10,11,12,12,12,13,13],"decoded":"Hello, world!! How's it going? (fine; thanks...)","decoded_with_specials":"Hello, world!! How's it going? (fine; thanks...)"} +{"kind":"sample","source":"edge:whitespace-mix","text":"line one\nline two\r\n\tindented\n trailing ","ids":[1056,825,198,1056,1378,319,197,484,306,291,198,220,489,604,287,256],"ids_no_specials":[1056,825,198,1056,1378,319,197,484,306,291,198,220,489,604,287,256],"tokens":["line","Ġone","Ċ","line","Ġtwo","čĊ","ĉ","ind","ent","ed","Ċ","Ġ","Ġtr","ail","ing","ĠĠ"],"offsets":[[0,4],[4,8],[8,9],[9,13],[13,17],[17,19],[19,20],[20,23],[23,26],[26,28],[28,29],[29,30],[30,33],[33,36],[36,39],[39,41]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,1,2,3,4,5,6,6,6,6,7,8,9,9,9,10],"decoded":"line one\nline two\r\n\tindented\n trailing ","decoded_with_specials":"line one\nline two\r\n\tindented\n trailing "} +{"kind":"sample","source":"edge:accents","text":"café naïve résumé — déjà vu","ids":[924,69,963,308,64,127,107,586,435,963,1242,963,636,242,294,963,73,127,254,348,84],"ids_no_specials":[924,69,963,308,64,127,107,586,435,963,1242,963,636,242,294,963,73,127,254,348,84],"tokens":["ca","f","é","Ġn","a","Ã","¯","ve","Ġr","é","sum","é","ĠâĢ","Ķ","Ġd","é","j","Ã","ł","Ġv","u"],"offsets":[[0,2],[2,3],[3,4],[4,6],[6,7],[7,8],[7,8],[8,10],[10,12],[12,13],[13,16],[16,17],[17,19],[18,19],[19,21],[21,22],[22,23],[23,24],[23,24],[24,26],[26,27]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,1,1,1,1,1,2,2,2,2,3,3,4,4,4,4,4,5,5],"decoded":"café naïve résumé — déjà vu","decoded_with_specials":"café naïve résumé — déjà vu"} +{"kind":"sample","source":"edge:accents-decomposed","text":"café naïve résumé","ids":[924,1859,136,223,308,64,72,136,230,586,312,136,223,1242,68,136,223],"ids_no_specials":[924,1859,136,223,308,64,72,136,230,586,312,136,223,1242,68,136,223],"tokens":["ca","fe","Ì","ģ","Ġn","a","i","Ì","Ī","ve","Ġre","Ì","ģ","sum","e","Ì","ģ"],"offsets":[[0,2],[2,4],[4,5],[4,5],[5,7],[7,8],[8,9],[9,10],[9,10],[10,12],[12,15],[15,16],[15,16],[16,19],[19,20],[20,21],[20,21]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,1,1,2,2,2,3,3,3,4,5,5,5,5,6,6],"decoded":"café naïve résumé","decoded_with_specials":"café naïve résumé"} +{"kind":"sample","source":"edge:emoji","text":"🤗 emoji, families 👨‍👩‍👧‍👦, flags 🇫🇷 and skin tones 👍🏽","ids":[172,253,97,245,976,78,73,72,11,282,309,321,550,220,172,253,239,101,378,235,172,253,239,102,378,235,172,253,239,100,378,235,172,253,239,99,11,1320,351,82,220,172,253,229,104,172,253,229,115,323,1901,258,259,263,288,220,172,253,239,235,172,253,237,121],"ids_no_specials":[172,253,97,245,976,78,73,72,11,282,309,321,550,220,172,253,239,101,378,235,172,253,239,102,378,235,172,253,239,100,378,235,172,253,239,99,11,1320,351,82,220,172,253,229,104,172,253,229,115,323,1901,258,259,263,288,220,172,253,239,235,172,253,237,121],"tokens":["ð","Ł","¤","Ĺ","Ġem","o","j","i",",","Ġf","am","il","ies","Ġ","ð","Ł","ij","¨","âĢ","į","ð","Ł","ij","©","âĢ","į","ð","Ł","ij","§","âĢ","į","ð","Ł","ij","¦",",","Ġfl","ag","s","Ġ","ð","Ł","ĩ","«","ð","Ł","ĩ","·","Ġand","Ġsk","in","Ġt","on","es","Ġ","ð","Ł","ij","į","ð","Ł","ı","½"],"offsets":[[0,1],[0,1],[0,1],[0,1],[1,4],[4,5],[5,6],[6,7],[7,8],[8,10],[10,12],[12,14],[14,17],[17,18],[18,19],[18,19],[18,19],[18,19],[19,20],[19,20],[20,21],[20,21],[20,21],[20,21],[21,22],[21,22],[22,23],[22,23],[22,23],[22,23],[23,24],[23,24],[24,25],[24,25],[24,25],[24,25],[25,26],[26,29],[29,31],[31,32],[32,33],[33,34],[33,34],[33,34],[33,34],[34,35],[34,35],[34,35],[34,35],[35,39],[39,42],[42,44],[44,46],[46,48],[48,50],[50,51],[51,52],[51,52],[51,52],[51,52],[52,53],[52,53],[52,53],[52,53]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,1,1,1,1,2,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,5,5,5,6,6,6,6,6,6,6,6,6,7,8,8,9,9,9,10,10,10,10,10,10,10,10,10],"decoded":"🤗 emoji, families 👨‍👩‍👧‍👦, flags 🇫🇷 and skin tones 👍🏽","decoded_with_specials":"🤗 emoji, families 👨‍👩‍👧‍👦, flags 🇫🇷 and skin tones 👍🏽"} +{"kind":"sample","source":"edge:cjk","text":"漢字とひらがなとカタカナが混ざった文章です。","ids":[162,120,95,161,255,245,159,223,101,159,223,110,159,224,231,159,223,234,159,223,103,159,223,101,159,224,104,159,224,123,159,224,104,159,225,232,159,223,234,162,115,115,159,223,244,159,223,96,159,223,253,162,244,229,163,104,254,159,223,100,159,223,247,1773],"ids_no_specials":[162,120,95,161,255,245,159,223,101,159,223,110,159,224,231,159,223,234,159,223,103,159,223,101,159,224,104,159,224,123,159,224,104,159,225,232,159,223,234,162,115,115,159,223,244,159,223,96,159,223,253,162,244,229,163,104,254,159,223,100,159,223,247,1773],"tokens":["æ","¼","¢","å","Ń","Ĺ","ã","ģ","¨","ã","ģ","²","ã","Ĥ","ī","ã","ģ","Į","ã","ģ","ª","ã","ģ","¨","ã","Ĥ","«","ã","Ĥ","¿","ã","Ĥ","«","ã","ĥ","Ĭ","ã","ģ","Į","æ","·","·","ã","ģ","ĸ","ã","ģ","£","ã","ģ","Ł","æ","ĸ","ĩ","ç","«","ł","ã","ģ","§","ã","ģ","Ļ","ãĢĤ"],"offsets":[[0,1],[0,1],[0,1],[1,2],[1,2],[1,2],[2,3],[2,3],[2,3],[3,4],[3,4],[3,4],[4,5],[4,5],[4,5],[5,6],[5,6],[5,6],[6,7],[6,7],[6,7],[7,8],[7,8],[7,8],[8,9],[8,9],[8,9],[9,10],[9,10],[9,10],[10,11],[10,11],[10,11],[11,12],[11,12],[11,12],[12,13],[12,13],[12,13],[13,14],[13,14],[13,14],[14,15],[14,15],[14,15],[15,16],[15,16],[15,16],[16,17],[16,17],[16,17],[17,18],[17,18],[17,18],[18,19],[18,19],[18,19],[19,20],[19,20],[19,20],[20,21],[20,21],[20,21],[21,22]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"decoded":"漢字とひらがなとカタカナが混ざった文章です。","decoded_with_specials":"漢字とひらがなとカタカナが混ざった文章です。"} +{"kind":"sample","source":"edge:korean","text":"한국어 텍스트 조각","ids":[169,243,250,166,113,255,168,244,112,220,169,227,235,168,232,97,169,232,116,220,168,94,108,166,108,223],"ids_no_specials":[169,243,250,166,113,255,168,244,112,220,169,227,235,168,232,97,169,232,116,220,168,94,108,166,108,223],"tokens":["í","ķ","ľ","ê","µ","Ń","ì","ĸ","´","Ġ","í","ħ","į","ì","Ĭ","¤","í","Ĭ","¸","Ġ","ì","¡","°","ê","°","ģ"],"offsets":[[0,1],[0,1],[0,1],[1,2],[1,2],[1,2],[2,3],[2,3],[2,3],[3,4],[4,5],[4,5],[4,5],[5,6],[5,6],[5,6],[6,7],[6,7],[6,7],[7,8],[8,9],[8,9],[8,9],[9,10],[9,10],[9,10]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2],"decoded":"한국어 텍스트 조각","decoded_with_specials":"한국어 텍스트 조각"} +{"kind":"sample","source":"edge:rtl","text":"مرحبا بالعالم — שלום עולם","ids":[149,227,148,109,148,255,148,101,148,100,220,148,101,148,100,149,226,148,117,148,100,149,226,149,227,636,242,220,147,102,147,250,147,243,147,251,220,147,95,147,243,147,250,147,251],"ids_no_specials":[149,227,148,109,148,255,148,101,148,100,220,148,101,148,100,149,226,148,117,148,100,149,226,149,227,636,242,220,147,102,147,250,147,243,147,251,220,147,95,147,243,147,250,147,251],"tokens":["Ù","ħ","Ø","±","Ø","Ń","Ø","¨","Ø","§","Ġ","Ø","¨","Ø","§","Ù","Ħ","Ø","¹","Ø","§","Ù","Ħ","Ù","ħ","ĠâĢ","Ķ","Ġ","×","©","×","ľ","×","ķ","×","Ŀ","Ġ","×","¢","×","ķ","×","ľ","×","Ŀ"],"offsets":[[0,1],[0,1],[1,2],[1,2],[2,3],[2,3],[3,4],[3,4],[4,5],[4,5],[5,6],[6,7],[6,7],[7,8],[7,8],[8,9],[8,9],[9,10],[9,10],[10,11],[10,11],[11,12],[11,12],[12,13],[12,13],[13,15],[14,15],[15,16],[16,17],[16,17],[17,18],[17,18],[18,19],[18,19],[19,20],[19,20],[20,21],[21,22],[21,22],[22,23],[22,23],[23,24],[23,24],[24,25],[24,25]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4],"decoded":"مرحبا بالعالم — שלום עולם","decoded_with_specials":"مرحبا بالعالم — שלום עולם"} +{"kind":"sample","source":"edge:numbers","text":"1234567890, 3.14159, 1,000,000th","ids":[16,17,18,19,20,21,22,23,24,15,11,220,18,13,16,19,16,20,24,11,220,16,11,15,15,15,11,15,15,15,339],"ids_no_specials":[16,17,18,19,20,21,22,23,24,15,11,220,18,13,16,19,16,20,24,11,220,16,11,15,15,15,11,15,15,15,339],"tokens":["1","2","3","4","5","6","7","8","9","0",",","Ġ","3",".","1","4","1","5","9",",","Ġ","1",",","0","0","0",",","0","0","0","th"],"offsets":[[0,1],[1,2],[2,3],[3,4],[4,5],[5,6],[6,7],[7,8],[8,9],[9,10],[10,11],[11,12],[12,13],[13,14],[14,15],[15,16],[16,17],[17,18],[18,19],[19,20],[20,21],[21,22],[22,23],[23,24],[24,25],[25,26],[26,27],[27,28],[28,29],[29,30],[30,32]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,1,1,1,2,2,2,3,4,5,6,7,8,8,8,9,9,10,11,12,13,14,14,14,15,16,16,16,17],"decoded":"1234567890, 3.14159, 1,000,000th","decoded_with_specials":"1234567890, 3.14159, 1,000,000th"} +{"kind":"sample","source":"edge:code","text":"def f(x):\n return x**2 # squared\nprint(f'{f(3)=}')","ids":[750,282,7,87,982,262,470,856,334,17,220,671,274,446,1605,198,1350,955,6,90,69,7,18,8,28,92,863],"ids_no_specials":[750,282,7,87,982,262,470,856,334,17,220,671,274,446,1605,198,1350,955,6,90,69,7,18,8,28,92,863],"tokens":["def","Ġf","(","x","):Ċ","ĠĠĠ","Ġreturn","Ġx","**","2","Ġ","Ġ#","Ġs","qu","ared","Ċ","print","(f","'","{","f","(","3",")","=","}","')"],"offsets":[[0,3],[3,5],[5,6],[6,7],[7,10],[10,13],[13,20],[20,22],[22,24],[24,25],[25,26],[26,28],[28,30],[30,32],[32,36],[36,37],[37,42],[42,44],[44,45],[45,46],[46,47],[47,48],[48,49],[49,50],[50,51],[51,52],[52,54]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,1,2,2,3,4,5,6,7,8,9,10,11,11,11,12,13,14,15,15,16,17,18,19,19,19,19],"decoded":"def f(x):\n return x**2 # squared\nprint(f'{f(3)=}')","decoded_with_specials":"def f(x):\n return x**2 # squared\nprint(f'{f(3)=}')"} +{"kind":"sample","source":"edge:long-word","text":"Donaudampfschifffahrtsgesellschaftskapitänsmützenabzeichen","ids":[35,263,64,661,1121,69,82,331,333,542,1466,81,83,82,70,288,613,82,331,64,723,82,74,391,275,127,97,77,82,76,127,120,83,89,268,370,89,68,713,268],"ids_no_specials":[35,263,64,661,1121,69,82,331,333,542,1466,81,83,82,70,288,613,82,331,64,723,82,74,391,275,127,97,77,82,76,127,120,83,89,268,370,89,68,713,268],"tokens":["D","on","a","ud","amp","f","s","ch","if","ff","ah","r","t","s","g","es","ell","s","ch","a","ft","s","k","ap","it","Ã","¤","n","s","m","Ã","¼","t","z","en","ab","z","e","ich","en"],"offsets":[[0,1],[1,3],[3,4],[4,6],[6,9],[9,10],[10,11],[11,13],[13,15],[15,17],[17,19],[19,20],[20,21],[21,22],[22,23],[23,25],[25,28],[28,29],[29,31],[31,32],[32,34],[34,35],[35,36],[36,38],[38,40],[40,41],[40,41],[41,42],[42,43],[43,44],[44,45],[44,45],[45,46],[46,47],[47,49],[49,51],[51,52],[52,53],[53,56],[56,58]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":"Donaudampfschifffahrtsgesellschaftskapitänsmützenabzeichen","decoded_with_specials":"Donaudampfschifffahrtsgesellschaftskapitänsmützenabzeichen"} +{"kind":"sample","source":"edge:url-email","text":"https://example.com/a/b?q=1&r=2#frag user.name+tag@example.co.uk","ids":[1254,82,1110,327,1516,905,14,64,14,65,30,80,28,16,5,81,28,17,2,1626,351,1196,13,606,10,83,351,31,327,1516,6830,13,84,74],"ids_no_specials":[1254,82,1110,327,1516,905,14,64,14,65,30,80,28,16,5,81,28,17,2,1626,351,1196,13,606,10,83,351,31,327,1516,6830,13,84,74],"tokens":["http","s","://","ex","ample",".com","/","a","/","b","?","q","=","1","&","r","=","2","#","fr","ag","Ġuser",".","name","+","t","ag","@","ex","ample",".co",".","u","k"],"offsets":[[0,4],[4,5],[5,8],[8,10],[10,15],[15,19],[19,20],[20,21],[21,22],[22,23],[23,24],[24,25],[25,26],[26,27],[27,28],[28,29],[29,30],[30,31],[31,32],[32,34],[34,36],[36,41],[41,42],[42,46],[46,47],[47,48],[48,50],[50,51],[51,53],[53,58],[58,61],[61,62],[62,63],[63,64]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,1,2,2,3,4,4,5,5,6,6,7,8,9,9,10,11,12,12,12,13,14,14,15,15,15,16,16,16,17,18,18,18],"decoded":"https://example.com/a/b?q=1&r=2#frag user.name+tag@example.co.uk","decoded_with_specials":"https://example.com/a/b?q=1&r=2#frag user.name+tag@example.co.uk"} +{"kind":"sample","source":"edge:unicode-spaces","text":"a b c　d","ids":[64,126,254,65,378,231,66,1277,222,67],"ids_no_specials":[64,126,254,65,378,231,66,1277,222,67],"tokens":["a","Â","ł","b","âĢ","ī","c","ãĢ","Ģ","d"],"offsets":[[0,1],[1,2],[1,2],[2,3],[3,4],[3,4],[4,5],[5,6],[5,6],[6,7]],"type_ids":[0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0],"word_ids":[0,1,1,1,2,2,2,3,3,3],"decoded":"a b c　d","decoded_with_specials":"a b c　d"} +{"kind":"sample","source":"edge:math","text":"∑ᵢ xᵢ² ≤ ∫₀^∞ e⁻ˣ dx ≈ 1","ids":[158,230,239,157,113,95,856,157,113,95,126,110,220,158,231,97,220,158,230,104,158,224,222,61,158,230,252,384,158,223,119,135,96,294,87,220,158,231,230,220,16],"ids_no_specials":[158,230,239,157,113,95,856,157,113,95,126,110,220,158,231,97,220,158,230,104,158,224,222,61,158,230,252,384,158,223,119,135,96,294,87,220,158,231,230,220,16],"tokens":["â","Ī","ij","á","µ","¢","Ġx","á","µ","¢","Â","²","Ġ","â","ī","¤","Ġ","â","Ī","«","â","Ĥ","Ģ","^","â","Ī","ŀ","Ġe","â","ģ","»","Ë","£","Ġd","x","Ġ","â","ī","Ī","Ġ","1"],"offsets":[[0,1],[0,1],[0,1],[1,2],[1,2],[1,2],[2,4],[4,5],[4,5],[4,5],[5,6],[5,6],[6,7],[7,8],[7,8],[7,8],[8,9],[9,10],[9,10],[9,10],[10,11],[10,11],[10,11],[11,12],[12,13],[12,13],[12,13],[13,15],[15,16],[15,16],[15,16],[16,17],[16,17],[17,19],[19,20],[20,21],[21,22],[21,22],[21,22],[22,23],[23,24]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,1,1,1,1,2,2,3,3,3,3,4,4,4,4,5,5,5,6,6,6,6,7,8,8,8,8,8,9,9,10,10,10,10,11,12],"decoded":"∑ᵢ xᵢ² ≤ ∫₀^∞ e⁻ˣ dx ≈ 1","decoded_with_specials":"∑ᵢ xᵢ² ≤ ∫₀^∞ e⁻ˣ dx ≈ 1"} +{"kind":"sample","source":"fixtures/lang/amh_Ethi.txt[:160]","text":"- ፍርድ ቤቱ በአቶ ዮናታን ተስፋዬ ላይ የ6 አመት ከ6 ወር ጽኑ የእስር ቅጣት አስተላለፈ\n- አንድነት ዶክተር ቴድሮስ ለዓለም ጤና ድርጅት ዋና ዳይሬክተርነት በመመረጣቸው ደስታውን ገለጸ\n- ህንድ መጠነ ሰፊ የድንጋይ ከሰል ሃይል ጣብያዎች የመገንባት እ","ids":[12,220,157,235,235,157,230,255,157,233,113,220,157,231,97,157,231,109,220,157,231,254,157,232,254,157,231,114,220,157,233,106,157,232,241,157,231,111,157,232,243,220,157,231,108,157,230,113,157,235,233,157,233,105,220,157,230,233,157,233,255,220,157,233,101,21,220,157,232,254,157,230,246,157,231,113,220,157,232,101,21,220,157,233,230,157,230,255,220,157,234,121,157,232,239,220,157,233,101,157,232,98,157,230,113,157,230,255,220,157,231,227,157,234,96,157,231,113,220,157,232,254,157,230,113,157,231,108,157,230,233,157,230,230,157,235,230,198,12,220,157,232,254,157,232,243,157,233,113,157,232,238,157,231,113,220,157,233,114,157,232,255,157,231,108,157,230,255,220,157,231,112,157,233,113,157,230,106,157,230,113,220,157,230,230,157,233,241,157,230,230,157,230,251,220,157,234,97,157,232,241,220,157,233,113,157,230,255,157,234,227,157,231,113,220,157,233,233,157,232,241,220,157,233,111,157,233,255,157,230,105,157,232,255,157,231,108,157,230,255,157,232,238,157,231,113,220,157,231,254,157,230,246,157,230,246,157,230,101,157,234,96,157,231,116,157,233,235,220,157,233,108,157,230,113,157,231,111,157,233,235,157,232,243,220,157,234,230,157,230,230,157,234,116,198,12,220,157,230,227,157,232,243,157,233,113,220,157,230,246,157,234,254,157,232,238,220,157,230,108,157,235,232,220,157,233,101,157,233,113,157,232,243,157,234,233,157,233,255,220,157,232,101,157,230,108,157,230,235,220,157,230,225,157,233,255,157,230,235,220,157,234,96,157,231,98,157,233,104,157,233,236,157,231,121,220,157,233,101,157,230,246,157,234,230,157,232,243,157,231,96,157,231,113,220,157,232,98],"ids_no_specials":[12,220,157,235,235,157,230,255,157,233,113,220,157,231,97,157,231,109,220,157,231,254,157,232,254,157,231,114,220,157,233,106,157,232,241,157,231,111,157,232,243,220,157,231,108,157,230,113,157,235,233,157,233,105,220,157,230,233,157,233,255,220,157,233,101,21,220,157,232,254,157,230,246,157,231,113,220,157,232,101,21,220,157,233,230,157,230,255,220,157,234,121,157,232,239,220,157,233,101,157,232,98,157,230,113,157,230,255,220,157,231,227,157,234,96,157,231,113,220,157,232,254,157,230,113,157,231,108,157,230,233,157,230,230,157,235,230,198,12,220,157,232,254,157,232,243,157,233,113,157,232,238,157,231,113,220,157,233,114,157,232,255,157,231,108,157,230,255,220,157,231,112,157,233,113,157,230,106,157,230,113,220,157,230,230,157,233,241,157,230,230,157,230,251,220,157,234,97,157,232,241,220,157,233,113,157,230,255,157,234,227,157,231,113,220,157,233,233,157,232,241,220,157,233,111,157,233,255,157,230,105,157,232,255,157,231,108,157,230,255,157,232,238,157,231,113,220,157,231,254,157,230,246,157,230,246,157,230,101,157,234,96,157,231,116,157,233,235,220,157,233,108,157,230,113,157,231,111,157,233,235,157,232,243,220,157,234,230,157,230,230,157,234,116,198,12,220,157,230,227,157,232,243,157,233,113,220,157,230,246,157,234,254,157,232,238,220,157,230,108,157,235,232,220,157,233,101,157,233,113,157,232,243,157,234,233,157,233,255,220,157,232,101,157,230,108,157,230,235,220,157,230,225,157,233,255,157,230,235,220,157,234,96,157,231,98,157,233,104,157,233,236,157,231,121,220,157,233,101,157,230,246,157,234,230,157,232,243,157,231,96,157,231,113,220,157,232,98],"tokens":["-","Ġ","á","į","į","á","Ī","Ń","á","ĭ","µ","Ġ","á","ī","¤","á","ī","±","Ġ","á","ī","ł","á","Ĭ","ł","á","ī","¶","Ġ","á","ĭ","®","á","Ĭ","ĵ","á","ī","³","á","Ĭ","ķ","Ġ","á","ī","°","á","Ī","µ","á","į","ĭ","á","ĭ","¬","Ġ","á","Ī","ĭ","á","ĭ","Ń","Ġ","á","ĭ","¨","6","Ġ","á","Ĭ","ł","á","Ī","ĺ","á","ī","µ","Ġ","á","Ĭ","¨","6","Ġ","á","ĭ","Ī","á","Ī","Ń","Ġ","á","Į","½","á","Ĭ","ij","Ġ","á","ĭ","¨","á","Ĭ","¥","á","Ī","µ","á","Ī","Ń","Ġ","á","ī","ħ","á","Į","£","á","ī","µ","Ġ","á","Ĭ","ł","á","Ī","µ","á","ī","°","á","Ī","ĭ","á","Ī","Ī","á","į","Ī","Ċ","-","Ġ","á","Ĭ","ł","á","Ĭ","ķ","á","ĭ","µ","á","Ĭ","IJ","á","ī","µ","Ġ","á","ĭ","¶","á","Ĭ","Ń","á","ī","°","á","Ī","Ń","Ġ","á","ī","´","á","ĭ","µ","á","Ī","®","á","Ī","µ","Ġ","á","Ī","Ī","á","ĭ","ĵ","á","Ī","Ī","á","Ī","Ŀ","Ġ","á","Į","¤","á","Ĭ","ĵ","Ġ","á","ĭ","µ","á","Ī","Ń","á","Į","ħ","á","ī","µ","Ġ","á","ĭ","ĭ","á","Ĭ","ĵ","Ġ","á","ĭ","³","á","ĭ","Ń","á","Ī","¬","á","Ĭ","Ń","á","ī","°","á","Ī","Ń","á","Ĭ","IJ","á","ī","µ","Ġ","á","ī","ł","á","Ī","ĺ","á","Ī","ĺ","á","Ī","¨","á","Į","£","á","ī","¸","á","ĭ","į","Ġ","á","ĭ","°","á","Ī","µ","á","ī","³","á","ĭ","į","á","Ĭ","ķ","Ġ","á","Į","Ī","á","Ī","Ī","á","Į","¸","Ċ","-","Ġ","á","Ī","ħ","á","Ĭ","ķ","á","ĭ","µ","Ġ","á","Ī","ĺ","á","Į","ł","á","Ĭ","IJ","Ġ","á","Ī","°","á","į","Ĭ","Ġ","á","ĭ","¨","á","ĭ","µ","á","Ĭ","ķ","á","Į","ĭ","á","ĭ","Ń","Ġ","á","Ĭ","¨","á","Ī","°","á","Ī","į","Ġ","á","Ī","ĥ","á","ĭ","Ń","á","Ī","į","Ġ","á","Į","£","á","ī","¥","á","ĭ","«","á","ĭ","İ","á","ī","½","Ġ","á","ĭ","¨","á","Ī","ĺ","á","Į","Ī","á","Ĭ","ķ","á","ī","£","á","ī","µ","Ġ","á","Ĭ","¥"],"offsets":[[0,1],[1,2],[2,3],[2,3],[2,3],[3,4],[3,4],[3,4],[4,5],[4,5],[4,5],[5,6],[6,7],[6,7],[6,7],[7,8],[7,8],[7,8],[8,9],[9,10],[9,10],[9,10],[10,11],[10,11],[10,11],[11,12],[11,12],[11,12],[12,13],[13,14],[13,14],[13,14],[14,15],[14,15],[14,15],[15,16],[15,16],[15,16],[16,17],[16,17],[16,17],[17,18],[18,19],[18,19],[18,19],[19,20],[19,20],[19,20],[20,21],[20,21],[20,21],[21,22],[21,22],[21,22],[22,23],[23,24],[23,24],[23,24],[24,25],[24,25],[24,25],[25,26],[26,27],[26,27],[26,27],[27,28],[28,29],[29,30],[29,30],[29,30],[30,31],[30,31],[30,31],[31,32],[31,32],[31,32],[32,33],[33,34],[33,34],[33,34],[34,35],[35,36],[36,37],[36,37],[36,37],[37,38],[37,38],[37,38],[38,39],[39,40],[39,40],[39,40],[40,41],[40,41],[40,41],[41,42],[42,43],[42,43],[42,43],[43,44],[43,44],[43,44],[44,45],[44,45],[44,45],[45,46],[45,46],[45,46],[46,47],[47,48],[47,48],[47,48],[48,49],[48,49],[48,49],[49,50],[49,50],[49,50],[50,51],[51,52],[51,52],[51,52],[52,53],[52,53],[52,53],[53,54],[53,54],[53,54],[54,55],[54,55],[54,55],[55,56],[55,56],[55,56],[56,57],[56,57],[56,57],[57,58],[58,59],[59,60],[60,61],[60,61],[60,61],[61,62],[61,62],[61,62],[62,63],[62,63],[62,63],[63,64],[63,64],[63,64],[64,65],[64,65],[64,65],[65,66],[66,67],[66,67],[66,67],[67,68],[67,68],[67,68],[68,69],[68,69],[68,69],[69,70],[69,70],[69,70],[70,71],[71,72],[71,72],[71,72],[72,73],[72,73],[72,73],[73,74],[73,74],[73,74],[74,75],[74,75],[74,75],[75,76],[76,77],[76,77],[76,77],[77,78],[77,78],[77,78],[78,79],[78,79],[78,79],[79,80],[79,80],[79,80],[80,81],[81,82],[81,82],[81,82],[82,83],[82,83],[82,83],[83,84],[84,85],[84,85],[84,85],[85,86],[85,86],[85,86],[86,87],[86,87],[86,87],[87,88],[87,88],[87,88],[88,89],[89,90],[89,90],[89,90],[90,91],[90,91],[90,91],[91,92],[92,93],[92,93],[92,93],[93,94],[93,94],[93,94],[94,95],[94,95],[94,95],[95,96],[95,96],[95,96],[96,97],[96,97],[96,97],[97,98],[97,98],[97,98],[98,99],[98,99],[98,99],[99,100],[99,100],[99,100],[100,101],[101,102],[101,102],[101,102],[102,103],[102,103],[102,103],[103,104],[103,104],[103,104],[104,105],[104,105],[104,105],[105,106],[105,106],[105,106],[106,107],[106,107],[106,107],[107,108],[107,108],[107,108],[108,109],[109,110],[109,110],[109,110],[110,111],[110,111],[110,111],[111,112],[111,112],[111,112],[112,113],[112,113],[112,113],[113,114],[113,114],[113,114],[114,115],[115,116],[115,116],[115,116],[116,117],[116,117],[116,117],[117,118],[117,118],[117,118],[118,119],[119,120],[120,121],[121,122],[121,122],[121,122],[122,123],[122,123],[122,123],[123,124],[123,124],[123,124],[124,125],[125,126],[125,126],[125,126],[126,127],[126,127],[126,127],[127,128],[127,128],[127,128],[128,129],[129,130],[129,130],[129,130],[130,131],[130,131],[130,131],[131,132],[132,133],[132,133],[132,133],[133,134],[133,134],[133,134],[134,135],[134,135],[134,135],[135,136],[135,136],[135,136],[136,137],[136,137],[136,137],[137,138],[138,139],[138,139],[138,139],[139,140],[139,140],[139,140],[140,141],[140,141],[140,141],[141,142],[142,143],[142,143],[142,143],[143,144],[143,144],[143,144],[144,145],[144,145],[144,145],[145,146],[146,147],[146,147],[146,147],[147,148],[147,148],[147,148],[148,149],[148,149],[148,149],[149,150],[149,150],[149,150],[150,151],[150,151],[150,151],[151,152],[152,153],[152,153],[152,153],[153,154],[153,154],[153,154],[154,155],[154,155],[154,155],[155,156],[155,156],[155,156],[156,157],[156,157],[156,157],[157,158],[157,158],[157,158],[158,159],[159,160],[159,160],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,7,7,7,7,8,9,9,9,9,9,9,9,9,9,9,10,10,10,10,11,12,12,12,12,12,12,12,13,13,13,13,13,13,13,14,14,14,14,14,14,14,14,14,14,14,14,14,15,15,15,15,15,15,15,15,15,15,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,17,18,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,20,20,20,20,20,20,20,20,20,20,20,20,20,21,21,21,21,21,21,21,21,21,21,21,21,21,22,22,22,22,22,22,22,22,22,22,22,22,22,23,23,23,23,23,23,23,24,24,24,24,24,24,24,24,24,24,24,24,24,25,25,25,25,25,25,25,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,29,29,29,29,29,29,29,29,29,29,30,31,32,32,32,32,32,32,32,32,32,32,33,33,33,33,33,33,33,33,33,33,34,34,34,34,34,34,34,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,36,36,36,36,36,36,36,36,36,36,37,37,37,37,37,37,37,37,37,37,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,40,40,40,40],"decoded":"- ፍርድ ቤቱ በአቶ ዮናታን ተስፋዬ ላይ የ6 አመት ከ6 ወር ጽኑ የእስር ቅጣት አስተላለፈ\n- አንድነት ዶክተር ቴድሮስ ለዓለም ጤና ድርጅት ዋና ዳይሬክተርነት በመመረጣቸው ደስታውን ገለጸ\n- ህንድ መጠነ ሰፊ የድንጋይ ከሰል ሃይል ጣብያዎች የመገንባት እ","decoded_with_specials":"- ፍርድ ቤቱ በአቶ ዮናታን ተስፋዬ ላይ የ6 አመት ከ6 ወር ጽኑ የእስር ቅጣት አስተላለፈ\n- አንድነት ዶክተር ቴድሮስ ለዓለም ጤና ድርጅት ዋና ዳይሬክተርነት በመመረጣቸው ደስታውን ገለጸ\n- ህንድ መጠነ ሰፊ የድንጋይ ከሰል ሃይል ጣብያዎች የመገንባት እ"} +{"kind":"sample","source":"fixtures/lang/arb_Arab.txt[:160]","text":"[بالصور] ترقوميا : الموت يطفىء باقة ورد قيد التفتح !!\nالخليل – دوت كوم – من غسان عبد الحميد - لم يخطر في بال أهالي \"ترقوميا\" التي انخلع قلبها وهي تودع التراب جث","ids":[58,148,101,148,100,149,226,148,113,149,230,148,109,60,220,148,103,148,109,149,224,149,230,149,227,149,232,148,100,549,220,148,100,149,226,149,227,149,230,148,103,220,149,232,148,115,149,223,149,231,148,94,220,148,101,148,100,149,224,148,102,220,149,230,148,109,148,107,220,149,224,149,232,148,107,220,148,100,149,226,148,103,149,223,148,103,148,255,753,0,198,148,100,149,226,148,106,149,226,149,232,149,226,1365,220,148,107,149,230,148,103,220,149,225,149,230,149,227,1365,220,149,227,149,228,220,148,118,148,111,148,100,149,228,220,148,117,148,101,148,107,220,148,100,149,226,148,255,149,227,149,232,148,107,481,220,149,226,149,227,220,149,232,148,106,148,115,148,109,220,149,223,149,232,220,148,101,148,100,149,226,220,148,96,149,229,148,100,149,226,149,232,330,148,103,148,109,149,224,149,230,149,227,149,232,148,100,1,220,148,100,149,226,148,103,149,232,220,148,100,149,228,148,106,149,226,148,117,220,149,224,149,226,148,101,149,229,148,100,220,149,230,149,229,149,232,220,148,103,149,230,148,107,148,117,220,148,100,149,226,148,103,148,109,148,100,148,101,220,148,105,148,104],"ids_no_specials":[58,148,101,148,100,149,226,148,113,149,230,148,109,60,220,148,103,148,109,149,224,149,230,149,227,149,232,148,100,549,220,148,100,149,226,149,227,149,230,148,103,220,149,232,148,115,149,223,149,231,148,94,220,148,101,148,100,149,224,148,102,220,149,230,148,109,148,107,220,149,224,149,232,148,107,220,148,100,149,226,148,103,149,223,148,103,148,255,753,0,198,148,100,149,226,148,106,149,226,149,232,149,226,1365,220,148,107,149,230,148,103,220,149,225,149,230,149,227,1365,220,149,227,149,228,220,148,118,148,111,148,100,149,228,220,148,117,148,101,148,107,220,148,100,149,226,148,255,149,227,149,232,148,107,481,220,149,226,149,227,220,149,232,148,106,148,115,148,109,220,149,223,149,232,220,148,101,148,100,149,226,220,148,96,149,229,148,100,149,226,149,232,330,148,103,148,109,149,224,149,230,149,227,149,232,148,100,1,220,148,100,149,226,148,103,149,232,220,148,100,149,228,148,106,149,226,148,117,220,149,224,149,226,148,101,149,229,148,100,220,149,230,149,229,149,232,220,148,103,149,230,148,107,148,117,220,148,100,149,226,148,103,148,109,148,100,148,101,220,148,105,148,104],"tokens":["[","Ø","¨","Ø","§","Ù","Ħ","Ø","µ","Ù","Ī","Ø","±","]","Ġ","Ø","ª","Ø","±","Ù","Ĥ","Ù","Ī","Ù","ħ","Ù","Ĭ","Ø","§","Ġ:","Ġ","Ø","§","Ù","Ħ","Ù","ħ","Ù","Ī","Ø","ª","Ġ","Ù","Ĭ","Ø","·","Ù","ģ","Ù","ī","Ø","¡","Ġ","Ø","¨","Ø","§","Ù","Ĥ","Ø","©","Ġ","Ù","Ī","Ø","±","Ø","¯","Ġ","Ù","Ĥ","Ù","Ĭ","Ø","¯","Ġ","Ø","§","Ù","Ħ","Ø","ª","Ù","ģ","Ø","ª","Ø","Ń","Ġ!","!","Ċ","Ø","§","Ù","Ħ","Ø","®","Ù","Ħ","Ù","Ĭ","Ù","Ħ","ĠâĢĵ","Ġ","Ø","¯","Ù","Ī","Ø","ª","Ġ","Ù","ĥ","Ù","Ī","Ù","ħ","ĠâĢĵ","Ġ","Ù","ħ","Ù","Ĩ","Ġ","Ø","º","Ø","³","Ø","§","Ù","Ĩ","Ġ","Ø","¹","Ø","¨","Ø","¯","Ġ","Ø","§","Ù","Ħ","Ø","Ń","Ù","ħ","Ù","Ĭ","Ø","¯","Ġ-","Ġ","Ù","Ħ","Ù","ħ","Ġ","Ù","Ĭ","Ø","®","Ø","·","Ø","±","Ġ","Ù","ģ","Ù","Ĭ","Ġ","Ø","¨","Ø","§","Ù","Ħ","Ġ","Ø","£","Ù","ĩ","Ø","§","Ù","Ħ","Ù","Ĭ","Ġ\"","Ø","ª","Ø","±","Ù","Ĥ","Ù","Ī","Ù","ħ","Ù","Ĭ","Ø","§","\"","Ġ","Ø","§","Ù","Ħ","Ø","ª","Ù","Ĭ","Ġ","Ø","§","Ù","Ĩ","Ø","®","Ù","Ħ","Ø","¹","Ġ","Ù","Ĥ","Ù","Ħ","Ø","¨","Ù","ĩ","Ø","§","Ġ","Ù","Ī","Ù","ĩ","Ù","Ĭ","Ġ","Ø","ª","Ù","Ī","Ø","¯","Ø","¹","Ġ","Ø","§","Ù","Ħ","Ø","ª","Ø","±","Ø","§","Ø","¨","Ġ","Ø","¬","Ø","«"],"offsets":[[0,1],[1,2],[1,2],[2,3],[2,3],[3,4],[3,4],[4,5],[4,5],[5,6],[5,6],[6,7],[6,7],[7,8],[8,9],[9,10],[9,10],[10,11],[10,11],[11,12],[11,12],[12,13],[12,13],[13,14],[13,14],[14,15],[14,15],[15,16],[15,16],[16,18],[18,19],[19,20],[19,20],[20,21],[20,21],[21,22],[21,22],[22,23],[22,23],[23,24],[23,24],[24,25],[25,26],[25,26],[26,27],[26,27],[27,28],[27,28],[28,29],[28,29],[29,30],[29,30],[30,31],[31,32],[31,32],[32,33],[32,33],[33,34],[33,34],[34,35],[34,35],[35,36],[36,37],[36,37],[37,38],[37,38],[38,39],[38,39],[39,40],[40,41],[40,41],[41,42],[41,42],[42,43],[42,43],[43,44],[44,45],[44,45],[45,46],[45,46],[46,47],[46,47],[47,48],[47,48],[48,49],[48,49],[49,50],[49,50],[50,52],[52,53],[53,54],[54,55],[54,55],[55,56],[55,56],[56,57],[56,57],[57,58],[57,58],[58,59],[58,59],[59,60],[59,60],[60,62],[62,63],[63,64],[63,64],[64,65],[64,65],[65,66],[65,66],[66,67],[67,68],[67,68],[68,69],[68,69],[69,70],[69,70],[70,72],[72,73],[73,74],[73,74],[74,75],[74,75],[75,76],[76,77],[76,77],[77,78],[77,78],[78,79],[78,79],[79,80],[79,80],[80,81],[81,82],[81,82],[82,83],[82,83],[83,84],[83,84],[84,85],[85,86],[85,86],[86,87],[86,87],[87,88],[87,88],[88,89],[88,89],[89,90],[89,90],[90,91],[90,91],[91,93],[93,94],[94,95],[94,95],[95,96],[95,96],[96,97],[97,98],[97,98],[98,99],[98,99],[99,100],[99,100],[100,101],[100,101],[101,102],[102,103],[102,103],[103,104],[103,104],[104,105],[105,106],[105,106],[106,107],[106,107],[107,108],[107,108],[108,109],[109,110],[109,110],[110,111],[110,111],[111,112],[111,112],[112,113],[112,113],[113,114],[113,114],[114,116],[116,117],[116,117],[117,118],[117,118],[118,119],[118,119],[119,120],[119,120],[120,121],[120,121],[121,122],[121,122],[122,123],[122,123],[123,124],[124,125],[125,126],[125,126],[126,127],[126,127],[127,128],[127,128],[128,129],[128,129],[129,130],[130,131],[130,131],[131,132],[131,132],[132,133],[132,133],[133,134],[133,134],[134,135],[134,135],[135,136],[136,137],[136,137],[137,138],[137,138],[138,139],[138,139],[139,140],[139,140],[140,141],[140,141],[141,142],[142,143],[142,143],[143,144],[143,144],[144,145],[144,145],[145,146],[146,147],[146,147],[147,148],[147,148],[148,149],[148,149],[149,150],[149,150],[150,151],[151,152],[151,152],[152,153],[152,153],[153,154],[153,154],[154,155],[154,155],[155,156],[155,156],[156,157],[156,157],[157,158],[158,159],[158,159],[159,160],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,3,4,4,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,8,8,8,8,8,8,8,9,9,9,9,9,9,9,9,9,9,9,9,9,10,10,10,11,11,11,11,11,11,11,11,11,11,11,11,12,13,13,13,13,13,13,13,14,14,14,14,14,14,14,15,16,16,16,16,16,17,17,17,17,17,17,17,17,17,18,18,18,18,18,18,18,19,19,19,19,19,19,19,19,19,19,19,19,19,20,21,21,21,21,21,22,22,22,22,22,22,22,22,22,23,23,23,23,23,24,24,24,24,24,24,24,25,25,25,25,25,25,25,25,25,25,25,26,27,27,27,27,27,27,27,27,27,27,27,27,27,27,28,29,29,29,29,29,29,29,29,29,30,30,30,30,30,30,30,30,30,30,30,31,31,31,31,31,31,31,31,31,31,31,32,32,32,32,32,32,32,33,33,33,33,33,33,33,33,33,34,34,34,34,34,34,34,34,34,34,34,34,34,35,35,35,35,35],"decoded":"[بالصور] ترقوميا : الموت يطفىء باقة ورد قيد التفتح !!\nالخليل – دوت كوم – من غسان عبد الحميد - لم يخطر في بال أهالي \"ترقوميا\" التي انخلع قلبها وهي تودع التراب جث","decoded_with_specials":"[بالصور] ترقوميا : الموت يطفىء باقة ورد قيد التفتح !!\nالخليل – دوت كوم – من غسان عبد الحميد - لم يخطر في بال أهالي \"ترقوميا\" التي انخلع قلبها وهي تودع التراب جث"} +{"kind":"sample","source":"fixtures/lang/ben_Beng.txt[:160]","text":"গ্রহ নীহারিকা\nগ্রহ নীহারিকা (ইংরেজি ভাষায়: Planetary nebula) এক বিশেষ ধরনের গ্যাসীয় নীহারিকা। যেসব তারার ভর কম, নির্দিষ্টভাবে বলতে গেলে যেসব তারার ভর সূর্যের ","ids":[156,99,245,156,100,235,156,99,108,156,99,117,220,156,99,101,156,100,222,156,99,117,156,99,122,156,99,108,156,99,123,156,99,243,156,99,122,198,156,99,245,156,100,235,156,99,108,156,99,117,220,156,99,101,156,100,222,156,99,117,156,99,122,156,99,108,156,99,123,156,99,243,156,99,122,320,156,99,229,156,99,224,156,99,108,156,100,229,156,99,250,156,99,123,220,156,99,255,156,99,122,156,99,115,156,99,122,156,99,107,156,99,120,25,1818,276,295,658,834,65,360,64,8,220,156,99,237,156,99,243,220,156,99,105,156,99,123,156,99,114,156,100,229,156,99,115,220,156,99,100,156,99,108,156,99,101,156,100,229,156,99,108,220,156,99,245,156,100,235,156,99,107,156,99,122,156,99,116,156,100,222,156,99,107,156,99,120,220,156,99,101,156,100,222,156,99,117,156,99,122,156,99,108,156,99,123,156,99,243,156,99,122,156,98,97,220,156,99,107,156,100,229,156,99,116,156,99,105,220,156,99,97,156,99,122,156,99,108,156,99,122,156,99,108,220,156,99,255,156,99,108,220,156,99,243,156,99,106,11,220,156,99,101,156,99,123,156,99,108,156,100,235,156,99,99,156,99,123,156,99,115,156,100,235,156,99,253,156,99,255,156,99,122,156,99,105,156,100,229,220,156,99,105,156,99,110,156,99,97,156,100,229,220,156,99,245,156,100,229,156,99,110,156,100,229,220,156,99,107,156,100,229,156,99,116,156,99,105,220,156,99,97,156,99,122,156,99,108,156,99,122,156,99,108,220,156,99,255,156,99,108,220,156,99,116,156,100,224,156,99,108,156,100,235,156,99,107,156,100,229,156,99,108,220],"ids_no_specials":[156,99,245,156,100,235,156,99,108,156,99,117,220,156,99,101,156,100,222,156,99,117,156,99,122,156,99,108,156,99,123,156,99,243,156,99,122,198,156,99,245,156,100,235,156,99,108,156,99,117,220,156,99,101,156,100,222,156,99,117,156,99,122,156,99,108,156,99,123,156,99,243,156,99,122,320,156,99,229,156,99,224,156,99,108,156,100,229,156,99,250,156,99,123,220,156,99,255,156,99,122,156,99,115,156,99,122,156,99,107,156,99,120,25,1818,276,295,658,834,65,360,64,8,220,156,99,237,156,99,243,220,156,99,105,156,99,123,156,99,114,156,100,229,156,99,115,220,156,99,100,156,99,108,156,99,101,156,100,229,156,99,108,220,156,99,245,156,100,235,156,99,107,156,99,122,156,99,116,156,100,222,156,99,107,156,99,120,220,156,99,101,156,100,222,156,99,117,156,99,122,156,99,108,156,99,123,156,99,243,156,99,122,156,98,97,220,156,99,107,156,100,229,156,99,116,156,99,105,220,156,99,97,156,99,122,156,99,108,156,99,122,156,99,108,220,156,99,255,156,99,108,220,156,99,243,156,99,106,11,220,156,99,101,156,99,123,156,99,108,156,100,235,156,99,99,156,99,123,156,99,115,156,100,235,156,99,253,156,99,255,156,99,122,156,99,105,156,100,229,220,156,99,105,156,99,110,156,99,97,156,100,229,220,156,99,245,156,100,229,156,99,110,156,100,229,220,156,99,107,156,100,229,156,99,116,156,99,105,220,156,99,97,156,99,122,156,99,108,156,99,122,156,99,108,220,156,99,255,156,99,108,220,156,99,116,156,100,224,156,99,108,156,100,235,156,99,107,156,100,229,156,99,108,220],"tokens":["à","¦","Ĺ","à","§","į","à","¦","°","à","¦","¹","Ġ","à","¦","¨","à","§","Ģ","à","¦","¹","à","¦","¾","à","¦","°","à","¦","¿","à","¦","ķ","à","¦","¾","Ċ","à","¦","Ĺ","à","§","į","à","¦","°","à","¦","¹","Ġ","à","¦","¨","à","§","Ģ","à","¦","¹","à","¦","¾","à","¦","°","à","¦","¿","à","¦","ķ","à","¦","¾","Ġ(","à","¦","ĩ","à","¦","Ĥ","à","¦","°","à","§","ĩ","à","¦","ľ","à","¦","¿","Ġ","à","¦","Ń","à","¦","¾","à","¦","·","à","¦","¾","à","¦","¯","à","¦","¼",":","ĠPl","an","et","ary","Ġne","b","ul","a",")","Ġ","à","¦","ı","à","¦","ķ","Ġ","à","¦","¬","à","¦","¿","à","¦","¶","à","§","ĩ","à","¦","·","Ġ","à","¦","§","à","¦","°","à","¦","¨","à","§","ĩ","à","¦","°","Ġ","à","¦","Ĺ","à","§","į","à","¦","¯","à","¦","¾","à","¦","¸","à","§","Ģ","à","¦","¯","à","¦","¼","Ġ","à","¦","¨","à","§","Ģ","à","¦","¹","à","¦","¾","à","¦","°","à","¦","¿","à","¦","ķ","à","¦","¾","à","¥","¤","Ġ","à","¦","¯","à","§","ĩ","à","¦","¸","à","¦","¬","Ġ","à","¦","¤","à","¦","¾","à","¦","°","à","¦","¾","à","¦","°","Ġ","à","¦","Ń","à","¦","°","Ġ","à","¦","ķ","à","¦","®",",","Ġ","à","¦","¨","à","¦","¿","à","¦","°","à","§","į","à","¦","¦","à","¦","¿","à","¦","·","à","§","į","à","¦","Ł","à","¦","Ń","à","¦","¾","à","¦","¬","à","§","ĩ","Ġ","à","¦","¬","à","¦","²","à","¦","¤","à","§","ĩ","Ġ","à","¦","Ĺ","à","§","ĩ","à","¦","²","à","§","ĩ","Ġ","à","¦","¯","à","§","ĩ","à","¦","¸","à","¦","¬","Ġ","à","¦","¤","à","¦","¾","à","¦","°","à","¦","¾","à","¦","°","Ġ","à","¦","Ń","à","¦","°","Ġ","à","¦","¸","à","§","Ĥ","à","¦","°","à","§","į","à","¦","¯","à","§","ĩ","à","¦","°","Ġ"],"offsets":[[0,1],[0,1],[0,1],[1,2],[1,2],[1,2],[2,3],[2,3],[2,3],[3,4],[3,4],[3,4],[4,5],[5,6],[5,6],[5,6],[6,7],[6,7],[6,7],[7,8],[7,8],[7,8],[8,9],[8,9],[8,9],[9,10],[9,10],[9,10],[10,11],[10,11],[10,11],[11,12],[11,12],[11,12],[12,13],[12,13],[12,13],[13,14],[14,15],[14,15],[14,15],[15,16],[15,16],[15,16],[16,17],[16,17],[16,17],[17,18],[17,18],[17,18],[18,19],[19,20],[19,20],[19,20],[20,21],[20,21],[20,21],[21,22],[21,22],[21,22],[22,23],[22,23],[22,23],[23,24],[23,24],[23,24],[24,25],[24,25],[24,25],[25,26],[25,26],[25,26],[26,27],[26,27],[26,27],[27,29],[29,30],[29,30],[29,30],[30,31],[30,31],[30,31],[31,32],[31,32],[31,32],[32,33],[32,33],[32,33],[33,34],[33,34],[33,34],[34,35],[34,35],[34,35],[35,36],[36,37],[36,37],[36,37],[37,38],[37,38],[37,38],[38,39],[38,39],[38,39],[39,40],[39,40],[39,40],[40,41],[40,41],[40,41],[41,42],[41,42],[41,42],[42,43],[43,46],[46,48],[48,50],[50,53],[53,56],[56,57],[57,59],[59,60],[60,61],[61,62],[62,63],[62,63],[62,63],[63,64],[63,64],[63,64],[64,65],[65,66],[65,66],[65,66],[66,67],[66,67],[66,67],[67,68],[67,68],[67,68],[68,69],[68,69],[68,69],[69,70],[69,70],[69,70],[70,71],[71,72],[71,72],[71,72],[72,73],[72,73],[72,73],[73,74],[73,74],[73,74],[74,75],[74,75],[74,75],[75,76],[75,76],[75,76],[76,77],[77,78],[77,78],[77,78],[78,79],[78,79],[78,79],[79,80],[79,80],[79,80],[80,81],[80,81],[80,81],[81,82],[81,82],[81,82],[82,83],[82,83],[82,83],[83,84],[83,84],[83,84],[84,85],[84,85],[84,85],[85,86],[86,87],[86,87],[86,87],[87,88],[87,88],[87,88],[88,89],[88,89],[88,89],[89,90],[89,90],[89,90],[90,91],[90,91],[90,91],[91,92],[91,92],[91,92],[92,93],[92,93],[92,93],[93,94],[93,94],[93,94],[94,95],[94,95],[94,95],[95,96],[96,97],[96,97],[96,97],[97,98],[97,98],[97,98],[98,99],[98,99],[98,99],[99,100],[99,100],[99,100],[100,101],[101,102],[101,102],[101,102],[102,103],[102,103],[102,103],[103,104],[103,104],[103,104],[104,105],[104,105],[104,105],[105,106],[105,106],[105,106],[106,107],[107,108],[107,108],[107,108],[108,109],[108,109],[108,109],[109,110],[110,111],[110,111],[110,111],[111,112],[111,112],[111,112],[112,113],[113,114],[114,115],[114,115],[114,115],[115,116],[115,116],[115,116],[116,117],[116,117],[116,117],[117,118],[117,118],[117,118],[118,119],[118,119],[118,119],[119,120],[119,120],[119,120],[120,121],[120,121],[120,121],[121,122],[121,122],[121,122],[122,123],[122,123],[122,123],[123,124],[123,124],[123,124],[124,125],[124,125],[124,125],[125,126],[125,126],[125,126],[126,127],[126,127],[126,127],[127,128],[128,129],[128,129],[128,129],[129,130],[129,130],[129,130],[130,131],[130,131],[130,131],[131,132],[131,132],[131,132],[132,133],[133,134],[133,134],[133,134],[134,135],[134,135],[134,135],[135,136],[135,136],[135,136],[136,137],[136,137],[136,137],[137,138],[138,139],[138,139],[138,139],[139,140],[139,140],[139,140],[140,141],[140,141],[140,141],[141,142],[141,142],[141,142],[142,143],[143,144],[143,144],[143,144],[144,145],[144,145],[144,145],[145,146],[145,146],[145,146],[146,147],[146,147],[146,147],[147,148],[147,148],[147,148],[148,149],[149,150],[149,150],[149,150],[150,151],[150,151],[150,151],[151,152],[152,153],[152,153],[152,153],[153,154],[153,154],[153,154],[154,155],[154,155],[154,155],[155,156],[155,156],[155,156],[156,157],[156,157],[156,157],[157,158],[157,158],[157,158],[158,159],[158,159],[158,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,1,1,1,1,1,1,1,1,1,2,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,5,5,5,5,5,5,6,6,6,6,7,7,7,8,8,8,8,8,8,8,8,8,9,9,9,9,10,10,10,10,10,10,11,11,11,11,11,11,12,12,12,12,12,12,13,13,13,14,15,15,15,16,16,16,16,16,16,17,17,17,17,17,17,18,18,18,19,19,19,19,20,20,20,20,20,20,21,21,21,21,21,21,22,22,22,22,23,23,23,23,24,24,24,24,25,26,26,26,26,26,26,26,27,27,27,27,28,28,28,28,28,28,29,29,29,29,29,29,30,30,30,30,30,30,30,30,30,30,31,31,31,31,31,31,32,32,32,32,33,33,33,33,33,33,34,34,34,34,34,34,35,35,35,35,35,35,36,36,36,37,37,37,37,38,38,38,38,38,38,39,39,39,39,39,39,40,40,40,40,40,40,41,41,41,41,41,41,42,42,42,42,43,43,43,43,43,43,43,43,43,44,44,44,44,45,45,45,45,45,45,46,46,46,46,46,46,47,47,47,47,47,47,47,48,48,48,48,48,48,48,49,50,50,50,50,51,51,51,51,51,51,52,52,52,52,52,52,53,53,53,53,53,53,54,54,54,54,54,54,54,54,54,55,55,55,55,55,55,56,56,56,57,57,57,57,57,57,57,57,57,57,58,58,58,59,59,59,59,60,60,60,60,60,60,61,61,61,62,62,62,62,63,63,63,63,63,63,63,63,63,64,64,64,64,65,65,65,65,65,65,66,66,66,66,66,66,67,67,67,67,67,67,67,68,68,68,68,69,69,69,69,69,69,70,70,70,70,70,70,71,71,71,71,71,71,72],"decoded":"গ্রহ নীহারিকা\nগ্রহ নীহারিকা (ইংরেজি ভাষায়: Planetary nebula) এক বিশেষ ধরনের গ্যাসীয় নীহারিকা। যেসব তারার ভর কম, নির্দিষ্টভাবে বলতে গেলে যেসব তারার ভর সূর্যের ","decoded_with_specials":"গ্রহ নীহারিকা\nগ্রহ নীহারিকা (ইংরেজি ভাষায়: Planetary nebula) এক বিশেষ ধরনের গ্যাসীয় নীহারিকা। যেসব তারার ভর কম, নির্দিষ্টভাবে বলতে গেলে যেসব তারার ভর সূর্যের "} +{"kind":"sample","source":"fixtures/lang/cmn_Hani.txt[:160]","text":"強力建議大家未來盡量避免英航阿~~~\n班機原訂航程\n6/9 台北→香港\n6/9 香港→倫敦 (原訂23:15起飛，4:50am抵達)\n6/10 倫敦→斯德哥爾摩 (原訂7:40am起飛，11:05am抵達)\n就在第二段，英航no.25班機在香港機場兩度離開閘口又兩度返航\n第一次因有旅客身體不適 (好，不怪他，消耗時間也","ids":[161,120,115,161,232,249,161,119,118,164,255,108,161,97,100,161,106,114,162,250,103,160,122,228,163,249,94,165,229,237,165,223,123,161,227,235,164,233,109,164,230,103,165,246,123,93,93,93,198,163,237,255,162,102,253,161,236,253,164,101,224,164,230,103,163,101,233,198,21,14,24,220,161,237,108,161,234,245,158,228,240,165,99,247,162,116,107,198,21,14,24,220,165,99,247,162,116,107,158,228,240,161,222,104,162,243,99,320,161,236,253,164,101,224,17,18,25,16,20,164,113,115,165,96,249,1540,234,19,25,20,15,309,162,232,113,165,223,242,340,21,14,16,15,220,161,222,104,162,243,99,158,228,240,162,244,107,161,122,115,161,241,98,163,230,122,162,239,102,320,161,236,253,164,101,224,22,25,19,15,309,164,113,115,165,96,249,1540,234,16,16,25,15,20,309,162,232,113,165,223,242,340,161,108,109,161,250,101,163,105,105,160,118,234,162,106,113,1540,234,164,233,109,164,230,103,77,78,13,17,20,163,237,255,162,102,253,161,250,101,165,99,247,162,116,107,162,102,253,161,254,112,161,227,102,161,118,99,165,249,95,165,244,233,165,244,246,161,237,96,161,237,230,161,227,102,161,118,99,164,123,242,164,230,103,198,163,105,105,160,116,222,162,105,94,161,249,254,162,250,231,162,245,227,161,106,95,164,118,104,165,104,242,160,116,235,165,223,102,320,161,98,121,1540,234,160,116,235,162,222,103,160,119,244,1540,234,162,114,230,164,222,245,162,247,224,165,244,241,160,117,253],"ids_no_specials":[161,120,115,161,232,249,161,119,118,164,255,108,161,97,100,161,106,114,162,250,103,160,122,228,163,249,94,165,229,237,165,223,123,161,227,235,164,233,109,164,230,103,165,246,123,93,93,93,198,163,237,255,162,102,253,161,236,253,164,101,224,164,230,103,163,101,233,198,21,14,24,220,161,237,108,161,234,245,158,228,240,165,99,247,162,116,107,198,21,14,24,220,165,99,247,162,116,107,158,228,240,161,222,104,162,243,99,320,161,236,253,164,101,224,17,18,25,16,20,164,113,115,165,96,249,1540,234,19,25,20,15,309,162,232,113,165,223,242,340,21,14,16,15,220,161,222,104,162,243,99,158,228,240,162,244,107,161,122,115,161,241,98,163,230,122,162,239,102,320,161,236,253,164,101,224,22,25,19,15,309,164,113,115,165,96,249,1540,234,16,16,25,15,20,309,162,232,113,165,223,242,340,161,108,109,161,250,101,163,105,105,160,118,234,162,106,113,1540,234,164,233,109,164,230,103,77,78,13,17,20,163,237,255,162,102,253,161,250,101,165,99,247,162,116,107,162,102,253,161,254,112,161,227,102,161,118,99,165,249,95,165,244,233,165,244,246,161,237,96,161,237,230,161,227,102,161,118,99,164,123,242,164,230,103,198,163,105,105,160,116,222,162,105,94,161,249,254,162,250,231,162,245,227,161,106,95,164,118,104,165,104,242,160,116,235,165,223,102,320,161,98,121,1540,234,160,116,235,162,222,103,160,119,244,1540,234,162,114,230,164,222,245,162,247,224,165,244,241,160,117,253],"tokens":["å","¼","·","å","Ĭ","Ľ","å","»","º","è","Ń","°","å","¤","§","å","®","¶","æ","ľ","ª","ä","¾","Ĩ","ç","Ľ","¡","é","ĩ","ı","é","ģ","¿","å","ħ","į","è","ĭ","±","è","Ī","ª","é","ĺ","¿","~","~","~","Ċ","ç","ı","Ń","æ","©","Ł","å","İ","Ł","è","¨","Ĥ","è","Ī","ª","ç","¨","ĭ","Ċ","6","/","9","Ġ","å","ı","°","å","Į","Ĺ","â","Ĩ","Ĵ","é","¦","Ļ","æ","¸","¯","Ċ","6","/","9","Ġ","é","¦","Ļ","æ","¸","¯","â","Ĩ","Ĵ","å","Ģ","«","æ","ķ","¦","Ġ(","å","İ","Ł","è","¨","Ĥ","2","3",":","1","5","è","µ","·","é","£","Ľ","ï¼","Į","4",":","5","0","am","æ","Ĭ","µ","é","ģ","Ķ",")Ċ","6","/","1","0","Ġ","å","Ģ","«","æ","ķ","¦","â","Ĩ","Ĵ","æ","ĸ","¯","å","¾","·","å","ĵ","¥","ç","Ī","¾","æ","ij","©","Ġ(","å","İ","Ł","è","¨","Ĥ","7",":","4","0","am","è","µ","·","é","£","Ľ","ï¼","Į","1","1",":","0","5","am","æ","Ĭ","µ","é","ģ","Ķ",")Ċ","å","°","±","å","ľ","¨","ç","¬","¬","ä","º","Į","æ","®","µ","ï¼","Į","è","ĭ","±","è","Ī","ª","n","o",".","2","5","ç","ı","Ń","æ","©","Ł","å","ľ","¨","é","¦","Ļ","æ","¸","¯","æ","©","Ł","å","ł","´","å","ħ","©","å","º","¦","é","Ľ","¢","é","ĸ","ĭ","é","ĸ","ĺ","å","ı","£","å","ı","Ī","å","ħ","©","å","º","¦","è","¿","Ķ","è","Ī","ª","Ċ","ç","¬","¬","ä","¸","Ģ","æ","¬","¡","å","Ľ","ł","æ","ľ","ī","æ","Ĺ","ħ","å","®","¢","è","º","«","é","«","Ķ","ä","¸","į","é","ģ","©","Ġ(","å","¥","½","ï¼","Į","ä","¸","į","æ","Ģ","ª","ä","»","ĸ","ï¼","Į","æ","¶","Ī","è","Ģ","Ĺ","æ","Ļ","Ĥ","é","ĸ","ĵ","ä","¹","Ł"],"offsets":[[0,1],[0,1],[0,1],[1,2],[1,2],[1,2],[2,3],[2,3],[2,3],[3,4],[3,4],[3,4],[4,5],[4,5],[4,5],[5,6],[5,6],[5,6],[6,7],[6,7],[6,7],[7,8],[7,8],[7,8],[8,9],[8,9],[8,9],[9,10],[9,10],[9,10],[10,11],[10,11],[10,11],[11,12],[11,12],[11,12],[12,13],[12,13],[12,13],[13,14],[13,14],[13,14],[14,15],[14,15],[14,15],[15,16],[16,17],[17,18],[18,19],[19,20],[19,20],[19,20],[20,21],[20,21],[20,21],[21,22],[21,22],[21,22],[22,23],[22,23],[22,23],[23,24],[23,24],[23,24],[24,25],[24,25],[24,25],[25,26],[26,27],[27,28],[28,29],[29,30],[30,31],[30,31],[30,31],[31,32],[31,32],[31,32],[32,33],[32,33],[32,33],[33,34],[33,34],[33,34],[34,35],[34,35],[34,35],[35,36],[36,37],[37,38],[38,39],[39,40],[40,41],[40,41],[40,41],[41,42],[41,42],[41,42],[42,43],[42,43],[42,43],[43,44],[43,44],[43,44],[44,45],[44,45],[44,45],[45,47],[47,48],[47,48],[47,48],[48,49],[48,49],[48,49],[49,50],[50,51],[51,52],[52,53],[53,54],[54,55],[54,55],[54,55],[55,56],[55,56],[55,56],[56,57],[56,57],[57,58],[58,59],[59,60],[60,61],[61,63],[63,64],[63,64],[63,64],[64,65],[64,65],[64,65],[65,67],[67,68],[68,69],[69,70],[70,71],[71,72],[72,73],[72,73],[72,73],[73,74],[73,74],[73,74],[74,75],[74,75],[74,75],[75,76],[75,76],[75,76],[76,77],[76,77],[76,77],[77,78],[77,78],[77,78],[78,79],[78,79],[78,79],[79,80],[79,80],[79,80],[80,82],[82,83],[82,83],[82,83],[83,84],[83,84],[83,84],[84,85],[85,86],[86,87],[87,88],[88,90],[90,91],[90,91],[90,91],[91,92],[91,92],[91,92],[92,93],[92,93],[93,94],[94,95],[95,96],[96,97],[97,98],[98,100],[100,101],[100,101],[100,101],[101,102],[101,102],[101,102],[102,104],[104,105],[104,105],[104,105],[105,106],[105,106],[105,106],[106,107],[106,107],[106,107],[107,108],[107,108],[107,108],[108,109],[108,109],[108,109],[109,110],[109,110],[110,111],[110,111],[110,111],[111,112],[111,112],[111,112],[112,113],[113,114],[114,115],[115,116],[116,117],[117,118],[117,118],[117,118],[118,119],[118,119],[118,119],[119,120],[119,120],[119,120],[120,121],[120,121],[120,121],[121,122],[121,122],[121,122],[122,123],[122,123],[122,123],[123,124],[123,124],[123,124],[124,125],[124,125],[124,125],[125,126],[125,126],[125,126],[126,127],[126,127],[126,127],[127,128],[127,128],[127,128],[128,129],[128,129],[128,129],[129,130],[129,130],[129,130],[130,131],[130,131],[130,131],[131,132],[131,132],[131,132],[132,133],[132,133],[132,133],[133,134],[133,134],[133,134],[134,135],[134,135],[134,135],[135,136],[136,137],[136,137],[136,137],[137,138],[137,138],[137,138],[138,139],[138,139],[138,139],[139,140],[139,140],[139,140],[140,141],[140,141],[140,141],[141,142],[141,142],[141,142],[142,143],[142,143],[142,143],[143,144],[143,144],[143,144],[144,145],[144,145],[144,145],[145,146],[145,146],[145,146],[146,147],[146,147],[146,147],[147,149],[149,150],[149,150],[149,150],[150,151],[150,151],[151,152],[151,152],[151,152],[152,153],[152,153],[152,153],[153,154],[153,154],[153,154],[154,155],[154,155],[155,156],[155,156],[155,156],[156,157],[156,157],[156,157],[157,158],[157,158],[157,158],[158,159],[158,159],[158,159],[159,160],[159,160],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,3,4,5,6,7,7,7,7,7,7,7,8,8,8,8,8,8,8,8,8,9,10,11,12,13,13,13,13,13,13,13,14,14,14,14,14,14,14,14,14,15,16,16,16,16,16,16,17,17,18,19,19,20,20,20,20,20,20,21,21,22,23,24,24,25,25,25,25,25,25,25,26,27,28,29,29,30,30,30,30,30,30,30,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,32,33,33,33,33,33,33,34,35,36,36,37,37,37,37,37,37,37,38,38,39,39,40,41,41,42,42,42,42,42,42,42,43,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,45,45,45,45,45,45,45,45,45,45,46,47,47,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,49,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,51,52,52,52,53,53,53,53,53,53,53,53,53,53,53,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54],"decoded":"強力建議大家未來盡量避免英航阿~~~\n班機原訂航程\n6/9 台北→香港\n6/9 香港→倫敦 (原訂23:15起飛，4:50am抵達)\n6/10 倫敦→斯德哥爾摩 (原訂7:40am起飛，11:05am抵達)\n就在第二段，英航no.25班機在香港機場兩度離開閘口又兩度返航\n第一次因有旅客身體不適 (好，不怪他，消耗時間也","decoded_with_specials":"強力建議大家未來盡量避免英航阿~~~\n班機原訂航程\n6/9 台北→香港\n6/9 香港→倫敦 (原訂23:15起飛，4:50am抵達)\n6/10 倫敦→斯德哥爾摩 (原訂7:40am起飛，11:05am抵達)\n就在第二段，英航no.25班機在香港機場兩度離開閘口又兩度返航\n第一次因有旅客身體不適 (好，不怪他，消耗時間也"} +{"kind":"sample","source":"fixtures/lang/ell_Grek.txt[:160]","text":"Θυμήσου με\nΗ πόλη προσφέρει πολλές ευκαιρίες - πήγαινε στο εμπορικό κέντρο, στο σαλόνι ομορφιάς , σε κλάμπ και διασκέδασε με την ψυχή σου.\nΔημιούργησε μαγευτικά","ids":[138,246,139,227,138,120,138,106,139,225,138,123,139,227,220,138,120,138,113,198,138,245,220,139,222,139,234,138,119,138,115,220,139,222,139,223,138,123,139,225,139,228,138,255,139,223,138,113,138,117,220,139,222,138,123,138,119,138,119,138,255,139,224,220,138,113,139,227,138,118,138,109,138,117,139,223,138,107,138,113,139,224,481,220,139,222,138,106,138,111,138,109,138,117,138,121,138,113,220,139,225,139,226,138,123,220,138,113,138,120,139,222,138,123,139,223,138,117,138,118,139,234,220,138,118,138,255,138,121,139,226,139,223,138,123,11,220,139,225,139,226,138,123,220,139,225,138,109,138,119,139,234,138,121,138,117,220,138,123,138,120,138,123,139,223,139,228,138,117,138,105,139,224,1154,220,139,225,138,113,220,138,118,138,119,138,105,138,120,139,222,220,138,118,138,109,138,117,220,138,112,138,117,138,109,139,225,138,118,138,255,138,112,138,109,139,225,138,113,220,138,120,138,113,220,139,226,138,115,138,121,220,139,230,139,227,139,229,138,106,220,139,225,138,123,139,227,624,138,242,138,115,138,120,138,117,138,123,139,235,139,223,138,111,138,115,139,225,138,113,220,138,120,138,109,138,111,138,113,139,227,139,226,138,117,138,118,138,105],"ids_no_specials":[138,246,139,227,138,120,138,106,139,225,138,123,139,227,220,138,120,138,113,198,138,245,220,139,222,139,234,138,119,138,115,220,139,222,139,223,138,123,139,225,139,228,138,255,139,223,138,113,138,117,220,139,222,138,123,138,119,138,119,138,255,139,224,220,138,113,139,227,138,118,138,109,138,117,139,223,138,107,138,113,139,224,481,220,139,222,138,106,138,111,138,109,138,117,138,121,138,113,220,139,225,139,226,138,123,220,138,113,138,120,139,222,138,123,139,223,138,117,138,118,139,234,220,138,118,138,255,138,121,139,226,139,223,138,123,11,220,139,225,139,226,138,123,220,139,225,138,109,138,119,139,234,138,121,138,117,220,138,123,138,120,138,123,139,223,139,228,138,117,138,105,139,224,1154,220,139,225,138,113,220,138,118,138,119,138,105,138,120,139,222,220,138,118,138,109,138,117,220,138,112,138,117,138,109,139,225,138,118,138,255,138,112,138,109,139,225,138,113,220,138,120,138,113,220,139,226,138,115,138,121,220,139,230,139,227,139,229,138,106,220,139,225,138,123,139,227,624,138,242,138,115,138,120,138,117,138,123,139,235,139,223,138,111,138,115,139,225,138,113,220,138,120,138,109,138,111,138,113,139,227,139,226,138,117,138,118,138,105],"tokens":["Î","ĺ","Ï","ħ","Î","¼","Î","®","Ï","ĥ","Î","¿","Ï","ħ","Ġ","Î","¼","Î","µ","Ċ","Î","Ĺ","Ġ","Ï","Ģ","Ï","Į","Î","»","Î","·","Ġ","Ï","Ģ","Ï","ģ","Î","¿","Ï","ĥ","Ï","Ĩ","Î","Ń","Ï","ģ","Î","µ","Î","¹","Ġ","Ï","Ģ","Î","¿","Î","»","Î","»","Î","Ń","Ï","Ĥ","Ġ","Î","µ","Ï","ħ","Î","º","Î","±","Î","¹","Ï","ģ","Î","¯","Î","µ","Ï","Ĥ","Ġ-","Ġ","Ï","Ģ","Î","®","Î","³","Î","±","Î","¹","Î","½","Î","µ","Ġ","Ï","ĥ","Ï","Ħ","Î","¿","Ġ","Î","µ","Î","¼","Ï","Ģ","Î","¿","Ï","ģ","Î","¹","Î","º","Ï","Į","Ġ","Î","º","Î","Ń","Î","½","Ï","Ħ","Ï","ģ","Î","¿",",","Ġ","Ï","ĥ","Ï","Ħ","Î","¿","Ġ","Ï","ĥ","Î","±","Î","»","Ï","Į","Î","½","Î","¹","Ġ","Î","¿","Î","¼","Î","¿","Ï","ģ","Ï","Ĩ","Î","¹","Î","¬","Ï","Ĥ","Ġ,","Ġ","Ï","ĥ","Î","µ","Ġ","Î","º","Î","»","Î","¬","Î","¼","Ï","Ģ","Ġ","Î","º","Î","±","Î","¹","Ġ","Î","´","Î","¹","Î","±","Ï","ĥ","Î","º","Î","Ń","Î","´","Î","±","Ï","ĥ","Î","µ","Ġ","Î","¼","Î","µ","Ġ","Ï","Ħ","Î","·","Î","½","Ġ","Ï","Ī","Ï","ħ","Ï","ĩ","Î","®","Ġ","Ï","ĥ","Î","¿","Ï","ħ",".Ċ","Î","Ķ","Î","·","Î","¼","Î","¹","Î","¿","Ï","į","Ï","ģ","Î","³","Î","·","Ï","ĥ","Î","µ","Ġ","Î","¼","Î","±","Î","³","Î","µ","Ï","ħ","Ï","Ħ","Î","¹","Î","º","Î","¬"],"offsets":[[0,1],[0,1],[1,2],[1,2],[2,3],[2,3],[3,4],[3,4],[4,5],[4,5],[5,6],[5,6],[6,7],[6,7],[7,8],[8,9],[8,9],[9,10],[9,10],[10,11],[11,12],[11,12],[12,13],[13,14],[13,14],[14,15],[14,15],[15,16],[15,16],[16,17],[16,17],[17,18],[18,19],[18,19],[19,20],[19,20],[20,21],[20,21],[21,22],[21,22],[22,23],[22,23],[23,24],[23,24],[24,25],[24,25],[25,26],[25,26],[26,27],[26,27],[27,28],[28,29],[28,29],[29,30],[29,30],[30,31],[30,31],[31,32],[31,32],[32,33],[32,33],[33,34],[33,34],[34,35],[35,36],[35,36],[36,37],[36,37],[37,38],[37,38],[38,39],[38,39],[39,40],[39,40],[40,41],[40,41],[41,42],[41,42],[42,43],[42,43],[43,44],[43,44],[44,46],[46,47],[47,48],[47,48],[48,49],[48,49],[49,50],[49,50],[50,51],[50,51],[51,52],[51,52],[52,53],[52,53],[53,54],[53,54],[54,55],[55,56],[55,56],[56,57],[56,57],[57,58],[57,58],[58,59],[59,60],[59,60],[60,61],[60,61],[61,62],[61,62],[62,63],[62,63],[63,64],[63,64],[64,65],[64,65],[65,66],[65,66],[66,67],[66,67],[67,68],[68,69],[68,69],[69,70],[69,70],[70,71],[70,71],[71,72],[71,72],[72,73],[72,73],[73,74],[73,74],[74,75],[75,76],[76,77],[76,77],[77,78],[77,78],[78,79],[78,79],[79,80],[80,81],[80,81],[81,82],[81,82],[82,83],[82,83],[83,84],[83,84],[84,85],[84,85],[85,86],[85,86],[86,87],[87,88],[87,88],[88,89],[88,89],[89,90],[89,90],[90,91],[90,91],[91,92],[91,92],[92,93],[92,93],[93,94],[93,94],[94,95],[94,95],[95,97],[97,98],[98,99],[98,99],[99,100],[99,100],[100,101],[101,102],[101,102],[102,103],[102,103],[103,104],[103,104],[104,105],[104,105],[105,106],[105,106],[106,107],[107,108],[107,108],[108,109],[108,109],[109,110],[109,110],[110,111],[111,112],[111,112],[112,113],[112,113],[113,114],[113,114],[114,115],[114,115],[115,116],[115,116],[116,117],[116,117],[117,118],[117,118],[118,119],[118,119],[119,120],[119,120],[120,121],[120,121],[121,122],[122,123],[122,123],[123,124],[123,124],[124,125],[125,126],[125,126],[126,127],[126,127],[127,128],[127,128],[128,129],[129,130],[129,130],[130,131],[130,131],[131,132],[131,132],[132,133],[132,133],[133,134],[134,135],[134,135],[135,136],[135,136],[136,137],[136,137],[137,139],[139,140],[139,140],[140,141],[140,141],[141,142],[141,142],[142,143],[142,143],[143,144],[143,144],[144,145],[144,145],[145,146],[145,146],[146,147],[146,147],[147,148],[147,148],[148,149],[148,149],[149,150],[149,150],[150,151],[151,152],[151,152],[152,153],[152,153],[153,154],[153,154],[154,155],[154,155],[155,156],[155,156],[156,157],[156,157],[157,158],[157,158],[158,159],[158,159],[159,160],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,2,3,3,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,8,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,10,10,10,10,10,10,10,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,12,12,12,12,12,12,12,12,12,12,12,12,12,13,14,14,14,14,14,14,14,15,15,15,15,15,15,15,15,15,15,15,15,15,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,17,18,18,18,18,18,19,19,19,19,19,19,19,19,19,19,19,20,20,20,20,20,20,20,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,22,22,22,22,22,23,23,23,23,23,23,23,24,24,24,24,24,24,24,24,24,25,25,25,25,25,25,25,26,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28],"decoded":"Θυμήσου με\nΗ πόλη προσφέρει πολλές ευκαιρίες - πήγαινε στο εμπορικό κέντρο, στο σαλόνι ομορφιάς , σε κλάμπ και διασκέδασε με την ψυχή σου.\nΔημιούργησε μαγευτικά","decoded_with_specials":"Θυμήσου με\nΗ πόλη προσφέρει πολλές ευκαιρίες - πήγαινε στο εμπορικό κέντρο, στο σαλόνι ομορφιάς , σε κλάμπ και διασκέδασε με την ψυχή σου.\nΔημιούργησε μαγευτικά"} +{"kind":"sample","source":"fixtures/lang/eng_Latn.txt[:160]","text":"|Viewing Single Post From: Spoilers for the Week of February 11th|\n|Lil||Feb 1 2013, 09:58 AM|\nDon't care about Chloe/Taniel/Jen-Jen. Don't care about Sami, rea","ids":[91,851,287,328,287,273,393,535,434,441,25,328,79,78,321,388,369,279,1205,1225,315,434,68,1323,84,658,220,16,16,339,91,198,91,43,321,8483,37,68,65,220,16,220,17,15,16,18,11,220,15,24,25,20,23,362,44,91,198,35,263,944,272,546,911,910,385,68,14,51,276,72,301,14,41,268,12,41,268,13,422,263,944,272,546,911,328,309,72,11,312,64],"ids_no_specials":[91,851,287,328,287,273,393,535,434,441,25,328,79,78,321,388,369,279,1205,1225,315,434,68,1323,84,658,220,16,16,339,91,198,91,43,321,8483,37,68,65,220,16,220,17,15,16,18,11,220,15,24,25,20,23,362,44,91,198,35,263,944,272,546,911,910,385,68,14,51,276,72,301,14,41,268,12,41,268,13,422,263,944,272,546,911,328,309,72,11,312,64],"tokens":["|","View","ing","ĠS","ing","le","ĠP","ost","ĠF","rom",":","ĠS","p","o","il","ers","Ġfor","Ġthe","ĠWe","ek","Ġof","ĠF","e","br","u","ary","Ġ","1","1","th","|","Ċ","|","L","il","||","F","e","b","Ġ","1","Ġ","2","0","1","3",",","Ġ","0","9",":","5","8","ĠA","M","|","Ċ","D","on","'t","Ġc","are","Ġabout","ĠCh","lo","e","/","T","an","i","el","/","J","en","-","J","en",".","ĠD","on","'t","Ġc","are","Ġabout","ĠS","am","i",",","Ġre","a"],"offsets":[[0,1],[1,5],[5,8],[8,10],[10,13],[13,15],[15,17],[17,20],[20,22],[22,25],[25,26],[26,28],[28,29],[29,30],[30,32],[32,35],[35,39],[39,43],[43,46],[46,48],[48,51],[51,53],[53,54],[54,56],[56,57],[57,60],[60,61],[61,62],[62,63],[63,65],[65,66],[66,67],[67,68],[68,69],[69,71],[71,73],[73,74],[74,75],[75,76],[76,77],[77,78],[78,79],[79,80],[80,81],[81,82],[82,83],[83,84],[84,85],[85,86],[86,87],[87,88],[88,89],[89,90],[90,92],[92,93],[93,94],[94,95],[95,96],[96,98],[98,100],[100,102],[102,105],[105,111],[111,114],[114,116],[116,117],[117,118],[118,119],[119,121],[121,122],[122,124],[124,125],[125,126],[126,128],[128,129],[129,130],[130,132],[132,133],[133,135],[135,137],[137,139],[139,141],[141,144],[144,150],[150,152],[152,154],[154,155],[155,156],[156,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,1,1,1,2,2,3,3,4,5,5,5,5,5,6,7,8,8,9,10,10,10,10,10,11,12,12,13,14,14,15,15,15,16,17,17,17,18,19,20,21,21,21,22,23,24,25,25,26,27,27,28,28,29,29,30,30,31,32,32,33,34,34,34,35,35,35,35,35,36,36,36,37,37,37,38,39,39,40,41,41,42,43,43,43,44,45,45],"decoded":"|Viewing Single Post From: Spoilers for the Week of February 11th|\n|Lil||Feb 1 2013, 09:58 AM|\nDon't care about Chloe/Taniel/Jen-Jen. Don't care about Sami, rea","decoded_with_specials":"|Viewing Single Post From: Spoilers for the Week of February 11th|\n|Lil||Feb 1 2013, 09:58 AM|\nDon't care about Chloe/Taniel/Jen-Jen. Don't care about Sami, rea"} +{"kind":"sample","source":"fixtures/lang/heb_Hebr.txt[:160]","text":"סיכונים – אלמנט מרכזי, חיוני לפעילות מסחרית. להבין מה הסיכון הוא מאוד חשוב. החוויה האנושית עולה כי מי יודע איך לקחת סיכונים בזמן, היא לנצח גדולים. לזכור פוליטיק","ids":[147,94,147,247,147,249,147,243,147,254,147,247,147,251,1365,220,147,238,147,250,147,252,147,254,147,246,220,147,252,147,101,147,249,147,244,147,247,11,220,147,245,147,247,147,243,147,254,147,247,220,147,250,147,97,147,95,147,247,147,250,147,243,147,103,220,147,252,147,94,147,245,147,101,147,247,147,103,13,220,147,250,147,242,147,239,147,247,147,253,220,147,252,147,242,220,147,242,147,94,147,247,147,249,147,243,147,253,220,147,242,147,243,147,238,220,147,252,147,238,147,243,147,241,220,147,245,147,102,147,243,147,239,13,220,147,242,147,245,147,243,147,243,147,247,147,242,220,147,242,147,238,147,254,147,243,147,102,147,247,147,103,220,147,95,147,243,147,250,147,242,220,147,249,147,247,220,147,252,147,247,220,147,247,147,243,147,241,147,95,220,147,238,147,247,147,248,220,147,250,147,100,147,245,147,103,220,147,94,147,247,147,249,147,243,147,254,147,247,147,251,220,147,239,147,244,147,252,147,253,11,220,147,242,147,247,147,238,220,147,250,147,254,147,99,147,245,220,147,240,147,241,147,243,147,250,147,247,147,251,13,220,147,250,147,244,147,249,147,243,147,101,220,147,97,147,243,147,250,147,247,147,246,147,247,147,100],"ids_no_specials":[147,94,147,247,147,249,147,243,147,254,147,247,147,251,1365,220,147,238,147,250,147,252,147,254,147,246,220,147,252,147,101,147,249,147,244,147,247,11,220,147,245,147,247,147,243,147,254,147,247,220,147,250,147,97,147,95,147,247,147,250,147,243,147,103,220,147,252,147,94,147,245,147,101,147,247,147,103,13,220,147,250,147,242,147,239,147,247,147,253,220,147,252,147,242,220,147,242,147,94,147,247,147,249,147,243,147,253,220,147,242,147,243,147,238,220,147,252,147,238,147,243,147,241,220,147,245,147,102,147,243,147,239,13,220,147,242,147,245,147,243,147,243,147,247,147,242,220,147,242,147,238,147,254,147,243,147,102,147,247,147,103,220,147,95,147,243,147,250,147,242,220,147,249,147,247,220,147,252,147,247,220,147,247,147,243,147,241,147,95,220,147,238,147,247,147,248,220,147,250,147,100,147,245,147,103,220,147,94,147,247,147,249,147,243,147,254,147,247,147,251,220,147,239,147,244,147,252,147,253,11,220,147,242,147,247,147,238,220,147,250,147,254,147,99,147,245,220,147,240,147,241,147,243,147,250,147,247,147,251,13,220,147,250,147,244,147,249,147,243,147,101,220,147,97,147,243,147,250,147,247,147,246,147,247,147,100],"tokens":["×","¡","×","Ļ","×","Ľ","×","ķ","×","ł","×","Ļ","×","Ŀ","ĠâĢĵ","Ġ","×","IJ","×","ľ","×","ŀ","×","ł","×","ĺ","Ġ","×","ŀ","×","¨","×","Ľ","×","ĸ","×","Ļ",",","Ġ","×","Ĺ","×","Ļ","×","ķ","×","ł","×","Ļ","Ġ","×","ľ","×","¤","×","¢","×","Ļ","×","ľ","×","ķ","×","ª","Ġ","×","ŀ","×","¡","×","Ĺ","×","¨","×","Ļ","×","ª",".","Ġ","×","ľ","×","Ķ","×","ij","×","Ļ","×","Ł","Ġ","×","ŀ","×","Ķ","Ġ","×","Ķ","×","¡","×","Ļ","×","Ľ","×","ķ","×","Ł","Ġ","×","Ķ","×","ķ","×","IJ","Ġ","×","ŀ","×","IJ","×","ķ","×","ĵ","Ġ","×","Ĺ","×","©","×","ķ","×","ij",".","Ġ","×","Ķ","×","Ĺ","×","ķ","×","ķ","×","Ļ","×","Ķ","Ġ","×","Ķ","×","IJ","×","ł","×","ķ","×","©","×","Ļ","×","ª","Ġ","×","¢","×","ķ","×","ľ","×","Ķ","Ġ","×","Ľ","×","Ļ","Ġ","×","ŀ","×","Ļ","Ġ","×","Ļ","×","ķ","×","ĵ","×","¢","Ġ","×","IJ","×","Ļ","×","ļ","Ġ","×","ľ","×","§","×","Ĺ","×","ª","Ġ","×","¡","×","Ļ","×","Ľ","×","ķ","×","ł","×","Ļ","×","Ŀ","Ġ","×","ij","×","ĸ","×","ŀ","×","Ł",",","Ġ","×","Ķ","×","Ļ","×","IJ","Ġ","×","ľ","×","ł","×","¦","×","Ĺ","Ġ","×","Ĵ","×","ĵ","×","ķ","×","ľ","×","Ļ","×","Ŀ",".","Ġ","×","ľ","×","ĸ","×","Ľ","×","ķ","×","¨","Ġ","×","¤","×","ķ","×","ľ","×","Ļ","×","ĺ","×","Ļ","×","§"],"offsets":[[0,1],[0,1],[1,2],[1,2],[2,3],[2,3],[3,4],[3,4],[4,5],[4,5],[5,6],[5,6],[6,7],[6,7],[7,9],[9,10],[10,11],[10,11],[11,12],[11,12],[12,13],[12,13],[13,14],[13,14],[14,15],[14,15],[15,16],[16,17],[16,17],[17,18],[17,18],[18,19],[18,19],[19,20],[19,20],[20,21],[20,21],[21,22],[22,23],[23,24],[23,24],[24,25],[24,25],[25,26],[25,26],[26,27],[26,27],[27,28],[27,28],[28,29],[29,30],[29,30],[30,31],[30,31],[31,32],[31,32],[32,33],[32,33],[33,34],[33,34],[34,35],[34,35],[35,36],[35,36],[36,37],[37,38],[37,38],[38,39],[38,39],[39,40],[39,40],[40,41],[40,41],[41,42],[41,42],[42,43],[42,43],[43,44],[44,45],[45,46],[45,46],[46,47],[46,47],[47,48],[47,48],[48,49],[48,49],[49,50],[49,50],[50,51],[51,52],[51,52],[52,53],[52,53],[53,54],[54,55],[54,55],[55,56],[55,56],[56,57],[56,57],[57,58],[57,58],[58,59],[58,59],[59,60],[59,60],[60,61],[61,62],[61,62],[62,63],[62,63],[63,64],[63,64],[64,65],[65,66],[65,66],[66,67],[66,67],[67,68],[67,68],[68,69],[68,69],[69,70],[70,71],[70,71],[71,72],[71,72],[72,73],[72,73],[73,74],[73,74],[74,75],[75,76],[76,77],[76,77],[77,78],[77,78],[78,79],[78,79],[79,80],[79,80],[80,81],[80,81],[81,82],[81,82],[82,83],[83,84],[83,84],[84,85],[84,85],[85,86],[85,86],[86,87],[86,87],[87,88],[87,88],[88,89],[88,89],[89,90],[89,90],[90,91],[91,92],[91,92],[92,93],[92,93],[93,94],[93,94],[94,95],[94,95],[95,96],[96,97],[96,97],[97,98],[97,98],[98,99],[99,100],[99,100],[100,101],[100,101],[101,102],[102,103],[102,103],[103,104],[103,104],[104,105],[104,105],[105,106],[105,106],[106,107],[107,108],[107,108],[108,109],[108,109],[109,110],[109,110],[110,111],[111,112],[111,112],[112,113],[112,113],[113,114],[113,114],[114,115],[114,115],[115,116],[116,117],[116,117],[117,118],[117,118],[118,119],[118,119],[119,120],[119,120],[120,121],[120,121],[121,122],[121,122],[122,123],[122,123],[123,124],[124,125],[124,125],[125,126],[125,126],[126,127],[126,127],[127,128],[127,128],[128,129],[129,130],[130,131],[130,131],[131,132],[131,132],[132,133],[132,133],[133,134],[134,135],[134,135],[135,136],[135,136],[136,137],[136,137],[137,138],[137,138],[138,139],[139,140],[139,140],[140,141],[140,141],[141,142],[141,142],[142,143],[142,143],[143,144],[143,144],[144,145],[144,145],[145,146],[146,147],[147,148],[147,148],[148,149],[148,149],[149,150],[149,150],[150,151],[150,151],[151,152],[151,152],[152,153],[153,154],[153,154],[154,155],[154,155],[155,156],[155,156],[156,157],[156,157],[157,158],[157,158],[158,159],[158,159],[159,160],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,4,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,7,7,7,7,7,7,8,9,9,9,9,9,9,9,9,9,9,9,10,10,10,10,10,11,11,11,11,11,11,11,11,11,11,11,11,11,12,12,12,12,12,12,12,13,13,13,13,13,13,13,13,13,14,14,14,14,14,14,14,14,14,15,16,16,16,16,16,16,16,16,16,16,16,16,16,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,18,18,18,18,18,18,18,18,18,19,19,19,19,19,20,20,20,20,20,21,21,21,21,21,21,21,21,21,22,22,22,22,22,22,22,23,23,23,23,23,23,23,23,23,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,25,25,25,25,25,25,25,25,25,26,27,27,27,27,27,27,27,28,28,28,28,28,28,28,28,28,29,29,29,29,29,29,29,29,29,29,29,29,29,30,31,31,31,31,31,31,31,31,31,31,31,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32],"decoded":"סיכונים – אלמנט מרכזי, חיוני לפעילות מסחרית. להבין מה הסיכון הוא מאוד חשוב. החוויה האנושית עולה כי מי יודע איך לקחת סיכונים בזמן, היא לנצח גדולים. לזכור פוליטיק","decoded_with_specials":"סיכונים – אלמנט מרכזי, חיוני לפעילות מסחרית. להבין מה הסיכון הוא מאוד חשוב. החוויה האנושית עולה כי מי יודע איך לקחת סיכונים בזמן, היא לנצח גדולים. לזכור פוליטיק"} +{"kind":"sample","source":"fixtures/lang/hin_Deva.txt[:160]","text":"PHOTOS: न्यूज पढ़ते-पढ़ते अचानक ये क्या करने लगी एंकर!\nकुछ समय पहले एक टीवी चैनल पर एंकर खबर पढ़ रही थी और पीछे की स्क्रीन पर 10 मिनिट का पोर्न वीडियो चलता रहा,","ids":[47,39,1793,46,50,25,220,156,97,101,156,98,235,156,97,107,156,98,224,156,97,250,220,156,97,103,156,97,95,156,97,120,156,97,97,156,98,229,12,156,97,103,156,97,95,156,97,120,156,97,97,156,98,229,220,156,97,227,156,97,248,156,97,122,156,97,101,156,97,243,220,156,97,107,156,98,229,220,156,97,243,156,98,235,156,97,107,156,97,122,220,156,97,243,156,97,108,156,97,101,156,98,229,220,156,97,110,156,97,245,156,98,222,220,156,97,237,156,97,224,156,97,243,156,97,108,0,198,156,97,243,156,98,223,156,97,249,220,156,97,116,156,97,106,156,97,107,220,156,97,103,156,97,117,156,97,110,156,98,229,220,156,97,237,156,97,243,220,156,97,253,156,98,222,156,97,113,156,98,222,220,156,97,248,156,98,230,156,97,101,156,97,110,220,156,97,103,156,97,108,220,156,97,237,156,97,224,156,97,243,156,97,108,220,156,97,244,156,97,105,156,97,108,220,156,97,103,156,97,95,156,97,120,220,156,97,108,156,97,117,156,98,222,220,156,97,98,156,98,222,220,156,97,242,156,97,108,220,156,97,103,156,98,222,156,97,249,156,98,229,220,156,97,243,156,98,222,220,156,97,116,156,98,235,156,97,243,156,98,235,156,97,108,156,98,222,156,97,101,220,156,97,103,156,97,108,220,16,15,220,156,97,106,156,97,123,156,97,101,156,97,123,156,97,253,220,156,97,243,156,97,122,220,156,97,103,156,98,233,156,97,108,156,98,235,156,97,101,220,156,97,113,156,98,222,156,97,94,156,97,123,156,97,107,156,98,233,220,156,97,248,156,97,110,156,97,97,156,97,122,220,156,97,108,156,97,117,156,97,122,11],"ids_no_specials":[47,39,1793,46,50,25,220,156,97,101,156,98,235,156,97,107,156,98,224,156,97,250,220,156,97,103,156,97,95,156,97,120,156,97,97,156,98,229,12,156,97,103,156,97,95,156,97,120,156,97,97,156,98,229,220,156,97,227,156,97,248,156,97,122,156,97,101,156,97,243,220,156,97,107,156,98,229,220,156,97,243,156,98,235,156,97,107,156,97,122,220,156,97,243,156,97,108,156,97,101,156,98,229,220,156,97,110,156,97,245,156,98,222,220,156,97,237,156,97,224,156,97,243,156,97,108,0,198,156,97,243,156,98,223,156,97,249,220,156,97,116,156,97,106,156,97,107,220,156,97,103,156,97,117,156,97,110,156,98,229,220,156,97,237,156,97,243,220,156,97,253,156,98,222,156,97,113,156,98,222,220,156,97,248,156,98,230,156,97,101,156,97,110,220,156,97,103,156,97,108,220,156,97,237,156,97,224,156,97,243,156,97,108,220,156,97,244,156,97,105,156,97,108,220,156,97,103,156,97,95,156,97,120,220,156,97,108,156,97,117,156,98,222,220,156,97,98,156,98,222,220,156,97,242,156,97,108,220,156,97,103,156,98,222,156,97,249,156,98,229,220,156,97,243,156,98,222,220,156,97,116,156,98,235,156,97,243,156,98,235,156,97,108,156,98,222,156,97,101,220,156,97,103,156,97,108,220,16,15,220,156,97,106,156,97,123,156,97,101,156,97,123,156,97,253,220,156,97,243,156,97,122,220,156,97,103,156,98,233,156,97,108,156,98,235,156,97,101,220,156,97,113,156,98,222,156,97,94,156,97,123,156,97,107,156,98,233,220,156,97,248,156,97,110,156,97,97,156,97,122,220,156,97,108,156,97,117,156,97,122,11],"tokens":["P","H","OT","O","S",":","Ġ","à","¤","¨","à","¥","į","à","¤","¯","à","¥","Ĥ","à","¤","ľ","Ġ","à","¤","ª","à","¤","¢","à","¤","¼","à","¤","¤","à","¥","ĩ","-","à","¤","ª","à","¤","¢","à","¤","¼","à","¤","¤","à","¥","ĩ","Ġ","à","¤","ħ","à","¤","ļ","à","¤","¾","à","¤","¨","à","¤","ķ","Ġ","à","¤","¯","à","¥","ĩ","Ġ","à","¤","ķ","à","¥","į","à","¤","¯","à","¤","¾","Ġ","à","¤","ķ","à","¤","°","à","¤","¨","à","¥","ĩ","Ġ","à","¤","²","à","¤","Ĺ","à","¥","Ģ","Ġ","à","¤","ı","à","¤","Ĥ","à","¤","ķ","à","¤","°","!","Ċ","à","¤","ķ","à","¥","ģ","à","¤","Ľ","Ġ","à","¤","¸","à","¤","®","à","¤","¯","Ġ","à","¤","ª","à","¤","¹","à","¤","²","à","¥","ĩ","Ġ","à","¤","ı","à","¤","ķ","Ġ","à","¤","Ł","à","¥","Ģ","à","¤","µ","à","¥","Ģ","Ġ","à","¤","ļ","à","¥","Ī","à","¤","¨","à","¤","²","Ġ","à","¤","ª","à","¤","°","Ġ","à","¤","ı","à","¤","Ĥ","à","¤","ķ","à","¤","°","Ġ","à","¤","ĸ","à","¤","¬","à","¤","°","Ġ","à","¤","ª","à","¤","¢","à","¤","¼","Ġ","à","¤","°","à","¤","¹","à","¥","Ģ","Ġ","à","¤","¥","à","¥","Ģ","Ġ","à","¤","Ķ","à","¤","°","Ġ","à","¤","ª","à","¥","Ģ","à","¤","Ľ","à","¥","ĩ","Ġ","à","¤","ķ","à","¥","Ģ","Ġ","à","¤","¸","à","¥","į","à","¤","ķ","à","¥","į","à","¤","°","à","¥","Ģ","à","¤","¨","Ġ","à","¤","ª","à","¤","°","Ġ","1","0","Ġ","à","¤","®","à","¤","¿","à","¤","¨","à","¤","¿","à","¤","Ł","Ġ","à","¤","ķ","à","¤","¾","Ġ","à","¤","ª","à","¥","ĭ","à","¤","°","à","¥","į","à","¤","¨","Ġ","à","¤","µ","à","¥","Ģ","à","¤","¡","à","¤","¿","à","¤","¯","à","¥","ĭ","Ġ","à","¤","ļ","à","¤","²","à","¤","¤","à","¤","¾","Ġ","à","¤","°","à","¤","¹","à","¤","¾",","],"offsets":[[0,1],[1,2],[2,4],[4,5],[5,6],[6,7],[7,8],[8,9],[8,9],[8,9],[9,10],[9,10],[9,10],[10,11],[10,11],[10,11],[11,12],[11,12],[11,12],[12,13],[12,13],[12,13],[13,14],[14,15],[14,15],[14,15],[15,16],[15,16],[15,16],[16,17],[16,17],[16,17],[17,18],[17,18],[17,18],[18,19],[18,19],[18,19],[19,20],[20,21],[20,21],[20,21],[21,22],[21,22],[21,22],[22,23],[22,23],[22,23],[23,24],[23,24],[23,24],[24,25],[24,25],[24,25],[25,26],[26,27],[26,27],[26,27],[27,28],[27,28],[27,28],[28,29],[28,29],[28,29],[29,30],[29,30],[29,30],[30,31],[30,31],[30,31],[31,32],[32,33],[32,33],[32,33],[33,34],[33,34],[33,34],[34,35],[35,36],[35,36],[35,36],[36,37],[36,37],[36,37],[37,38],[37,38],[37,38],[38,39],[38,39],[38,39],[39,40],[40,41],[40,41],[40,41],[41,42],[41,42],[41,42],[42,43],[42,43],[42,43],[43,44],[43,44],[43,44],[44,45],[45,46],[45,46],[45,46],[46,47],[46,47],[46,47],[47,48],[47,48],[47,48],[48,49],[49,50],[49,50],[49,50],[50,51],[50,51],[50,51],[51,52],[51,52],[51,52],[52,53],[52,53],[52,53],[53,54],[54,55],[55,56],[55,56],[55,56],[56,57],[56,57],[56,57],[57,58],[57,58],[57,58],[58,59],[59,60],[59,60],[59,60],[60,61],[60,61],[60,61],[61,62],[61,62],[61,62],[62,63],[63,64],[63,64],[63,64],[64,65],[64,65],[64,65],[65,66],[65,66],[65,66],[66,67],[66,67],[66,67],[67,68],[68,69],[68,69],[68,69],[69,70],[69,70],[69,70],[70,71],[71,72],[71,72],[71,72],[72,73],[72,73],[72,73],[73,74],[73,74],[73,74],[74,75],[74,75],[74,75],[75,76],[76,77],[76,77],[76,77],[77,78],[77,78],[77,78],[78,79],[78,79],[78,79],[79,80],[79,80],[79,80],[80,81],[81,82],[81,82],[81,82],[82,83],[82,83],[82,83],[83,84],[84,85],[84,85],[84,85],[85,86],[85,86],[85,86],[86,87],[86,87],[86,87],[87,88],[87,88],[87,88],[88,89],[89,90],[89,90],[89,90],[90,91],[90,91],[90,91],[91,92],[91,92],[91,92],[92,93],[93,94],[93,94],[93,94],[94,95],[94,95],[94,95],[95,96],[95,96],[95,96],[96,97],[97,98],[97,98],[97,98],[98,99],[98,99],[98,99],[99,100],[99,100],[99,100],[100,101],[101,102],[101,102],[101,102],[102,103],[102,103],[102,103],[103,104],[104,105],[104,105],[104,105],[105,106],[105,106],[105,106],[106,107],[107,108],[107,108],[107,108],[108,109],[108,109],[108,109],[109,110],[109,110],[109,110],[110,111],[110,111],[110,111],[111,112],[112,113],[112,113],[112,113],[113,114],[113,114],[113,114],[114,115],[115,116],[115,116],[115,116],[116,117],[116,117],[116,117],[117,118],[117,118],[117,118],[118,119],[118,119],[118,119],[119,120],[119,120],[119,120],[120,121],[120,121],[120,121],[121,122],[121,122],[121,122],[122,123],[123,124],[123,124],[123,124],[124,125],[124,125],[124,125],[125,126],[126,127],[127,128],[128,129],[129,130],[129,130],[129,130],[130,131],[130,131],[130,131],[131,132],[131,132],[131,132],[132,133],[132,133],[132,133],[133,134],[133,134],[133,134],[134,135],[135,136],[135,136],[135,136],[136,137],[136,137],[136,137],[137,138],[138,139],[138,139],[138,139],[139,140],[139,140],[139,140],[140,141],[140,141],[140,141],[141,142],[141,142],[141,142],[142,143],[142,143],[142,143],[143,144],[144,145],[144,145],[144,145],[145,146],[145,146],[145,146],[146,147],[146,147],[146,147],[147,148],[147,148],[147,148],[148,149],[148,149],[148,149],[149,150],[149,150],[149,150],[150,151],[151,152],[151,152],[151,152],[152,153],[152,153],[152,153],[153,154],[153,154],[153,154],[154,155],[154,155],[154,155],[155,156],[156,157],[156,157],[156,157],[157,158],[157,158],[157,158],[158,159],[158,159],[158,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,1,2,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,5,5,5,5,5,5,5,6,6,6,6,6,6,7,7,7,7,8,8,8,8,8,8,9,9,9,9,9,9,10,10,10,11,11,11,11,11,11,11,12,12,12,12,12,12,12,12,12,13,13,13,13,14,14,14,15,15,15,15,16,16,16,16,16,16,17,17,17,18,18,18,18,18,18,18,18,18,18,19,19,19,20,20,20,20,20,20,20,21,21,21,22,22,22,22,23,23,23,23,23,23,23,23,23,24,24,25,25,25,26,26,26,26,26,26,27,27,27,27,27,27,27,27,27,27,28,28,28,28,28,28,28,28,28,28,29,29,29,30,30,30,30,30,30,30,31,31,31,31,32,32,32,32,32,32,33,33,33,34,34,34,34,35,35,35,35,35,35,35,35,35,36,36,36,36,36,36,36,37,37,37,37,38,38,38,38,38,38,38,38,38,39,39,39,39,39,39,39,39,39,39,40,40,40,40,40,40,40,41,41,41,42,42,42,42,42,42,42,43,43,43,44,44,44,44,45,45,45,46,46,46,46,46,46,46,47,47,47,47,48,48,48,48,48,48,49,49,49,50,50,50,50,51,51,51,52,52,52,52,53,53,53,53,53,53,54,54,54,54,54,54,55,55,55,55,55,55,56,56,56,56,56,56,56,57,58,58,59,59,59,59,60,60,60,60,60,60,61,61,61,61,61,61,62,62,62,62,63,63,63,64,64,64,64,65,65,65,65,65,65,66,66,66,66,66,66,67,67,67,67,68,68,68,68,68,68,69,69,69,69,69,69,70,70,70,71,71,71,71,71,71,71,71,71,71,72,72,72,73,73,73,73,73,73,73,74,74,74,74],"decoded":"PHOTOS: न्यूज पढ़ते-पढ़ते अचानक ये क्या करने लगी एंकर!\nकुछ समय पहले एक टीवी चैनल पर एंकर खबर पढ़ रही थी और पीछे की स्क्रीन पर 10 मिनिट का पोर्न वीडियो चलता रहा,","decoded_with_specials":"PHOTOS: न्यूज पढ़ते-पढ़ते अचानक ये क्या करने लगी एंकर!\nकुछ समय पहले एक टीवी चैनल पर एंकर खबर पढ़ रही थी और पीछे की स्क्रीन पर 10 मिनिट का पोर्न वीडियो चलता रहा,"} +{"kind":"sample","source":"fixtures/lang/jpn_Jpan.txt[:160]","text":"Omni Dallas Parkwest Hotelでは4ツ星ホテルで、アイアンホース・ゴルフコース、Love Field Airport とテキサス・スタジアムから7.8kmかかるのところにあります。 優れたホテルにオープンし、ダラスにある古代の建築の象徴です。\n部屋\n快適なゲストルームには、モダンな設備を備えたプレ","ids":[46,76,77,72,422,541,300,393,838,86,477,472,354,301,159,223,100,159,223,107,19,159,225,226,162,246,253,159,225,249,159,225,228,159,225,104,159,223,100,1277,223,159,224,95,159,224,97,159,224,95,159,225,111,159,225,249,159,225,120,159,224,117,159,225,119,159,224,112,159,225,104,159,225,243,159,224,111,159,225,120,159,224,117,1277,223,43,994,434,816,362,404,403,220,159,223,101,159,225,228,159,224,255,159,224,113,159,224,117,159,225,119,159,224,117,159,224,123,159,224,116,159,224,95,159,225,254,159,223,233,159,224,231,22,13,23,74,76,159,223,233,159,223,233,159,224,233,159,223,106,159,223,101,159,223,241,159,224,235,159,223,104,159,223,224,159,224,232,159,223,122,159,223,247,1773,220,161,226,103,159,224,234,159,223,253,159,225,249,159,225,228,159,225,104,159,223,104,159,224,103,159,225,120,159,225,245,159,225,111,159,223,245,1277,223,159,225,222,159,225,102,159,224,117,159,223,104,159,223,224,159,224,233,161,237,97,160,119,96,159,223,106,161,119,118,163,107,231,159,223,106,164,109,94,161,122,112,159,223,100,159,223,247,1773,198,165,225,101,161,109,233,198,161,123,104,165,223,102,159,223,103,159,224,110,159,224,117,159,225,230,159,225,104,159,225,120,159,225,254,159,223,104,159,223,107,1277,223,159,225,95,159,225,222,159,225,111,159,223,103,164,101,255,161,224,247,159,224,240,161,224,247,159,223,230,159,223,253,159,225,245,159,225,105],"ids_no_specials":[46,76,77,72,422,541,300,393,838,86,477,472,354,301,159,223,100,159,223,107,19,159,225,226,162,246,253,159,225,249,159,225,228,159,225,104,159,223,100,1277,223,159,224,95,159,224,97,159,224,95,159,225,111,159,225,249,159,225,120,159,224,117,159,225,119,159,224,112,159,225,104,159,225,243,159,224,111,159,225,120,159,224,117,1277,223,43,994,434,816,362,404,403,220,159,223,101,159,225,228,159,224,255,159,224,113,159,224,117,159,225,119,159,224,117,159,224,123,159,224,116,159,224,95,159,225,254,159,223,233,159,224,231,22,13,23,74,76,159,223,233,159,223,233,159,224,233,159,223,106,159,223,101,159,223,241,159,224,235,159,223,104,159,223,224,159,224,232,159,223,122,159,223,247,1773,220,161,226,103,159,224,234,159,223,253,159,225,249,159,225,228,159,225,104,159,223,104,159,224,103,159,225,120,159,225,245,159,225,111,159,223,245,1277,223,159,225,222,159,225,102,159,224,117,159,223,104,159,223,224,159,224,233,161,237,97,160,119,96,159,223,106,161,119,118,163,107,231,159,223,106,164,109,94,161,122,112,159,223,100,159,223,247,1773,198,165,225,101,161,109,233,198,161,123,104,165,223,102,159,223,103,159,224,110,159,224,117,159,225,230,159,225,104,159,225,120,159,225,254,159,223,104,159,223,107,1277,223,159,225,95,159,225,222,159,225,111,159,223,103,164,101,255,161,224,247,159,224,240,161,224,247,159,223,230,159,223,253,159,225,245,159,225,105],"tokens":["O","m","n","i","ĠD","all","as","ĠP","ark","w","est","ĠH","ot","el","ã","ģ","§","ã","ģ","¯","4","ã","ĥ","Ħ","æ","ĺ","Ł","ã","ĥ","Ľ","ã","ĥ","Ĩ","ã","ĥ","«","ã","ģ","§","ãĢ","ģ","ã","Ĥ","¢","ã","Ĥ","¤","ã","Ĥ","¢","ã","ĥ","³","ã","ĥ","Ľ","ã","ĥ","¼","ã","Ĥ","¹","ã","ĥ","»","ã","Ĥ","´","ã","ĥ","«","ã","ĥ","ķ","ã","Ĥ","³","ã","ĥ","¼","ã","Ĥ","¹","ãĢ","ģ","L","ove","ĠF","ield","ĠA","ir","port","Ġ","ã","ģ","¨","ã","ĥ","Ĩ","ã","Ĥ","Ń","ã","Ĥ","µ","ã","Ĥ","¹","ã","ĥ","»","ã","Ĥ","¹","ã","Ĥ","¿","ã","Ĥ","¸","ã","Ĥ","¢","ã","ĥ","ł","ã","ģ","ĭ","ã","Ĥ","ī","7",".","8","k","m","ã","ģ","ĭ","ã","ģ","ĭ","ã","Ĥ","ĭ","ã","ģ","®","ã","ģ","¨","ã","ģ","ĵ","ã","Ĥ","į","ã","ģ","«","ã","ģ","Ĥ","ã","Ĥ","Ĭ","ã","ģ","¾","ã","ģ","Ļ","ãĢĤ","Ġ","å","Ħ","ª","ã","Ĥ","Į","ã","ģ","Ł","ã","ĥ","Ľ","ã","ĥ","Ĩ","ã","ĥ","«","ã","ģ","«","ã","Ĥ","ª","ã","ĥ","¼","ã","ĥ","Ĺ","ã","ĥ","³","ã","ģ","Ĺ","ãĢ","ģ","ã","ĥ","Ģ","ã","ĥ","©","ã","Ĥ","¹","ã","ģ","«","ã","ģ","Ĥ","ã","Ĥ","ĭ","å","ı","¤","ä","»","£","ã","ģ","®","å","»","º","ç","¯","ī","ã","ģ","®","è","±","¡","å","¾","´","ã","ģ","§","ã","ģ","Ļ","ãĢĤ","Ċ","é","ĥ","¨","å","±","ĭ","Ċ","å","¿","«","é","ģ","©","ã","ģ","ª","ã","Ĥ","²","ã","Ĥ","¹","ã","ĥ","Ī","ã","ĥ","«","ã","ĥ","¼","ã","ĥ","ł","ã","ģ","«","ã","ģ","¯","ãĢ","ģ","ã","ĥ","¢","ã","ĥ","Ģ","ã","ĥ","³","ã","ģ","ª","è","¨","Ń","å","Ĥ","Ļ","ã","Ĥ","Ĵ","å","Ĥ","Ļ","ã","ģ","Ī","ã","ģ","Ł","ã","ĥ","Ĺ","ã","ĥ","¬"],"offsets":[[0,1],[1,2],[2,3],[3,4],[4,6],[6,9],[9,11],[11,13],[13,16],[16,17],[17,20],[20,22],[22,24],[24,26],[26,27],[26,27],[26,27],[27,28],[27,28],[27,28],[28,29],[29,30],[29,30],[29,30],[30,31],[30,31],[30,31],[31,32],[31,32],[31,32],[32,33],[32,33],[32,33],[33,34],[33,34],[33,34],[34,35],[34,35],[34,35],[35,36],[35,36],[36,37],[36,37],[36,37],[37,38],[37,38],[37,38],[38,39],[38,39],[38,39],[39,40],[39,40],[39,40],[40,41],[40,41],[40,41],[41,42],[41,42],[41,42],[42,43],[42,43],[42,43],[43,44],[43,44],[43,44],[44,45],[44,45],[44,45],[45,46],[45,46],[45,46],[46,47],[46,47],[46,47],[47,48],[47,48],[47,48],[48,49],[48,49],[48,49],[49,50],[49,50],[49,50],[50,51],[50,51],[51,52],[52,55],[55,57],[57,61],[61,63],[63,65],[65,69],[69,70],[70,71],[70,71],[70,71],[71,72],[71,72],[71,72],[72,73],[72,73],[72,73],[73,74],[73,74],[73,74],[74,75],[74,75],[74,75],[75,76],[75,76],[75,76],[76,77],[76,77],[76,77],[77,78],[77,78],[77,78],[78,79],[78,79],[78,79],[79,80],[79,80],[79,80],[80,81],[80,81],[80,81],[81,82],[81,82],[81,82],[82,83],[82,83],[82,83],[83,84],[84,85],[85,86],[86,87],[87,88],[88,89],[88,89],[88,89],[89,90],[89,90],[89,90],[90,91],[90,91],[90,91],[91,92],[91,92],[91,92],[92,93],[92,93],[92,93],[93,94],[93,94],[93,94],[94,95],[94,95],[94,95],[95,96],[95,96],[95,96],[96,97],[96,97],[96,97],[97,98],[97,98],[97,98],[98,99],[98,99],[98,99],[99,100],[99,100],[99,100],[100,101],[101,102],[102,103],[102,103],[102,103],[103,104],[103,104],[103,104],[104,105],[104,105],[104,105],[105,106],[105,106],[105,106],[106,107],[106,107],[106,107],[107,108],[107,108],[107,108],[108,109],[108,109],[108,109],[109,110],[109,110],[109,110],[110,111],[110,111],[110,111],[111,112],[111,112],[111,112],[112,113],[112,113],[112,113],[113,114],[113,114],[113,114],[114,115],[114,115],[115,116],[115,116],[115,116],[116,117],[116,117],[116,117],[117,118],[117,118],[117,118],[118,119],[118,119],[118,119],[119,120],[119,120],[119,120],[120,121],[120,121],[120,121],[121,122],[121,122],[121,122],[122,123],[122,123],[122,123],[123,124],[123,124],[123,124],[124,125],[124,125],[124,125],[125,126],[125,126],[125,126],[126,127],[126,127],[126,127],[127,128],[127,128],[127,128],[128,129],[128,129],[128,129],[129,130],[129,130],[129,130],[130,131],[130,131],[130,131],[131,132],[132,133],[133,134],[133,134],[133,134],[134,135],[134,135],[134,135],[135,136],[136,137],[136,137],[136,137],[137,138],[137,138],[137,138],[138,139],[138,139],[138,139],[139,140],[139,140],[139,140],[140,141],[140,141],[140,141],[141,142],[141,142],[141,142],[142,143],[142,143],[142,143],[143,144],[143,144],[143,144],[144,145],[144,145],[144,145],[145,146],[145,146],[145,146],[146,147],[146,147],[146,147],[147,148],[147,148],[148,149],[148,149],[148,149],[149,150],[149,150],[149,150],[150,151],[150,151],[150,151],[151,152],[151,152],[151,152],[152,153],[152,153],[152,153],[153,154],[153,154],[153,154],[154,155],[154,155],[154,155],[155,156],[155,156],[155,156],[156,157],[156,157],[156,157],[157,158],[157,158],[157,158],[158,159],[158,159],[158,159],[159,160],[159,160],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,1,1,1,2,2,2,2,3,3,3,3,3,3,3,3,3,4,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,8,8,8,8,9,9,10,10,10,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,13,14,15,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,17,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,20,20,21,21,21,21,21,21,22,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24],"decoded":"Omni Dallas Parkwest Hotelでは4ツ星ホテルで、アイアンホース・ゴルフコース、Love Field Airport とテキサス・スタジアムから7.8kmかかるのところにあります。 優れたホテルにオープンし、ダラスにある古代の建築の象徴です。\n部屋\n快適なゲストルームには、モダンな設備を備えたプレ","decoded_with_specials":"Omni Dallas Parkwest Hotelでは4ツ星ホテルで、アイアンホース・ゴルフコース、Love Field Airport とテキサス・スタジアムから7.8kmかかるのところにあります。 優れたホテルにオープンし、ダラスにある古代の建築の象徴です。\n部屋\n快適なゲストルームには、モダンな設備を備えたプレ"} +{"kind":"sample","source":"fixtures/lang/kat_Geor.txt[:160]","text":"ჩვენ მსოფლიოს სხვადასხვა ქვეყანაში ვცხოვრობთ და სხვადასხვა ენაზე ვსაუბრობთ, მაგრამ საერთო მიზანი გვაერთიანებს. ჩვენი მთავარი მიზანი სამყაროს შემოქმედისა და ბიბლ","ids":[157,225,102,157,225,243,157,225,242,157,225,250,220,157,225,249,157,225,94,157,225,251,157,225,97,157,225,248,157,225,246,157,225,251,157,225,94,220,157,225,94,157,225,106,157,225,243,157,225,238,157,225,241,157,225,238,157,225,94,157,225,106,157,225,243,157,225,238,220,157,225,98,157,225,243,157,225,242,157,225,100,157,225,238,157,225,250,157,225,238,157,225,101,157,225,246,220,157,225,243,157,225,103,157,225,106,157,225,251,157,225,243,157,225,254,157,225,251,157,225,239,157,225,245,220,157,225,241,157,225,238,220,157,225,94,157,225,106,157,225,243,157,225,238,157,225,241,157,225,238,157,225,94,157,225,106,157,225,243,157,225,238,220,157,225,242,157,225,250,157,225,238,157,225,244,157,225,242,220,157,225,243,157,225,94,157,225,238,157,225,96,157,225,239,157,225,254,157,225,251,157,225,239,157,225,245,11,220,157,225,249,157,225,238,157,225,240,157,225,254,157,225,238,157,225,249,220,157,225,94,157,225,238,157,225,242,157,225,254,157,225,245,157,225,251,220,157,225,249,157,225,246,157,225,244,157,225,238,157,225,250,157,225,246,220,157,225,240,157,225,243,157,225,238,157,225,242,157,225,254,157,225,245,157,225,246,157,225,238,157,225,250,157,225,242,157,225,239,157,225,94,13,220,157,225,102,157,225,243,157,225,242,157,225,250,157,225,246,220,157,225,249,157,225,245,157,225,238,157,225,243,157,225,238,157,225,254,157,225,246,220,157,225,249,157,225,246,157,225,244,157,225,238,157,225,250,157,225,246,220,157,225,94,157,225,238,157,225,249,157,225,100,157,225,238,157,225,254,157,225,251,157,225,94,220,157,225,101,157,225,242,157,225,249,157,225,251,157,225,98,157,225,249,157,225,242,157,225,241,157,225,246,157,225,94,157,225,238,220,157,225,241,157,225,238,220,157,225,239,157,225,246,157,225,239,157,225,248],"ids_no_specials":[157,225,102,157,225,243,157,225,242,157,225,250,220,157,225,249,157,225,94,157,225,251,157,225,97,157,225,248,157,225,246,157,225,251,157,225,94,220,157,225,94,157,225,106,157,225,243,157,225,238,157,225,241,157,225,238,157,225,94,157,225,106,157,225,243,157,225,238,220,157,225,98,157,225,243,157,225,242,157,225,100,157,225,238,157,225,250,157,225,238,157,225,101,157,225,246,220,157,225,243,157,225,103,157,225,106,157,225,251,157,225,243,157,225,254,157,225,251,157,225,239,157,225,245,220,157,225,241,157,225,238,220,157,225,94,157,225,106,157,225,243,157,225,238,157,225,241,157,225,238,157,225,94,157,225,106,157,225,243,157,225,238,220,157,225,242,157,225,250,157,225,238,157,225,244,157,225,242,220,157,225,243,157,225,94,157,225,238,157,225,96,157,225,239,157,225,254,157,225,251,157,225,239,157,225,245,11,220,157,225,249,157,225,238,157,225,240,157,225,254,157,225,238,157,225,249,220,157,225,94,157,225,238,157,225,242,157,225,254,157,225,245,157,225,251,220,157,225,249,157,225,246,157,225,244,157,225,238,157,225,250,157,225,246,220,157,225,240,157,225,243,157,225,238,157,225,242,157,225,254,157,225,245,157,225,246,157,225,238,157,225,250,157,225,242,157,225,239,157,225,94,13,220,157,225,102,157,225,243,157,225,242,157,225,250,157,225,246,220,157,225,249,157,225,245,157,225,238,157,225,243,157,225,238,157,225,254,157,225,246,220,157,225,249,157,225,246,157,225,244,157,225,238,157,225,250,157,225,246,220,157,225,94,157,225,238,157,225,249,157,225,100,157,225,238,157,225,254,157,225,251,157,225,94,220,157,225,101,157,225,242,157,225,249,157,225,251,157,225,98,157,225,249,157,225,242,157,225,241,157,225,246,157,225,94,157,225,238,220,157,225,241,157,225,238,220,157,225,239,157,225,246,157,225,239,157,225,248],"tokens":["á","ĥ","©","á","ĥ","ķ","á","ĥ","Ķ","á","ĥ","ľ","Ġ","á","ĥ","Ľ","á","ĥ","¡","á","ĥ","Ŀ","á","ĥ","¤","á","ĥ","ļ","á","ĥ","ĺ","á","ĥ","Ŀ","á","ĥ","¡","Ġ","á","ĥ","¡","á","ĥ","®","á","ĥ","ķ","á","ĥ","IJ","á","ĥ","ĵ","á","ĥ","IJ","á","ĥ","¡","á","ĥ","®","á","ĥ","ķ","á","ĥ","IJ","Ġ","á","ĥ","¥","á","ĥ","ķ","á","ĥ","Ķ","á","ĥ","§","á","ĥ","IJ","á","ĥ","ľ","á","ĥ","IJ","á","ĥ","¨","á","ĥ","ĺ","Ġ","á","ĥ","ķ","á","ĥ","ª","á","ĥ","®","á","ĥ","Ŀ","á","ĥ","ķ","á","ĥ","ł","á","ĥ","Ŀ","á","ĥ","ij","á","ĥ","Ĺ","Ġ","á","ĥ","ĵ","á","ĥ","IJ","Ġ","á","ĥ","¡","á","ĥ","®","á","ĥ","ķ","á","ĥ","IJ","á","ĥ","ĵ","á","ĥ","IJ","á","ĥ","¡","á","ĥ","®","á","ĥ","ķ","á","ĥ","IJ","Ġ","á","ĥ","Ķ","á","ĥ","ľ","á","ĥ","IJ","á","ĥ","ĸ","á","ĥ","Ķ","Ġ","á","ĥ","ķ","á","ĥ","¡","á","ĥ","IJ","á","ĥ","£","á","ĥ","ij","á","ĥ","ł","á","ĥ","Ŀ","á","ĥ","ij","á","ĥ","Ĺ",",","Ġ","á","ĥ","Ľ","á","ĥ","IJ","á","ĥ","Ĵ","á","ĥ","ł","á","ĥ","IJ","á","ĥ","Ľ","Ġ","á","ĥ","¡","á","ĥ","IJ","á","ĥ","Ķ","á","ĥ","ł","á","ĥ","Ĺ","á","ĥ","Ŀ","Ġ","á","ĥ","Ľ","á","ĥ","ĺ","á","ĥ","ĸ","á","ĥ","IJ","á","ĥ","ľ","á","ĥ","ĺ","Ġ","á","ĥ","Ĵ","á","ĥ","ķ","á","ĥ","IJ","á","ĥ","Ķ","á","ĥ","ł","á","ĥ","Ĺ","á","ĥ","ĺ","á","ĥ","IJ","á","ĥ","ľ","á","ĥ","Ķ","á","ĥ","ij","á","ĥ","¡",".","Ġ","á","ĥ","©","á","ĥ","ķ","á","ĥ","Ķ","á","ĥ","ľ","á","ĥ","ĺ","Ġ","á","ĥ","Ľ","á","ĥ","Ĺ","á","ĥ","IJ","á","ĥ","ķ","á","ĥ","IJ","á","ĥ","ł","á","ĥ","ĺ","Ġ","á","ĥ","Ľ","á","ĥ","ĺ","á","ĥ","ĸ","á","ĥ","IJ","á","ĥ","ľ","á","ĥ","ĺ","Ġ","á","ĥ","¡","á","ĥ","IJ","á","ĥ","Ľ","á","ĥ","§","á","ĥ","IJ","á","ĥ","ł","á","ĥ","Ŀ","á","ĥ","¡","Ġ","á","ĥ","¨","á","ĥ","Ķ","á","ĥ","Ľ","á","ĥ","Ŀ","á","ĥ","¥","á","ĥ","Ľ","á","ĥ","Ķ","á","ĥ","ĵ","á","ĥ","ĺ","á","ĥ","¡","á","ĥ","IJ","Ġ","á","ĥ","ĵ","á","ĥ","IJ","Ġ","á","ĥ","ij","á","ĥ","ĺ","á","ĥ","ij","á","ĥ","ļ"],"offsets":[[0,1],[0,1],[0,1],[1,2],[1,2],[1,2],[2,3],[2,3],[2,3],[3,4],[3,4],[3,4],[4,5],[5,6],[5,6],[5,6],[6,7],[6,7],[6,7],[7,8],[7,8],[7,8],[8,9],[8,9],[8,9],[9,10],[9,10],[9,10],[10,11],[10,11],[10,11],[11,12],[11,12],[11,12],[12,13],[12,13],[12,13],[13,14],[14,15],[14,15],[14,15],[15,16],[15,16],[15,16],[16,17],[16,17],[16,17],[17,18],[17,18],[17,18],[18,19],[18,19],[18,19],[19,20],[19,20],[19,20],[20,21],[20,21],[20,21],[21,22],[21,22],[21,22],[22,23],[22,23],[22,23],[23,24],[23,24],[23,24],[24,25],[25,26],[25,26],[25,26],[26,27],[26,27],[26,27],[27,28],[27,28],[27,28],[28,29],[28,29],[28,29],[29,30],[29,30],[29,30],[30,31],[30,31],[30,31],[31,32],[31,32],[31,32],[32,33],[32,33],[32,33],[33,34],[33,34],[33,34],[34,35],[35,36],[35,36],[35,36],[36,37],[36,37],[36,37],[37,38],[37,38],[37,38],[38,39],[38,39],[38,39],[39,40],[39,40],[39,40],[40,41],[40,41],[40,41],[41,42],[41,42],[41,42],[42,43],[42,43],[42,43],[43,44],[43,44],[43,44],[44,45],[45,46],[45,46],[45,46],[46,47],[46,47],[46,47],[47,48],[48,49],[48,49],[48,49],[49,50],[49,50],[49,50],[50,51],[50,51],[50,51],[51,52],[51,52],[51,52],[52,53],[52,53],[52,53],[53,54],[53,54],[53,54],[54,55],[54,55],[54,55],[55,56],[55,56],[55,56],[56,57],[56,57],[56,57],[57,58],[57,58],[57,58],[58,59],[59,60],[59,60],[59,60],[60,61],[60,61],[60,61],[61,62],[61,62],[61,62],[62,63],[62,63],[62,63],[63,64],[63,64],[63,64],[64,65],[65,66],[65,66],[65,66],[66,67],[66,67],[66,67],[67,68],[67,68],[67,68],[68,69],[68,69],[68,69],[69,70],[69,70],[69,70],[70,71],[70,71],[70,71],[71,72],[71,72],[71,72],[72,73],[72,73],[72,73],[73,74],[73,74],[73,74],[74,75],[75,76],[76,77],[76,77],[76,77],[77,78],[77,78],[77,78],[78,79],[78,79],[78,79],[79,80],[79,80],[79,80],[80,81],[80,81],[80,81],[81,82],[81,82],[81,82],[82,83],[83,84],[83,84],[83,84],[84,85],[84,85],[84,85],[85,86],[85,86],[85,86],[86,87],[86,87],[86,87],[87,88],[87,88],[87,88],[88,89],[88,89],[88,89],[89,90],[90,91],[90,91],[90,91],[91,92],[91,92],[91,92],[92,93],[92,93],[92,93],[93,94],[93,94],[93,94],[94,95],[94,95],[94,95],[95,96],[95,96],[95,96],[96,97],[97,98],[97,98],[97,98],[98,99],[98,99],[98,99],[99,100],[99,100],[99,100],[100,101],[100,101],[100,101],[101,102],[101,102],[101,102],[102,103],[102,103],[102,103],[103,104],[103,104],[103,104],[104,105],[104,105],[104,105],[105,106],[105,106],[105,106],[106,107],[106,107],[106,107],[107,108],[107,108],[107,108],[108,109],[108,109],[108,109],[109,110],[110,111],[111,112],[111,112],[111,112],[112,113],[112,113],[112,113],[113,114],[113,114],[113,114],[114,115],[114,115],[114,115],[115,116],[115,116],[115,116],[116,117],[117,118],[117,118],[117,118],[118,119],[118,119],[118,119],[119,120],[119,120],[119,120],[120,121],[120,121],[120,121],[121,122],[121,122],[121,122],[122,123],[122,123],[122,123],[123,124],[123,124],[123,124],[124,125],[125,126],[125,126],[125,126],[126,127],[126,127],[126,127],[127,128],[127,128],[127,128],[128,129],[128,129],[128,129],[129,130],[129,130],[129,130],[130,131],[130,131],[130,131],[131,132],[132,133],[132,133],[132,133],[133,134],[133,134],[133,134],[134,135],[134,135],[134,135],[135,136],[135,136],[135,136],[136,137],[136,137],[136,137],[137,138],[137,138],[137,138],[138,139],[138,139],[138,139],[139,140],[139,140],[139,140],[140,141],[141,142],[141,142],[141,142],[142,143],[142,143],[142,143],[143,144],[143,144],[143,144],[144,145],[144,145],[144,145],[145,146],[145,146],[145,146],[146,147],[146,147],[146,147],[147,148],[147,148],[147,148],[148,149],[148,149],[148,149],[149,150],[149,150],[149,150],[150,151],[150,151],[150,151],[151,152],[151,152],[151,152],[152,153],[153,154],[153,154],[153,154],[154,155],[154,155],[154,155],[155,156],[156,157],[156,157],[156,157],[157,158],[157,158],[157,158],[158,159],[158,159],[158,159],[159,160],[159,160],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,9,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,14,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,20,20,20,20,20,20,20,21,21,21,21,21,21,21,21,21,21,21,21,21],"decoded":"ჩვენ მსოფლიოს სხვადასხვა ქვეყანაში ვცხოვრობთ და სხვადასხვა ენაზე ვსაუბრობთ, მაგრამ საერთო მიზანი გვაერთიანებს. ჩვენი მთავარი მიზანი სამყაროს შემოქმედისა და ბიბლ","decoded_with_specials":"ჩვენ მსოფლიოს სხვადასხვა ქვეყანაში ვცხოვრობთ და სხვადასხვა ენაზე ვსაუბრობთ, მაგრამ საერთო მიზანი გვაერთიანებს. ჩვენი მთავარი მიზანი სამყაროს შემოქმედისა და ბიბლ"} +{"kind":"sample","source":"fixtures/lang/kor_Hang.txt[:160]","text":"전화번호:\n위치: 뉴질랜드 > 남섬 > 말버러 > 블레넘\n가격대 (1박): 117,886 원 - 140,244 원\n4.5성급 — Lugano Motor Lodge 4.5*\n- 예약 옵션:\n- 트립어드바이저는 호텔스닷컴, Expedia, Agoda, Asia Web Direct 및 Boo","ids":[168,254,226,169,247,242,167,110,230,169,246,116,510,168,250,226,168,117,246,25,220,167,231,112,168,100,230,167,252,250,167,241,250,861,220,167,224,101,168,226,105,861,220,167,100,238,167,110,226,167,253,105,861,220,167,116,242,167,254,230,167,226,246,198,166,108,222,166,110,102,167,234,222,320,16,167,108,243,1648,220,16,16,22,11,23,23,21,220,168,249,238,481,220,16,19,15,11,17,19,19,220,168,249,238,198,19,13,20,168,226,109,166,116,231,636,242,444,768,276,78,386,354,269,444,347,709,220,19,13,20,5618,12,220,168,246,230,168,243,121,220,168,246,113,168,227,246,510,12,220,169,232,116,167,99,121,168,244,112,167,241,250,167,108,242,168,251,112,168,254,222,167,232,242,220,169,246,116,169,227,242,168,232,97,167,233,115,168,119,112,11,1374,79,291,685,11,362,70,347,64,11,1634,685,1205,65,422,1226,220,167,108,237,425,78,78],"ids_no_specials":[168,254,226,169,247,242,167,110,230,169,246,116,510,168,250,226,168,117,246,25,220,167,231,112,168,100,230,167,252,250,167,241,250,861,220,167,224,101,168,226,105,861,220,167,100,238,167,110,226,167,253,105,861,220,167,116,242,167,254,230,167,226,246,198,166,108,222,166,110,102,167,234,222,320,16,167,108,243,1648,220,16,16,22,11,23,23,21,220,168,249,238,481,220,16,19,15,11,17,19,19,220,168,249,238,198,19,13,20,168,226,109,166,116,231,636,242,444,768,276,78,386,354,269,444,347,709,220,19,13,20,5618,12,220,168,246,230,168,243,121,220,168,246,113,168,227,246,510,12,220,169,232,116,167,99,121,168,244,112,167,241,250,167,108,242,168,251,112,168,254,222,167,232,242,220,169,246,116,169,227,242,168,232,97,167,233,115,168,119,112,11,1374,79,291,685,11,362,70,347,64,11,1634,685,1205,65,422,1226,220,167,108,237,425,78,78],"tokens":["ì","ł","Ħ","í","Ļ","Ķ","ë","²","Ī","í","ĺ","¸",":Ċ","ì","ľ","Ħ","ì","¹","ĺ",":","Ġ","ë","ī","´","ì","§","Ī","ë","ŀ","ľ","ë","ĵ","ľ","Ġ>","Ġ","ë","Ĥ","¨","ì","Ħ","¬","Ġ>","Ġ","ë","§","IJ","ë","²","Ħ","ë","Ł","¬","Ġ>","Ġ","ë","¸","Ķ","ë","ł","Ī","ë","Ħ","ĺ","Ċ","ê","°","Ģ","ê","²","©","ë","Į","Ģ","Ġ(","1","ë","°","ķ","):","Ġ","1","1","7",",","8","8","6","Ġ","ì","Ľ","IJ","Ġ-","Ġ","1","4","0",",","2","4","4","Ġ","ì","Ľ","IJ","Ċ","4",".","5","ì","Ħ","±","ê","¸","ī","ĠâĢ","Ķ","ĠL","ug","an","o","ĠM","ot","or","ĠL","od","ge","Ġ","4",".","5","*Ċ","-","Ġ","ì","ĺ","Ī","ì","ķ","½","Ġ","ì","ĺ","µ","ì","ħ","ĺ",":Ċ","-","Ġ","í","Ĭ","¸","ë","¦","½","ì","ĸ","´","ë","ĵ","ľ","ë","°","Ķ","ì","Ŀ","´","ì","ł","Ģ","ë","Ĭ","Ķ","Ġ","í","ĺ","¸","í","ħ","Ķ","ì","Ĭ","¤","ë","ĭ","·","ì","»","´",",","ĠEx","p","ed","ia",",","ĠA","g","od","a",",","ĠAs","ia","ĠWe","b","ĠD","irect","Ġ","ë","°","ı","ĠB","o","o"],"offsets":[[0,1],[0,1],[0,1],[1,2],[1,2],[1,2],[2,3],[2,3],[2,3],[3,4],[3,4],[3,4],[4,6],[6,7],[6,7],[6,7],[7,8],[7,8],[7,8],[8,9],[9,10],[10,11],[10,11],[10,11],[11,12],[11,12],[11,12],[12,13],[12,13],[12,13],[13,14],[13,14],[13,14],[14,16],[16,17],[17,18],[17,18],[17,18],[18,19],[18,19],[18,19],[19,21],[21,22],[22,23],[22,23],[22,23],[23,24],[23,24],[23,24],[24,25],[24,25],[24,25],[25,27],[27,28],[28,29],[28,29],[28,29],[29,30],[29,30],[29,30],[30,31],[30,31],[30,31],[31,32],[32,33],[32,33],[32,33],[33,34],[33,34],[33,34],[34,35],[34,35],[34,35],[35,37],[37,38],[38,39],[38,39],[38,39],[39,41],[41,42],[42,43],[43,44],[44,45],[45,46],[46,47],[47,48],[48,49],[49,50],[50,51],[50,51],[50,51],[51,53],[53,54],[54,55],[55,56],[56,57],[57,58],[58,59],[59,60],[60,61],[61,62],[62,63],[62,63],[62,63],[63,64],[64,65],[65,66],[66,67],[67,68],[67,68],[67,68],[68,69],[68,69],[68,69],[69,71],[70,71],[71,73],[73,75],[75,77],[77,78],[78,80],[80,82],[82,84],[84,86],[86,88],[88,90],[90,91],[91,92],[92,93],[93,94],[94,96],[96,97],[97,98],[98,99],[98,99],[98,99],[99,100],[99,100],[99,100],[100,101],[101,102],[101,102],[101,102],[102,103],[102,103],[102,103],[103,105],[105,106],[106,107],[107,108],[107,108],[107,108],[108,109],[108,109],[108,109],[109,110],[109,110],[109,110],[110,111],[110,111],[110,111],[111,112],[111,112],[111,112],[112,113],[112,113],[112,113],[113,114],[113,114],[113,114],[114,115],[114,115],[114,115],[115,116],[116,117],[116,117],[116,117],[117,118],[117,118],[117,118],[118,119],[118,119],[118,119],[119,120],[119,120],[119,120],[120,121],[120,121],[120,121],[121,122],[122,125],[125,126],[126,128],[128,130],[130,131],[131,133],[133,134],[134,136],[136,137],[137,138],[138,141],[141,143],[143,146],[146,147],[147,149],[149,154],[154,155],[155,156],[155,156],[155,156],[156,158],[158,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,0,0,0,0,0,1,2,2,2,2,2,2,3,4,4,4,4,4,4,4,4,4,4,4,4,4,5,6,6,6,6,6,6,6,7,8,8,8,8,8,8,8,8,8,8,9,10,10,10,10,10,10,10,10,10,10,11,12,12,12,12,12,12,12,12,12,13,14,15,15,15,16,17,18,18,18,19,20,20,20,21,21,21,21,22,23,24,24,24,25,26,26,26,27,27,27,27,28,29,30,31,32,32,32,32,32,32,33,33,34,34,34,34,35,35,35,36,36,36,37,38,39,40,41,42,43,43,43,43,43,43,43,44,44,44,44,44,44,44,45,46,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,49,50,50,50,50,51,52,52,52,52,53,54,54,55,55,56,56,57,57,57,57,58,58,58],"decoded":"전화번호:\n위치: 뉴질랜드 > 남섬 > 말버러 > 블레넘\n가격대 (1박): 117,886 원 - 140,244 원\n4.5성급 — Lugano Motor Lodge 4.5*\n- 예약 옵션:\n- 트립어드바이저는 호텔스닷컴, Expedia, Agoda, Asia Web Direct 및 Boo","decoded_with_specials":"전화번호:\n위치: 뉴질랜드 > 남섬 > 말버러 > 블레넘\n가격대 (1박): 117,886 원 - 140,244 원\n4.5성급 — Lugano Motor Lodge 4.5*\n- 예약 옵션:\n- 트립어드바이저는 호텔스닷컴, Expedia, Agoda, Asia Web Direct 및 Boo"} +{"kind":"sample","source":"fixtures/lang/rus_Cyrl.txt[:160]","text":"Покупая продукты в супермаркетах, сегодня уже мало кто верит в их качество. Как не сделать свое меню экстремальным и на что следует обращать внимание – читайте!","ids":[140,253,1456,140,118,141,225,140,123,1478,141,237,1278,123,141,222,1456,140,112,141,225,140,118,1792,141,233,1278,110,220,141,223,141,225,140,123,1504,141,222,140,120,1478,141,222,140,118,1504,1792,1478,141,227,11,220,141,223,1504,140,111,1456,140,112,140,121,141,237,220,141,225,140,114,1504,1278,120,1478,140,119,1456,1278,118,1792,1456,1278,110,1504,141,222,1802,1792,1278,110,1278,116,141,227,1278,118,1478,141,229,1504,141,223,1792,140,110,1456,13,1278,248,1478,140,118,1278,121,1504,220,141,223,140,112,1504,140,119,1478,1792,141,234,220,141,223,140,110,1456,1504,1278,120,1504,140,121,141,236,220,141,235,140,118,141,223,1792,141,222,1504,140,120,1478,140,119,141,234,140,121,141,233,140,120,1278,116,1278,121,1478,220,141,229,1792,1456,220,141,223,140,119,1504,140,112,141,225,1504,1792,1278,122,140,109,141,222,1478,141,231,1478,1792,141,234,1278,110,140,121,1802,140,120,1478,140,121,1802,1504,1365,220,141,229,1802,1792,1478,140,117,1792,1504,0],"ids_no_specials":[140,253,1456,140,118,141,225,140,123,1478,141,237,1278,123,141,222,1456,140,112,141,225,140,118,1792,141,233,1278,110,220,141,223,141,225,140,123,1504,141,222,140,120,1478,141,222,140,118,1504,1792,1478,141,227,11,220,141,223,1504,140,111,1456,140,112,140,121,141,237,220,141,225,140,114,1504,1278,120,1478,140,119,1456,1278,118,1792,1456,1278,110,1504,141,222,1802,1792,1278,110,1278,116,141,227,1278,118,1478,141,229,1504,141,223,1792,140,110,1456,13,1278,248,1478,140,118,1278,121,1504,220,141,223,140,112,1504,140,119,1478,1792,141,234,220,141,223,140,110,1456,1504,1278,120,1504,140,121,141,236,220,141,235,140,118,141,223,1792,141,222,1504,140,120,1478,140,119,141,234,140,121,141,233,140,120,1278,116,1278,121,1478,220,141,229,1792,1456,220,141,223,140,119,1504,140,112,141,225,1504,1792,1278,122,140,109,141,222,1478,141,231,1478,1792,141,234,1278,110,140,121,1802,140,120,1478,140,121,1802,1504,1365,220,141,229,1802,1792,1478,140,117,1792,1504,0],"tokens":["Ð","Ł","о","Ð","º","Ñ","ĥ","Ð","¿","а","Ñ","ı","ĠÐ","¿","Ñ","Ģ","о","Ð","´","Ñ","ĥ","Ð","º","ÑĤ","Ñ","ĭ","ĠÐ","²","Ġ","Ñ","ģ","Ñ","ĥ","Ð","¿","е","Ñ","Ģ","Ð","¼","а","Ñ","Ģ","Ð","º","е","ÑĤ","а","Ñ","ħ",",","Ġ","Ñ","ģ","е","Ð","³","о","Ð","´","Ð","½","Ñ","ı","Ġ","Ñ","ĥ","Ð","¶","е","ĠÐ","¼","а","Ð","»","о","ĠÐ","º","ÑĤ","о","ĠÐ","²","е","Ñ","Ģ","и","ÑĤ","ĠÐ","²","ĠÐ","¸","Ñ","ħ","ĠÐ","º","а","Ñ","ĩ","е","Ñ","ģ","ÑĤ","Ð","²","о",".","ĠÐ","ļ","а","Ð","º","ĠÐ","½","е","Ġ","Ñ","ģ","Ð","´","е","Ð","»","а","ÑĤ","Ñ","Į","Ġ","Ñ","ģ","Ð","²","о","е","ĠÐ","¼","е","Ð","½","Ñ","İ","Ġ","Ñ","į","Ð","º","Ñ","ģ","ÑĤ","Ñ","Ģ","е","Ð","¼","а","Ð","»","Ñ","Į","Ð","½","Ñ","ĭ","Ð","¼","ĠÐ","¸","ĠÐ","½","а","Ġ","Ñ","ĩ","ÑĤ","о","Ġ","Ñ","ģ","Ð","»","е","Ð","´","Ñ","ĥ","е","ÑĤ","ĠÐ","¾","Ð","±","Ñ","Ģ","а","Ñ","ī","а","ÑĤ","Ñ","Į","ĠÐ","²","Ð","½","и","Ð","¼","а","Ð","½","и","е","ĠâĢĵ","Ġ","Ñ","ĩ","и","ÑĤ","а","Ð","¹","ÑĤ","е","!"],"offsets":[[0,1],[0,1],[1,2],[2,3],[2,3],[3,4],[3,4],[4,5],[4,5],[5,6],[6,7],[6,7],[7,9],[8,9],[9,10],[9,10],[10,11],[11,12],[11,12],[12,13],[12,13],[13,14],[13,14],[14,15],[15,16],[15,16],[16,18],[17,18],[18,19],[19,20],[19,20],[20,21],[20,21],[21,22],[21,22],[22,23],[23,24],[23,24],[24,25],[24,25],[25,26],[26,27],[26,27],[27,28],[27,28],[28,29],[29,30],[30,31],[31,32],[31,32],[32,33],[33,34],[34,35],[34,35],[35,36],[36,37],[36,37],[37,38],[38,39],[38,39],[39,40],[39,40],[40,41],[40,41],[41,42],[42,43],[42,43],[43,44],[43,44],[44,45],[45,47],[46,47],[47,48],[48,49],[48,49],[49,50],[50,52],[51,52],[52,53],[53,54],[54,56],[55,56],[56,57],[57,58],[57,58],[58,59],[59,60],[60,62],[61,62],[62,64],[63,64],[64,65],[64,65],[65,67],[66,67],[67,68],[68,69],[68,69],[69,70],[70,71],[70,71],[71,72],[72,73],[72,73],[73,74],[74,75],[75,77],[76,77],[77,78],[78,79],[78,79],[79,81],[80,81],[81,82],[82,83],[83,84],[83,84],[84,85],[84,85],[85,86],[86,87],[86,87],[87,88],[88,89],[89,90],[89,90],[90,91],[91,92],[91,92],[92,93],[92,93],[93,94],[94,95],[95,97],[96,97],[97,98],[98,99],[98,99],[99,100],[99,100],[100,101],[101,102],[101,102],[102,103],[102,103],[103,104],[103,104],[104,105],[105,106],[105,106],[106,107],[107,108],[107,108],[108,109],[109,110],[109,110],[110,111],[110,111],[111,112],[111,112],[112,113],[112,113],[113,114],[113,114],[114,116],[115,116],[116,118],[117,118],[118,119],[119,120],[120,121],[120,121],[121,122],[122,123],[123,124],[124,125],[124,125],[125,126],[125,126],[126,127],[127,128],[127,128],[128,129],[128,129],[129,130],[130,131],[131,133],[132,133],[133,134],[133,134],[134,135],[134,135],[135,136],[136,137],[136,137],[137,138],[138,139],[139,140],[139,140],[140,142],[141,142],[142,143],[142,143],[143,144],[144,145],[144,145],[145,146],[146,147],[146,147],[147,148],[148,149],[149,151],[151,152],[152,153],[152,153],[153,154],[154,155],[155,156],[156,157],[156,157],[157,158],[158,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,5,5,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,7,7,7,7,7,7,8,8,8,8,9,9,9,9,9,9,9,10,10,11,11,11,11,12,12,12,12,12,12,12,12,12,12,12,12,13,14,14,14,14,14,15,15,15,16,16,16,16,16,16,16,16,16,16,16,16,17,17,17,17,17,17,17,18,18,18,18,18,18,18,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,20,20,21,21,21,22,22,22,22,22,23,23,23,23,23,23,23,23,23,23,23,23,24,24,24,24,24,24,24,24,24,24,24,24,24,25,25,25,25,25,25,25,25,25,25,25,25,26,27,27,27,27,27,27,27,27,27,27,28],"decoded":"Покупая продукты в супермаркетах, сегодня уже мало кто верит в их качество. Как не сделать свое меню экстремальным и на что следует обращать внимание – читайте!","decoded_with_specials":"Покупая продукты в супермаркетах, сегодня уже мало кто верит в их качество. Как не сделать свое меню экстремальным и на что следует обращать внимание – читайте!"} +{"kind":"sample","source":"fixtures/lang/tam_Taml.txt[:160]","text":"‘ஹலோ கலெக்டர் சாரா… சரக்கு வேணும்… கடை எப்ப திறப்பீங்க?’\nஒரு படத்தில் ஒயின்ஷாப்புக்குள் திருடப் போன வடிவேலு நன்றாக சரக்கடித்து விட்டு, ஒயின்ஷாப் ஓனருக்கு போன் ப","ids":[378,246,156,106,117,156,106,110,156,107,233,220,156,106,243,156,106,110,156,107,228,156,106,243,156,107,235,156,106,253,156,106,108,156,107,235,220,156,106,248,156,106,122,156,106,108,156,106,122,378,99,220,156,106,248,156,106,108,156,106,243,156,107,235,156,106,243,156,107,223,220,156,106,113,156,107,229,156,106,96,156,107,223,156,106,106,156,107,235,378,99,220,156,106,243,156,106,253,156,107,230,220,156,106,236,156,106,103,156,107,235,156,106,103,220,156,106,97,156,106,123,156,106,109,156,106,103,156,107,235,156,106,103,156,107,222,156,106,247,156,107,235,156,106,243,30,527,198,156,106,240,156,106,108,156,107,223,220,156,106,103,156,106,253,156,106,97,156,107,235,156,106,97,156,106,123,156,106,110,156,107,235,220,156,106,240,156,106,107,156,106,123,156,106,102,156,107,235,156,106,115,156,106,122,156,106,103,156,107,235,156,106,103,156,107,223,156,106,243,156,107,235,156,106,243,156,107,223,156,106,111,156,107,235,220,156,106,97,156,106,123,156,106,108,156,107,223,156,106,253,156,106,103,156,107,235,220,156,106,103,156,107,233,156,106,102,220,156,106,113,156,106,253,156,106,123,156,106,113,156,107,229,156,106,110,156,107,223,220,156,106,101,156,106,102,156,107,235,156,106,109,156,106,122,156,106,243,220,156,106,248,156,106,108,156,106,243,156,107,235,156,106,243,156,106,253,156,106,123,156,106,97,156,107,235,156,106,97,156,107,223,220,156,106,113,156,106,123,156,106,253,156,107,235,156,106,253,156,107,223,11,220,156,106,240,156,106,107,156,106,123,156,106,102,156,107,235,156,106,115,156,106,122,156,106,103,156,107,235,220,156,106,241,156,106,102,156,106,108,156,107,223,156,106,243,156,107,235,156,106,243,156,107,223,220,156,106,103,156,107,233,156,106,102,156,107,235,220,156,106,103],"ids_no_specials":[378,246,156,106,117,156,106,110,156,107,233,220,156,106,243,156,106,110,156,107,228,156,106,243,156,107,235,156,106,253,156,106,108,156,107,235,220,156,106,248,156,106,122,156,106,108,156,106,122,378,99,220,156,106,248,156,106,108,156,106,243,156,107,235,156,106,243,156,107,223,220,156,106,113,156,107,229,156,106,96,156,107,223,156,106,106,156,107,235,378,99,220,156,106,243,156,106,253,156,107,230,220,156,106,236,156,106,103,156,107,235,156,106,103,220,156,106,97,156,106,123,156,106,109,156,106,103,156,107,235,156,106,103,156,107,222,156,106,247,156,107,235,156,106,243,30,527,198,156,106,240,156,106,108,156,107,223,220,156,106,103,156,106,253,156,106,97,156,107,235,156,106,97,156,106,123,156,106,110,156,107,235,220,156,106,240,156,106,107,156,106,123,156,106,102,156,107,235,156,106,115,156,106,122,156,106,103,156,107,235,156,106,103,156,107,223,156,106,243,156,107,235,156,106,243,156,107,223,156,106,111,156,107,235,220,156,106,97,156,106,123,156,106,108,156,107,223,156,106,253,156,106,103,156,107,235,220,156,106,103,156,107,233,156,106,102,220,156,106,113,156,106,253,156,106,123,156,106,113,156,107,229,156,106,110,156,107,223,220,156,106,101,156,106,102,156,107,235,156,106,109,156,106,122,156,106,243,220,156,106,248,156,106,108,156,106,243,156,107,235,156,106,243,156,106,253,156,106,123,156,106,97,156,107,235,156,106,97,156,107,223,220,156,106,113,156,106,123,156,106,253,156,107,235,156,106,253,156,107,223,11,220,156,106,240,156,106,107,156,106,123,156,106,102,156,107,235,156,106,115,156,106,122,156,106,103,156,107,235,220,156,106,241,156,106,102,156,106,108,156,107,223,156,106,243,156,107,235,156,106,243,156,107,223,220,156,106,103,156,107,233,156,106,102,156,107,235,220,156,106,103],"tokens":["âĢ","ĺ","à","®","¹","à","®","²","à","¯","ĭ","Ġ","à","®","ķ","à","®","²","à","¯","Ĩ","à","®","ķ","à","¯","į","à","®","Ł","à","®","°","à","¯","į","Ġ","à","®","ļ","à","®","¾","à","®","°","à","®","¾","âĢ","¦","Ġ","à","®","ļ","à","®","°","à","®","ķ","à","¯","į","à","®","ķ","à","¯","ģ","Ġ","à","®","µ","à","¯","ĩ","à","®","£","à","¯","ģ","à","®","®","à","¯","į","âĢ","¦","Ġ","à","®","ķ","à","®","Ł","à","¯","Ī","Ġ","à","®","İ","à","®","ª","à","¯","į","à","®","ª","Ġ","à","®","¤","à","®","¿","à","®","±","à","®","ª","à","¯","į","à","®","ª","à","¯","Ģ","à","®","Ļ","à","¯","į","à","®","ķ","?","âĢĻ","Ċ","à","®","Ĵ","à","®","°","à","¯","ģ","Ġ","à","®","ª","à","®","Ł","à","®","¤","à","¯","į","à","®","¤","à","®","¿","à","®","²","à","¯","į","Ġ","à","®","Ĵ","à","®","¯","à","®","¿","à","®","©","à","¯","į","à","®","·","à","®","¾","à","®","ª","à","¯","į","à","®","ª","à","¯","ģ","à","®","ķ","à","¯","į","à","®","ķ","à","¯","ģ","à","®","³","à","¯","į","Ġ","à","®","¤","à","®","¿","à","®","°","à","¯","ģ","à","®","Ł","à","®","ª","à","¯","į","Ġ","à","®","ª","à","¯","ĭ","à","®","©","Ġ","à","®","µ","à","®","Ł","à","®","¿","à","®","µ","à","¯","ĩ","à","®","²","à","¯","ģ","Ġ","à","®","¨","à","®","©","à","¯","į","à","®","±","à","®","¾","à","®","ķ","Ġ","à","®","ļ","à","®","°","à","®","ķ","à","¯","į","à","®","ķ","à","®","Ł","à","®","¿","à","®","¤","à","¯","į","à","®","¤","à","¯","ģ","Ġ","à","®","µ","à","®","¿","à","®","Ł","à","¯","į","à","®","Ł","à","¯","ģ",",","Ġ","à","®","Ĵ","à","®","¯","à","®","¿","à","®","©","à","¯","į","à","®","·","à","®","¾","à","®","ª","à","¯","į","Ġ","à","®","ĵ","à","®","©","à","®","°","à","¯","ģ","à","®","ķ","à","¯","į","à","®","ķ","à","¯","ģ","Ġ","à","®","ª","à","¯","ĭ","à","®","©","à","¯","į","Ġ","à","®","ª"],"offsets":[[0,1],[0,1],[1,2],[1,2],[1,2],[2,3],[2,3],[2,3],[3,4],[3,4],[3,4],[4,5],[5,6],[5,6],[5,6],[6,7],[6,7],[6,7],[7,8],[7,8],[7,8],[8,9],[8,9],[8,9],[9,10],[9,10],[9,10],[10,11],[10,11],[10,11],[11,12],[11,12],[11,12],[12,13],[12,13],[12,13],[13,14],[14,15],[14,15],[14,15],[15,16],[15,16],[15,16],[16,17],[16,17],[16,17],[17,18],[17,18],[17,18],[18,19],[18,19],[19,20],[20,21],[20,21],[20,21],[21,22],[21,22],[21,22],[22,23],[22,23],[22,23],[23,24],[23,24],[23,24],[24,25],[24,25],[24,25],[25,26],[25,26],[25,26],[26,27],[27,28],[27,28],[27,28],[28,29],[28,29],[28,29],[29,30],[29,30],[29,30],[30,31],[30,31],[30,31],[31,32],[31,32],[31,32],[32,33],[32,33],[32,33],[33,34],[33,34],[34,35],[35,36],[35,36],[35,36],[36,37],[36,37],[36,37],[37,38],[37,38],[37,38],[38,39],[39,40],[39,40],[39,40],[40,41],[40,41],[40,41],[41,42],[41,42],[41,42],[42,43],[42,43],[42,43],[43,44],[44,45],[44,45],[44,45],[45,46],[45,46],[45,46],[46,47],[46,47],[46,47],[47,48],[47,48],[47,48],[48,49],[48,49],[48,49],[49,50],[49,50],[49,50],[50,51],[50,51],[50,51],[51,52],[51,52],[51,52],[52,53],[52,53],[52,53],[53,54],[53,54],[53,54],[54,55],[55,56],[56,57],[57,58],[57,58],[57,58],[58,59],[58,59],[58,59],[59,60],[59,60],[59,60],[60,61],[61,62],[61,62],[61,62],[62,63],[62,63],[62,63],[63,64],[63,64],[63,64],[64,65],[64,65],[64,65],[65,66],[65,66],[65,66],[66,67],[66,67],[66,67],[67,68],[67,68],[67,68],[68,69],[68,69],[68,69],[69,70],[70,71],[70,71],[70,71],[71,72],[71,72],[71,72],[72,73],[72,73],[72,73],[73,74],[73,74],[73,74],[74,75],[74,75],[74,75],[75,76],[75,76],[75,76],[76,77],[76,77],[76,77],[77,78],[77,78],[77,78],[78,79],[78,79],[78,79],[79,80],[79,80],[79,80],[80,81],[80,81],[80,81],[81,82],[81,82],[81,82],[82,83],[82,83],[82,83],[83,84],[83,84],[83,84],[84,85],[84,85],[84,85],[85,86],[85,86],[85,86],[86,87],[86,87],[86,87],[87,88],[88,89],[88,89],[88,89],[89,90],[89,90],[89,90],[90,91],[90,91],[90,91],[91,92],[91,92],[91,92],[92,93],[92,93],[92,93],[93,94],[93,94],[93,94],[94,95],[94,95],[94,95],[95,96],[96,97],[96,97],[96,97],[97,98],[97,98],[97,98],[98,99],[98,99],[98,99],[99,100],[100,101],[100,101],[100,101],[101,102],[101,102],[101,102],[102,103],[102,103],[102,103],[103,104],[103,104],[103,104],[104,105],[104,105],[104,105],[105,106],[105,106],[105,106],[106,107],[106,107],[106,107],[107,108],[108,109],[108,109],[108,109],[109,110],[109,110],[109,110],[110,111],[110,111],[110,111],[111,112],[111,112],[111,112],[112,113],[112,113],[112,113],[113,114],[113,114],[113,114],[114,115],[115,116],[115,116],[115,116],[116,117],[116,117],[116,117],[117,118],[117,118],[117,118],[118,119],[118,119],[118,119],[119,120],[119,120],[119,120],[120,121],[120,121],[120,121],[121,122],[121,122],[121,122],[122,123],[122,123],[122,123],[123,124],[123,124],[123,124],[124,125],[124,125],[124,125],[125,126],[125,126],[125,126],[126,127],[127,128],[127,128],[127,128],[128,129],[128,129],[128,129],[129,130],[129,130],[129,130],[130,131],[130,131],[130,131],[131,132],[131,132],[131,132],[132,133],[132,133],[132,133],[133,134],[134,135],[135,136],[135,136],[135,136],[136,137],[136,137],[136,137],[137,138],[137,138],[137,138],[138,139],[138,139],[138,139],[139,140],[139,140],[139,140],[140,141],[140,141],[140,141],[141,142],[141,142],[141,142],[142,143],[142,143],[142,143],[143,144],[143,144],[143,144],[144,145],[145,146],[145,146],[145,146],[146,147],[146,147],[146,147],[147,148],[147,148],[147,148],[148,149],[148,149],[148,149],[149,150],[149,150],[149,150],[150,151],[150,151],[150,151],[151,152],[151,152],[151,152],[152,153],[152,153],[152,153],[153,154],[154,155],[154,155],[154,155],[155,156],[155,156],[155,156],[156,157],[156,157],[156,157],[157,158],[157,158],[157,158],[158,159],[159,160],[159,160],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,0,1,1,1,2,2,2,2,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,5,5,5,6,6,6,6,7,7,7,7,7,7,8,8,8,8,8,9,9,9,9,9,9,9,9,9,9,10,10,10,10,10,10,11,11,11,12,12,12,12,13,13,13,13,13,13,14,14,14,14,14,14,15,15,15,15,15,16,16,16,16,16,16,16,17,17,17,18,18,18,18,18,18,18,19,19,19,19,19,19,20,20,20,20,21,21,21,21,21,21,21,21,21,22,22,22,22,22,22,23,23,23,23,23,23,24,24,24,24,24,24,25,25,25,26,26,26,26,26,26,27,27,27,28,28,28,28,28,28,28,28,28,28,29,29,29,29,29,29,30,30,30,30,30,30,31,31,31,32,32,32,32,32,32,32,33,33,33,33,33,33,34,34,34,34,34,34,35,35,35,35,35,35,36,36,36,36,36,36,37,37,37,37,37,37,38,38,38,38,38,38,39,39,39,39,39,39,40,40,40,41,41,41,41,42,42,42,42,42,42,43,43,43,43,43,43,43,43,43,44,44,44,45,45,45,45,46,46,46,46,46,46,47,47,47,47,47,47,47,48,48,48,48,48,48,49,49,49,49,49,49,50,50,50,51,51,51,51,51,51,51,52,52,52,52,52,52,53,53,53,53,53,53,54,54,54,54,54,54,54,54,54,54,55,55,55,55,55,55,55,55,55,56,56,56,56,56,56,57,57,57,57,57,57,58,58,58,59,59,59,59,60,60,60,60,60,60,61,61,61,61,61,61,62,62,62,62,63,63,63,63,63,63,63,64,64,64,64,64,64,65,65,65,65,65,65,66,66,66,66,66,66,67,67,67,68,68,68,68,68,68,68,68,68,68,69,69,69,69,69,69,70,70,70,70,70,70,71,71,71,72,72,72,72,73,73,73,73,73,73,74,74,74,75,75,75,75],"decoded":"‘ஹலோ கலெக்டர் சாரா… சரக்கு வேணும்… கடை எப்ப திறப்பீங்க?’\nஒரு படத்தில் ஒயின்ஷாப்புக்குள் திருடப் போன வடிவேலு நன்றாக சரக்கடித்து விட்டு, ஒயின்ஷாப் ஓனருக்கு போன் ப","decoded_with_specials":"‘ஹலோ கலெக்டர் சாரா… சரக்கு வேணும்… கடை எப்ப திறப்பீங்க?’\nஒரு படத்தில் ஒயின்ஷாப்புக்குள் திருடப் போன வடிவேலு நன்றாக சரக்கடித்து விட்டு, ஒயின்ஷாப் ஓனருக்கு போன் ப"} +{"kind":"sample","source":"fixtures/lang/tha_Thai.txt[:160]","text":"นับเจดีย์ดูนะครับว่าครบยี่สิบองค์รึเปล่า คำว่าซาว ทางเหนือแปลว่่ายี่สิบครับ ผมนับดูแล้วก็ครบนะ ลองดู เสร็จแล้วก็เข้าเยี่ยมชมพิพธภัณฑ์ เดินเข้าไปด้านในหน่อยก็จะม","ids":[156,116,247,156,116,109,156,116,248,156,117,222,156,116,230,156,116,242,156,116,113,156,116,95,156,117,234,156,116,242,156,116,117,156,116,247,156,116,108,156,116,226,156,116,96,156,116,109,156,116,248,156,116,100,156,117,230,156,116,110,156,116,226,156,116,96,156,116,248,156,116,95,156,116,113,156,117,230,156,116,103,156,116,112,156,116,248,156,116,255,156,116,229,156,116,226,156,117,234,156,116,96,156,116,114,156,117,222,156,116,249,156,116,98,156,117,230,156,116,110,220,156,116,226,156,116,111,156,116,100,156,117,230,156,116,110,156,116,233,156,116,110,156,116,100,220,156,116,245,156,116,110,156,116,229,156,117,222,156,116,104,156,116,247,156,116,115,156,116,255,156,117,223,156,116,249,156,116,98,156,116,100,156,117,230,156,117,230,156,116,110,156,116,95,156,116,113,156,117,230,156,116,103,156,116,112,156,116,248,156,116,226,156,116,96,156,116,109,156,116,248,220,156,116,250,156,116,94,156,116,247,156,116,109,156,116,248,156,116,242,156,116,117,156,117,223,156,116,98,156,117,231,156,116,100,156,116,223,156,117,229,156,116,226,156,116,96,156,116,248,156,116,247,156,116,108,220,156,116,98,156,116,255,156,116,229,156,116,242,156,116,117,220,156,117,222,156,116,103,156,116,96,156,117,229,156,116,230,156,117,223,156,116,98,156,117,231,156,116,100,156,116,223,156,117,229,156,117,222,156,116,224,156,117,231,156,116,110,156,117,222,156,116,95,156,116,113,156,117,230,156,116,95,156,116,94,156,116,232,156,116,94,156,116,252,156,116,112,156,116,252,156,116,246,156,116,254,156,116,109,156,116,241,156,116,239,156,117,234,220,156,117,222,156,116,242,156,116,112,156,116,247,156,117,222,156,116,224,156,117,231,156,116,110,156,117,226,156,116,249,156,116,242,156,117,231,156,116,110,156,116,247,156,117,225,156,116,247,156,116,104,156,116,247,156,117,230,156,116,255,156,116,95,156,116,223,156,117,229,156,116,230,156,116,108,156,116,94],"ids_no_specials":[156,116,247,156,116,109,156,116,248,156,117,222,156,116,230,156,116,242,156,116,113,156,116,95,156,117,234,156,116,242,156,116,117,156,116,247,156,116,108,156,116,226,156,116,96,156,116,109,156,116,248,156,116,100,156,117,230,156,116,110,156,116,226,156,116,96,156,116,248,156,116,95,156,116,113,156,117,230,156,116,103,156,116,112,156,116,248,156,116,255,156,116,229,156,116,226,156,117,234,156,116,96,156,116,114,156,117,222,156,116,249,156,116,98,156,117,230,156,116,110,220,156,116,226,156,116,111,156,116,100,156,117,230,156,116,110,156,116,233,156,116,110,156,116,100,220,156,116,245,156,116,110,156,116,229,156,117,222,156,116,104,156,116,247,156,116,115,156,116,255,156,117,223,156,116,249,156,116,98,156,116,100,156,117,230,156,117,230,156,116,110,156,116,95,156,116,113,156,117,230,156,116,103,156,116,112,156,116,248,156,116,226,156,116,96,156,116,109,156,116,248,220,156,116,250,156,116,94,156,116,247,156,116,109,156,116,248,156,116,242,156,116,117,156,117,223,156,116,98,156,117,231,156,116,100,156,116,223,156,117,229,156,116,226,156,116,96,156,116,248,156,116,247,156,116,108,220,156,116,98,156,116,255,156,116,229,156,116,242,156,116,117,220,156,117,222,156,116,103,156,116,96,156,117,229,156,116,230,156,117,223,156,116,98,156,117,231,156,116,100,156,116,223,156,117,229,156,117,222,156,116,224,156,117,231,156,116,110,156,117,222,156,116,95,156,116,113,156,117,230,156,116,95,156,116,94,156,116,232,156,116,94,156,116,252,156,116,112,156,116,252,156,116,246,156,116,254,156,116,109,156,116,241,156,116,239,156,117,234,220,156,117,222,156,116,242,156,116,112,156,116,247,156,117,222,156,116,224,156,117,231,156,116,110,156,117,226,156,116,249,156,116,242,156,117,231,156,116,110,156,116,247,156,117,225,156,116,247,156,116,104,156,116,247,156,117,230,156,116,255,156,116,95,156,116,223,156,117,229,156,116,230,156,116,108,156,116,94],"tokens":["à","¸","Ļ","à","¸","±","à","¸","ļ","à","¹","Ģ","à","¸","Ī","à","¸","Ķ","à","¸","µ","à","¸","¢","à","¹","Į","à","¸","Ķ","à","¸","¹","à","¸","Ļ","à","¸","°","à","¸","Ħ","à","¸","£","à","¸","±","à","¸","ļ","à","¸","§","à","¹","Ī","à","¸","²","à","¸","Ħ","à","¸","£","à","¸","ļ","à","¸","¢","à","¸","µ","à","¹","Ī","à","¸","ª","à","¸","´","à","¸","ļ","à","¸","Ń","à","¸","ĩ","à","¸","Ħ","à","¹","Į","à","¸","£","à","¸","¶","à","¹","Ģ","à","¸","Ľ","à","¸","¥","à","¹","Ī","à","¸","²","Ġ","à","¸","Ħ","à","¸","³","à","¸","§","à","¹","Ī","à","¸","²","à","¸","ĭ","à","¸","²","à","¸","§","Ġ","à","¸","Ĺ","à","¸","²","à","¸","ĩ","à","¹","Ģ","à","¸","«","à","¸","Ļ","à","¸","·","à","¸","Ń","à","¹","ģ","à","¸","Ľ","à","¸","¥","à","¸","§","à","¹","Ī","à","¹","Ī","à","¸","²","à","¸","¢","à","¸","µ","à","¹","Ī","à","¸","ª","à","¸","´","à","¸","ļ","à","¸","Ħ","à","¸","£","à","¸","±","à","¸","ļ","Ġ","à","¸","ľ","à","¸","¡","à","¸","Ļ","à","¸","±","à","¸","ļ","à","¸","Ķ","à","¸","¹","à","¹","ģ","à","¸","¥","à","¹","ī","à","¸","§","à","¸","ģ","à","¹","ĩ","à","¸","Ħ","à","¸","£","à","¸","ļ","à","¸","Ļ","à","¸","°","Ġ","à","¸","¥","à","¸","Ń","à","¸","ĩ","à","¸","Ķ","à","¸","¹","Ġ","à","¹","Ģ","à","¸","ª","à","¸","£","à","¹","ĩ","à","¸","Ī","à","¹","ģ","à","¸","¥","à","¹","ī","à","¸","§","à","¸","ģ","à","¹","ĩ","à","¹","Ģ","à","¸","Ĥ","à","¹","ī","à","¸","²","à","¹","Ģ","à","¸","¢","à","¸","µ","à","¹","Ī","à","¸","¢","à","¸","¡","à","¸","Ĭ","à","¸","¡","à","¸","ŀ","à","¸","´","à","¸","ŀ","à","¸","ĺ","à","¸","ł","à","¸","±","à","¸","ĵ","à","¸","ij","à","¹","Į","Ġ","à","¹","Ģ","à","¸","Ķ","à","¸","´","à","¸","Ļ","à","¹","Ģ","à","¸","Ĥ","à","¹","ī","à","¸","²","à","¹","Ħ","à","¸","Ľ","à","¸","Ķ","à","¹","ī","à","¸","²","à","¸","Ļ","à","¹","ĥ","à","¸","Ļ","à","¸","«","à","¸","Ļ","à","¹","Ī","à","¸","Ń","à","¸","¢","à","¸","ģ","à","¹","ĩ","à","¸","Ī","à","¸","°","à","¸","¡"],"offsets":[[0,1],[0,1],[0,1],[1,2],[1,2],[1,2],[2,3],[2,3],[2,3],[3,4],[3,4],[3,4],[4,5],[4,5],[4,5],[5,6],[5,6],[5,6],[6,7],[6,7],[6,7],[7,8],[7,8],[7,8],[8,9],[8,9],[8,9],[9,10],[9,10],[9,10],[10,11],[10,11],[10,11],[11,12],[11,12],[11,12],[12,13],[12,13],[12,13],[13,14],[13,14],[13,14],[14,15],[14,15],[14,15],[15,16],[15,16],[15,16],[16,17],[16,17],[16,17],[17,18],[17,18],[17,18],[18,19],[18,19],[18,19],[19,20],[19,20],[19,20],[20,21],[20,21],[20,21],[21,22],[21,22],[21,22],[22,23],[22,23],[22,23],[23,24],[23,24],[23,24],[24,25],[24,25],[24,25],[25,26],[25,26],[25,26],[26,27],[26,27],[26,27],[27,28],[27,28],[27,28],[28,29],[28,29],[28,29],[29,30],[29,30],[29,30],[30,31],[30,31],[30,31],[31,32],[31,32],[31,32],[32,33],[32,33],[32,33],[33,34],[33,34],[33,34],[34,35],[34,35],[34,35],[35,36],[35,36],[35,36],[36,37],[36,37],[36,37],[37,38],[37,38],[37,38],[38,39],[38,39],[38,39],[39,40],[39,40],[39,40],[40,41],[41,42],[41,42],[41,42],[42,43],[42,43],[42,43],[43,44],[43,44],[43,44],[44,45],[44,45],[44,45],[45,46],[45,46],[45,46],[46,47],[46,47],[46,47],[47,48],[47,48],[47,48],[48,49],[48,49],[48,49],[49,50],[50,51],[50,51],[50,51],[51,52],[51,52],[51,52],[52,53],[52,53],[52,53],[53,54],[53,54],[53,54],[54,55],[54,55],[54,55],[55,56],[55,56],[55,56],[56,57],[56,57],[56,57],[57,58],[57,58],[57,58],[58,59],[58,59],[58,59],[59,60],[59,60],[59,60],[60,61],[60,61],[60,61],[61,62],[61,62],[61,62],[62,63],[62,63],[62,63],[63,64],[63,64],[63,64],[64,65],[64,65],[64,65],[65,66],[65,66],[65,66],[66,67],[66,67],[66,67],[67,68],[67,68],[67,68],[68,69],[68,69],[68,69],[69,70],[69,70],[69,70],[70,71],[70,71],[70,71],[71,72],[71,72],[71,72],[72,73],[72,73],[72,73],[73,74],[73,74],[73,74],[74,75],[74,75],[74,75],[75,76],[76,77],[76,77],[76,77],[77,78],[77,78],[77,78],[78,79],[78,79],[78,79],[79,80],[79,80],[79,80],[80,81],[80,81],[80,81],[81,82],[81,82],[81,82],[82,83],[82,83],[82,83],[83,84],[83,84],[83,84],[84,85],[84,85],[84,85],[85,86],[85,86],[85,86],[86,87],[86,87],[86,87],[87,88],[87,88],[87,88],[88,89],[88,89],[88,89],[89,90],[89,90],[89,90],[90,91],[90,91],[90,91],[91,92],[91,92],[91,92],[92,93],[92,93],[92,93],[93,94],[93,94],[93,94],[94,95],[95,96],[95,96],[95,96],[96,97],[96,97],[96,97],[97,98],[97,98],[97,98],[98,99],[98,99],[98,99],[99,100],[99,100],[99,100],[100,101],[101,102],[101,102],[101,102],[102,103],[102,103],[102,103],[103,104],[103,104],[103,104],[104,105],[104,105],[104,105],[105,106],[105,106],[105,106],[106,107],[106,107],[106,107],[107,108],[107,108],[107,108],[108,109],[108,109],[108,109],[109,110],[109,110],[109,110],[110,111],[110,111],[110,111],[111,112],[111,112],[111,112],[112,113],[112,113],[112,113],[113,114],[113,114],[113,114],[114,115],[114,115],[114,115],[115,116],[115,116],[115,116],[116,117],[116,117],[116,117],[117,118],[117,118],[117,118],[118,119],[118,119],[118,119],[119,120],[119,120],[119,120],[120,121],[120,121],[120,121],[121,122],[121,122],[121,122],[122,123],[122,123],[122,123],[123,124],[123,124],[123,124],[124,125],[124,125],[124,125],[125,126],[125,126],[125,126],[126,127],[126,127],[126,127],[127,128],[127,128],[127,128],[128,129],[128,129],[128,129],[129,130],[129,130],[129,130],[130,131],[130,131],[130,131],[131,132],[131,132],[131,132],[132,133],[132,133],[132,133],[133,134],[134,135],[134,135],[134,135],[135,136],[135,136],[135,136],[136,137],[136,137],[136,137],[137,138],[137,138],[137,138],[138,139],[138,139],[138,139],[139,140],[139,140],[139,140],[140,141],[140,141],[140,141],[141,142],[141,142],[141,142],[142,143],[142,143],[142,143],[143,144],[143,144],[143,144],[144,145],[144,145],[144,145],[145,146],[145,146],[145,146],[146,147],[146,147],[146,147],[147,148],[147,148],[147,148],[148,149],[148,149],[148,149],[149,150],[149,150],[149,150],[150,151],[150,151],[150,151],[151,152],[151,152],[151,152],[152,153],[152,153],[152,153],[153,154],[153,154],[153,154],[154,155],[154,155],[154,155],[155,156],[155,156],[155,156],[156,157],[156,157],[156,157],[157,158],[157,158],[157,158],[158,159],[158,159],[158,159],[159,160],[159,160],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,7,7,7,7,7,7,8,8,8,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,10,10,10,10,10,10,11,11,11,11,11,11,11,11,11,11,11,11,12,12,12,12,12,12,13,13,13,13,13,13,13,13,13,13,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,17,17,17,17,17,17,18,18,18,18,18,18,19,19,19,19,19,19,20,20,20,21,21,21,21,21,21,21,21,21,21,21,21,22,22,22,22,22,22,23,23,23,23,23,23,23,23,23,23,24,24,24,24,24,24,24,24,24,25,25,25,25,25,25,25,25,25,26,26,26,26,26,26,26,26,26,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,28,28,28,28,28,28,28,28,28,28,28,28,28,29,29,29,30,30,30,30,30,30,30,30,30,30,31,31,31,31,31,31,31,31,31,31,31,31,32,32,32,32,32,32,32,32,32,33,33,33,33,33,33,33,33,33,34,34,34,34,34,34,34,34,34,34,34,34,35,35,35,35,35,35,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,37,37,37,37,37,37,37,37,37,37,37,37,38,38,38,38,38,38,38,38,38,39,39,39,40,40,40,40,40,40,40,41,41,41,41,41,41,41,41,41,41,41,41,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,44,44,44,44,44,44,44,44,44,44,44,44,45,45,45,45,45,45,45,45,45,45,45,45],"decoded":"นับเจดีย์ดูนะครับว่าครบยี่สิบองค์รึเปล่า คำว่าซาว ทางเหนือแปลว่่ายี่สิบครับ ผมนับดูแล้วก็ครบนะ ลองดู เสร็จแล้วก็เข้าเยี่ยมชมพิพธภัณฑ์ เดินเข้าไปด้านในหน่อยก็จะม","decoded_with_specials":"นับเจดีย์ดูนะครับว่าครบยี่สิบองค์รึเปล่า คำว่าซาว ทางเหนือแปลว่่ายี่สิบครับ ผมนับดูแล้วก็ครบนะ ลองดู เสร็จแล้วก็เข้าเยี่ยมชมพิพธภัณฑ์ เดินเข้าไปด้านในหน่อยก็จะม"} +{"kind":"sample","source":"fixtures/modalities/added_normalized_dense.txt[:160]","text":"FLIBBERJAST CRUNGLEDORF FLIBBERJAST SNORLAXIAN fast BLORPTRONIC ZORPTASTIC split WIDGETRON split stage SNORLAXIAN BLORPTRONIC BLORPTRONIC \n QUIBBLENAUT CRUNGLED","ids":[37,43,40,33,33,640,41,32,784,356,49,1861,38,867,35,868,37,434,43,40,33,33,640,41,32,784,328,45,868,43,32,55,40,1093,282,559,425,43,868,47,51,49,711,1317,1863,868,47,51,32,784,1317,274,500,275,467,915,38,1348,49,711,274,500,275,357,424,328,45,868,43,32,55,40,1093,425,43,868,47,51,49,711,1317,425,43,868,47,51,49,711,1317,715,1207,52,40,33,33,867,45,32,1381,356,49,1861,38,867,35],"ids_no_specials":[37,43,40,33,33,640,41,32,784,356,49,1861,38,867,35,868,37,434,43,40,33,33,640,41,32,784,328,45,868,43,32,55,40,1093,282,559,425,43,868,47,51,49,711,1317,1863,868,47,51,32,784,1317,274,500,275,467,915,38,1348,49,711,274,500,275,357,424,328,45,868,43,32,55,40,1093,425,43,868,47,51,49,711,1317,425,43,868,47,51,49,711,1317,715,1207,52,40,33,33,867,45,32,1381,356,49,1861,38,867,35],"tokens":["F","L","I","B","B","ER","J","A","ST","ĠC","R","UN","G","LE","D","OR","F","ĠF","L","I","B","B","ER","J","A","ST","ĠS","N","OR","L","A","X","I","AN","Ġf","ast","ĠB","L","OR","P","T","R","ON","IC","ĠZ","OR","P","T","A","ST","IC","Ġs","pl","it","ĠW","ID","G","ET","R","ON","Ġs","pl","it","Ġst","age","ĠS","N","OR","L","A","X","I","AN","ĠB","L","OR","P","T","R","ON","IC","ĠB","L","OR","P","T","R","ON","IC","ĠĊ","ĠQ","U","I","B","B","LE","N","A","UT","ĠC","R","UN","G","LE","D"],"offsets":[[0,1],[1,2],[2,3],[3,4],[4,5],[5,7],[7,8],[8,9],[9,11],[11,13],[13,14],[14,16],[16,17],[17,19],[19,20],[20,22],[22,23],[23,25],[25,26],[26,27],[27,28],[28,29],[29,31],[31,32],[32,33],[33,35],[35,37],[37,38],[38,40],[40,41],[41,42],[42,43],[43,44],[44,46],[46,48],[48,51],[51,53],[53,54],[54,56],[56,57],[57,58],[58,59],[59,61],[61,63],[63,65],[65,67],[67,68],[68,69],[69,70],[70,72],[72,74],[74,76],[76,78],[78,80],[80,82],[82,84],[84,85],[85,87],[87,88],[88,90],[90,92],[92,94],[94,96],[96,99],[99,102],[102,104],[104,105],[105,107],[107,108],[108,109],[109,110],[110,111],[111,113],[113,115],[115,116],[116,118],[118,119],[119,120],[120,121],[121,123],[123,125],[125,127],[127,128],[128,130],[130,131],[131,132],[132,133],[133,135],[135,137],[137,139],[139,141],[141,142],[142,143],[143,144],[144,145],[145,147],[147,148],[148,149],[149,151],[151,153],[153,154],[154,156],[156,157],[157,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,4,4,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,7,7,7,8,8,8,8,8,8,9,9,9,10,10,11,11,11,11,11,11,11,11,12,12,12,12,12,12,12,12,13,13,13,13,13,13,13,13,14,15,15,15,15,15,15,15,15,15,16,16,16,16,16,16],"decoded":"FLIBBERJAST CRUNGLEDORF FLIBBERJAST SNORLAXIAN fast BLORPTRONIC ZORPTASTIC split WIDGETRON split stage SNORLAXIAN BLORPTRONIC BLORPTRONIC \n QUIBBLENAUT CRUNGLED","decoded_with_specials":"FLIBBERJAST CRUNGLEDORF FLIBBERJAST SNORLAXIAN fast BLORPTRONIC ZORPTASTIC split WIDGETRON split stage SNORLAXIAN BLORPTRONIC BLORPTRONIC \n QUIBBLENAUT CRUNGLED"} +{"kind":"sample","source":"fixtures/modalities/added_normalized_sparse.txt[:160]","text":"ZORPTASTIC for we for decode CRUNGLEDORF here through ZORPTASTIC and the we WIDGETRON model \n normalize bytes and back flows text CRUNGLEDORF WUZZLEFANG chunk b","ids":[57,868,47,51,32,784,1317,369,582,369,1622,534,356,49,1861,38,867,35,868,37,1588,1526,1863,868,47,51,32,784,1317,323,279,582,467,915,38,1348,49,711,1614,715,308,493,278,551,553,83,288,323,1182,1320,363,82,1467,356,49,1861,38,867,35,868,37,467,52,57,57,867,37,1093,38,521,359,74,293],"ids_no_specials":[57,868,47,51,32,784,1317,369,582,369,1622,534,356,49,1861,38,867,35,868,37,1588,1526,1863,868,47,51,32,784,1317,323,279,582,467,915,38,1348,49,711,1614,715,308,493,278,551,553,83,288,323,1182,1320,363,82,1467,356,49,1861,38,867,35,868,37,467,52,57,57,867,37,1093,38,521,359,74,293],"tokens":["Z","OR","P","T","A","ST","IC","Ġfor","Ġwe","Ġfor","Ġdec","ode","ĠC","R","UN","G","LE","D","OR","F","Ġhere","Ġthrough","ĠZ","OR","P","T","A","ST","IC","Ġand","Ġthe","Ġwe","ĠW","ID","G","ET","R","ON","Ġmodel","ĠĊ","Ġn","orm","al","ize","Ġby","t","es","Ġand","Ġback","Ġfl","ow","s","Ġtext","ĠC","R","UN","G","LE","D","OR","F","ĠW","U","Z","Z","LE","F","AN","G","Ġch","un","k","Ġb"],"offsets":[[0,1],[1,3],[3,4],[4,5],[5,6],[6,8],[8,10],[10,14],[14,17],[17,21],[21,25],[25,28],[28,30],[30,31],[31,33],[33,34],[34,36],[36,37],[37,39],[39,40],[40,45],[45,53],[53,55],[55,57],[57,58],[58,59],[59,60],[60,62],[62,64],[64,68],[68,72],[72,75],[75,77],[77,79],[79,80],[80,82],[82,83],[83,85],[85,91],[91,93],[93,95],[95,98],[98,100],[100,103],[103,106],[106,107],[107,109],[109,113],[113,118],[118,121],[121,123],[123,124],[124,129],[129,131],[131,132],[132,134],[134,135],[135,137],[137,138],[138,140],[140,141],[141,143],[143,144],[144,145],[145,146],[146,148],[148,149],[149,151],[151,152],[152,155],[155,157],[157,158],[158,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,1,2,3,4,4,5,5,5,5,5,5,5,5,6,7,8,8,8,8,8,8,8,9,10,11,12,12,12,12,12,12,13,14,15,15,15,15,16,16,16,17,18,19,19,19,20,21,21,21,21,21,21,21,21,22,22,22,22,22,22,22,22,23,23,23,24],"decoded":"ZORPTASTIC for we for decode CRUNGLEDORF here through ZORPTASTIC and the we WIDGETRON model \n normalize bytes and back flows text CRUNGLEDORF WUZZLEFANG chunk b","decoded_with_specials":"ZORPTASTIC for we for decode CRUNGLEDORF here through ZORPTASTIC and the we WIDGETRON model \n normalize bytes and back flows text CRUNGLEDORF WUZZLEFANG chunk b"} +{"kind":"sample","source":"fixtures/modalities/added_special_dense.txt[:160]","text":"fast <|xs2|> chunk <|xs1|> <|xs4|> <|xs3|> <|xs4|> modality language <|xs0|> reads <|xs2|> and and \n <|xs1|> <|xs0|> <|xs1|> <|xs4|> <|xs4|> <|xs1|> back and re","ids":[69,559,366,91,87,82,17,91,29,521,359,74,366,91,87,82,16,91,29,366,91,87,82,19,91,29,366,91,87,82,18,91,29,366,91,87,82,19,91,29,1463,278,487,326,524,84,424,366,91,87,82,15,91,29,1349,82,366,91,87,82,17,91,29,323,323,715,366,91,87,82,16,91,29,366,91,87,82,15,91,29,366,91,87,82,16,91,29,366,91,87,82,19,91,29,366,91,87,82,19,91,29,366,91,87,82,16,91,29,1182,323,312],"ids_no_specials":[69,559,366,91,87,82,17,91,29,521,359,74,366,91,87,82,16,91,29,366,91,87,82,19,91,29,366,91,87,82,18,91,29,366,91,87,82,19,91,29,1463,278,487,326,524,84,424,366,91,87,82,15,91,29,1349,82,366,91,87,82,17,91,29,323,323,715,366,91,87,82,16,91,29,366,91,87,82,15,91,29,366,91,87,82,16,91,29,366,91,87,82,19,91,29,366,91,87,82,19,91,29,366,91,87,82,16,91,29,1182,323,312],"tokens":["f","ast","Ġ<","|","x","s","2","|",">","Ġch","un","k","Ġ<","|","x","s","1","|",">","Ġ<","|","x","s","4","|",">","Ġ<","|","x","s","3","|",">","Ġ<","|","x","s","4","|",">","Ġmod","al","ity","Ġl","ang","u","age","Ġ<","|","x","s","0","|",">","Ġread","s","Ġ<","|","x","s","2","|",">","Ġand","Ġand","ĠĊ","Ġ<","|","x","s","1","|",">","Ġ<","|","x","s","0","|",">","Ġ<","|","x","s","1","|",">","Ġ<","|","x","s","4","|",">","Ġ<","|","x","s","4","|",">","Ġ<","|","x","s","1","|",">","Ġback","Ġand","Ġre"],"offsets":[[0,1],[1,4],[4,6],[6,7],[7,8],[8,9],[9,10],[10,11],[11,12],[12,15],[15,17],[17,18],[18,20],[20,21],[21,22],[22,23],[23,24],[24,25],[25,26],[26,28],[28,29],[29,30],[30,31],[31,32],[32,33],[33,34],[34,36],[36,37],[37,38],[38,39],[39,40],[40,41],[41,42],[42,44],[44,45],[45,46],[46,47],[47,48],[48,49],[49,50],[50,54],[54,56],[56,59],[59,61],[61,64],[64,65],[65,68],[68,70],[70,71],[71,72],[72,73],[73,74],[74,75],[75,76],[76,81],[81,82],[82,84],[84,85],[85,86],[86,87],[87,88],[88,89],[89,90],[90,94],[94,98],[98,100],[100,102],[102,103],[103,104],[104,105],[105,106],[106,107],[107,108],[108,110],[110,111],[111,112],[112,113],[113,114],[114,115],[115,116],[116,118],[118,119],[119,120],[120,121],[121,122],[122,123],[123,124],[124,126],[126,127],[127,128],[128,129],[129,130],[130,131],[131,132],[132,134],[134,135],[135,136],[136,137],[137,138],[138,139],[139,140],[140,142],[142,143],[143,144],[144,145],[145,146],[146,147],[147,148],[148,153],[153,157],[157,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,1,1,2,2,3,4,4,5,5,5,6,6,7,7,8,9,9,10,10,11,11,12,13,13,14,14,15,15,16,17,17,18,18,19,19,20,21,21,22,22,22,23,23,23,23,24,24,25,25,26,27,27,28,28,29,29,30,30,31,32,32,33,34,35,36,36,37,37,38,39,39,40,40,41,41,42,43,43,44,44,45,45,46,47,47,48,48,49,49,50,51,51,52,52,53,53,54,55,55,56,56,57,57,58,59,59,60,61,62],"decoded":"fast <|xs2|> chunk <|xs1|> <|xs4|> <|xs3|> <|xs4|> modality language <|xs0|> reads <|xs2|> and and \n <|xs1|> <|xs0|> <|xs1|> <|xs4|> <|xs4|> <|xs1|> back and re","decoded_with_specials":"fast <|xs2|> chunk <|xs1|> <|xs4|> <|xs3|> <|xs4|> modality language <|xs0|> reads <|xs2|> and and \n <|xs1|> <|xs0|> <|xs1|> <|xs4|> <|xs4|> <|xs1|> back and re"} +{"kind":"sample","source":"fixtures/modalities/added_special_sparse.txt[:160]","text":"<|xs0|> every for encode <|xs0|> bytes the normalize decode text model <|xs4|> <|xs3|> and \n and <|xs3|> here <|xs1|> again model here text <|xs2|> <|xs2|> lang","ids":[27,91,87,82,15,91,29,1449,369,662,1851,366,91,87,82,15,91,29,553,83,288,279,308,493,278,551,1622,534,1467,1614,366,91,87,82,19,91,29,366,91,87,82,18,91,29,323,715,323,366,91,87,82,18,91,29,1588,366,91,87,82,16,91,29,1549,1614,1588,1467,366,91,87,82,17,91,29,366,91,87,82,17,91,29,326,524],"ids_no_specials":[27,91,87,82,15,91,29,1449,369,662,1851,366,91,87,82,15,91,29,553,83,288,279,308,493,278,551,1622,534,1467,1614,366,91,87,82,19,91,29,366,91,87,82,18,91,29,323,715,323,366,91,87,82,18,91,29,1588,366,91,87,82,16,91,29,1549,1614,1588,1467,366,91,87,82,17,91,29,366,91,87,82,17,91,29,326,524],"tokens":["<","|","x","s","0","|",">","Ġevery","Ġfor","Ġen","code","Ġ<","|","x","s","0","|",">","Ġby","t","es","Ġthe","Ġn","orm","al","ize","Ġdec","ode","Ġtext","Ġmodel","Ġ<","|","x","s","4","|",">","Ġ<","|","x","s","3","|",">","Ġand","ĠĊ","Ġand","Ġ<","|","x","s","3","|",">","Ġhere","Ġ<","|","x","s","1","|",">","Ġagain","Ġmodel","Ġhere","Ġtext","Ġ<","|","x","s","2","|",">","Ġ<","|","x","s","2","|",">","Ġl","ang"],"offsets":[[0,1],[1,2],[2,3],[3,4],[4,5],[5,6],[6,7],[7,13],[13,17],[17,20],[20,24],[24,26],[26,27],[27,28],[28,29],[29,30],[30,31],[31,32],[32,35],[35,36],[36,38],[38,42],[42,44],[44,47],[47,49],[49,52],[52,56],[56,59],[59,64],[64,70],[70,72],[72,73],[73,74],[74,75],[75,76],[76,77],[77,78],[78,80],[80,81],[81,82],[82,83],[83,84],[84,85],[85,86],[86,90],[90,92],[92,96],[96,98],[98,99],[99,100],[100,101],[101,102],[102,103],[103,104],[104,109],[109,111],[111,112],[112,113],[113,114],[114,115],[115,116],[116,117],[117,123],[123,129],[129,134],[134,139],[139,141],[141,142],[142,143],[143,144],[144,145],[145,146],[146,147],[147,149],[149,150],[150,151],[151,152],[152,153],[153,154],[154,155],[155,157],[157,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,1,1,2,3,3,4,5,6,6,7,7,8,8,9,10,10,11,11,11,12,13,13,13,13,14,14,15,16,17,17,18,18,19,20,20,21,21,22,22,23,24,24,25,26,27,28,28,29,29,30,31,31,32,33,33,34,34,35,36,36,37,38,39,40,41,41,42,42,43,44,44,45,45,46,46,47,48,48,49,49],"decoded":"<|xs0|> every for encode <|xs0|> bytes the normalize decode text model <|xs4|> <|xs3|> and \n and <|xs3|> here <|xs1|> again model here text <|xs2|> <|xs2|> lang","decoded_with_specials":"<|xs0|> every for encode <|xs0|> bytes the normalize decode text model <|xs4|> <|xs3|> and \n and <|xs3|> here <|xs1|> again model here text <|xs2|> <|xs2|> lang"} +{"kind":"sample","source":"fixtures/modalities/agentic-traces.txt[:160]","text":"Analyze this codebase and summarize what it is and how it works.\nI need to explore the codebase structure by reading the main entry points and configuration fil","ids":[32,77,278,88,89,68,419,272,534,65,519,323,274,372,76,277,551,1128,432,374,323,1246,432,975,82,624,40,1184,311,1343,385,265,279,272,534,65,519,607,667,552,553,1349,287,279,1887,1197,884,1459,82,323,390,904,324,367,1461],"ids_no_specials":[32,77,278,88,89,68,419,272,534,65,519,323,274,372,76,277,551,1128,432,374,323,1246,432,975,82,624,40,1184,311,1343,385,265,279,272,534,65,519,607,667,552,553,1349,287,279,1887,1197,884,1459,82,323,390,904,324,367,1461],"tokens":["A","n","al","y","z","e","Ġthis","Ġc","ode","b","ase","Ġand","Ġs","um","m","ar","ize","Ġwhat","Ġit","Ġis","Ġand","Ġhow","Ġit","Ġwork","s",".Ċ","I","Ġneed","Ġto","Ġexp","lo","re","Ġthe","Ġc","ode","b","ase","Ġstr","uct","ure","Ġby","Ġread","ing","Ġthe","Ġmain","Ġent","ry","Ġpoint","s","Ġand","Ġcon","fig","ur","ation","Ġfil"],"offsets":[[0,1],[1,2],[2,4],[4,5],[5,6],[6,7],[7,12],[12,14],[14,17],[17,18],[18,21],[21,25],[25,27],[27,29],[29,30],[30,32],[32,35],[35,40],[40,43],[43,46],[46,50],[50,54],[54,57],[57,62],[62,63],[63,65],[65,66],[66,71],[71,74],[74,78],[78,80],[80,82],[82,86],[86,88],[88,91],[91,92],[92,95],[95,99],[99,102],[102,105],[105,108],[108,113],[113,116],[116,120],[120,125],[125,129],[129,131],[131,137],[137,138],[138,142],[142,146],[146,149],[149,151],[151,156],[156,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,1,2,2,2,2,3,4,4,4,4,4,5,6,7,8,9,10,11,11,12,13,14,15,16,16,16,17,18,18,18,18,19,19,19,20,21,21,22,23,24,24,25,25,26,27,27,27,27,28],"decoded":"Analyze this codebase and summarize what it is and how it works.\nI need to explore the codebase structure by reading the main entry points and configuration fil","decoded_with_specials":"Analyze this codebase and summarize what it is and how it works.\nI need to explore the codebase structure by reading the main entry points and configuration fil"} +{"kind":"sample","source":"fixtures/modalities/agentic_swe.txt[:160]","text":"[system]\nYou are a helpful assistant that can interact with a computer to solve tasks.\n\n[user]\n\n/testbed\n\nI've uploaded a pytho","ids":[58,82,612,921,2610,525,264,1492,1262,1071,380,517,429,646,946,531,448,264,469,628,261,311,274,337,586,259,1073,82,382,58,872,921,27,454,1078,291,761,457,82,397,14,83,477,65,291,198,522,454,1078,291,761,457,82,397,40,6,586,705,1078,291,264,281,88,339,78],"ids_no_specials":[58,82,612,921,2610,525,264,1492,1262,1071,380,517,429,646,946,531,448,264,469,628,261,311,274,337,586,259,1073,82,382,58,872,921,27,454,1078,291,761,457,82,397,14,83,477,65,291,198,522,454,1078,291,761,457,82,397,40,6,586,705,1078,291,264,281,88,339,78],"tokens":["[","s","ystem","]Ċ","You","Ġare","Ġa","Ġhelp","ful","Ġass","ist","ant","Ġthat","Ġcan","Ġinter","act","Ġwith","Ġa","Ġcom","put","er","Ġto","Ġs","ol","ve","Ġt","ask","s",".ĊĊ","[","user","]Ċ","<","up","load","ed","_f","ile","s",">Ċ","/","t","est","b","ed","Ċ","Ċ","I","'","ve","Ġup","load","ed","Ġa","Ġp","y","th","o"],"offsets":[[0,1],[1,2],[2,7],[7,9],[9,12],[12,16],[16,18],[18,23],[23,26],[26,30],[30,33],[33,36],[36,41],[41,45],[45,51],[51,54],[54,59],[59,61],[61,65],[65,68],[68,70],[70,73],[73,75],[75,77],[77,79],[79,81],[81,84],[84,85],[85,88],[88,89],[89,93],[93,95],[95,96],[96,98],[98,102],[102,104],[104,106],[106,109],[109,110],[110,112],[112,113],[113,114],[114,117],[117,118],[118,120],[120,121],[121,123],[123,125],[125,129],[129,131],[131,133],[133,136],[136,137],[137,139],[139,140],[140,141],[141,143],[143,146],[146,150],[150,152],[152,154],[154,156],[156,157],[157,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,1,2,3,4,5,5,6,6,6,7,8,9,9,10,11,12,12,12,13,14,14,14,15,15,15,16,17,17,18,19,19,19,19,20,20,20,21,22,22,22,22,22,23,24,25,25,25,26,26,26,27,28,29,29,30,30,30,31,32,32,32,32],"decoded":"[system]\nYou are a helpful assistant that can interact with a computer to solve tasks.\n\n[user]\n\n/testbed\n\nI've uploaded a pytho","decoded_with_specials":"[system]\nYou are a helpful assistant that can interact with a computer to solve tasks.\n\n[user]\n\n/testbed\n\nI've uploaded a pytho"} +{"kind":"sample","source":"fixtures/modalities/code_mixed.txt[:160]","text":"// django-f3f9601cff03b38942e70ce8042bdfdec6002449/django/__init__.py\nfrom django.utils.version import get_version\n\nVERSION = (6, 2, 0, \"alpha\", 0)\n\n__version__","ids":[322,294,73,524,78,12,69,18,69,24,21,15,16,66,542,15,18,65,18,23,24,19,17,68,22,15,346,23,15,19,17,65,67,69,450,66,21,15,15,17,19,19,24,14,67,73,524,78,14,563,258,275,563,13,3288,198,1499,294,73,524,78,13,1314,82,13,85,1325,1159,633,62,85,1325,271,53,640,50,1271,284,320,21,11,220,17,11,220,15,11,330,278,759,64,497,220,15,692,563,85,1325,563],"ids_no_specials":[322,294,73,524,78,12,69,18,69,24,21,15,16,66,542,15,18,65,18,23,24,19,17,68,22,15,346,23,15,19,17,65,67,69,450,66,21,15,15,17,19,19,24,14,67,73,524,78,14,563,258,275,563,13,3288,198,1499,294,73,524,78,13,1314,82,13,85,1325,1159,633,62,85,1325,271,53,640,50,1271,284,320,21,11,220,17,11,220,15,11,330,278,759,64,497,220,15,692,563,85,1325,563],"tokens":["//","Ġd","j","ang","o","-","f","3","f","9","6","0","1","c","ff","0","3","b","3","8","9","4","2","e","7","0","ce","8","0","4","2","b","d","f","de","c","6","0","0","2","4","4","9","/","d","j","ang","o","/","__","in","it","__",".","py","Ċ","from","Ġd","j","ang","o",".","util","s",".","v","ersion","Ġimport","Ġget","_","v","ersion","ĊĊ","V","ER","S","ION","Ġ=","Ġ(","6",",","Ġ","2",",","Ġ","0",",","Ġ\"","al","ph","a","\",","Ġ","0",")ĊĊ","__","v","ersion","__"],"offsets":[[0,2],[2,4],[4,5],[5,8],[8,9],[9,10],[10,11],[11,12],[12,13],[13,14],[14,15],[15,16],[16,17],[17,18],[18,20],[20,21],[21,22],[22,23],[23,24],[24,25],[25,26],[26,27],[27,28],[28,29],[29,30],[30,31],[31,33],[33,34],[34,35],[35,36],[36,37],[37,38],[38,39],[39,40],[40,42],[42,43],[43,44],[44,45],[45,46],[46,47],[47,48],[48,49],[49,50],[50,51],[51,52],[52,53],[53,56],[56,57],[57,58],[58,60],[60,62],[62,64],[64,66],[66,67],[67,69],[69,70],[70,74],[74,76],[76,77],[77,80],[80,81],[81,82],[82,86],[86,87],[87,88],[88,89],[89,95],[95,102],[102,106],[106,107],[107,108],[108,114],[114,116],[116,117],[117,119],[119,120],[120,123],[123,125],[125,127],[127,128],[128,129],[129,130],[130,131],[131,132],[132,133],[133,134],[134,135],[135,137],[137,139],[139,141],[141,142],[142,144],[144,145],[145,146],[146,149],[149,151],[151,152],[152,158],[158,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,1,1,1,1,2,2,3,4,5,5,5,6,7,7,8,8,9,10,10,10,11,11,12,13,13,14,15,15,15,16,17,17,17,17,17,18,18,18,19,19,19,20,21,21,21,21,21,22,22,23,23,24,24,25,26,27,28,28,28,28,29,29,29,30,30,30,31,32,33,33,33,34,35,35,35,35,36,37,38,39,40,41,42,43,44,45,46,47,47,47,48,49,50,51,52,53,53,54],"decoded":"// django-f3f9601cff03b38942e70ce8042bdfdec6002449/django/__init__.py\nfrom django.utils.version import get_version\n\nVERSION = (6, 2, 0, \"alpha\", 0)\n\n__version__","decoded_with_specials":"// django-f3f9601cff03b38942e70ce8042bdfdec6002449/django/__init__.py\nfrom django.utils.version import get_version\n\nVERSION = (6, 2, 0, \"alpha\", 0)\n\n__version__"} +{"kind":"sample","source":"fixtures/modalities/math_latex.txt[:160]","text":"Bayes and his Theorem\n\nMy earlier post on Bayesian probability seems to have generated quite a lot of readers, so this lunchtime I thought I’d add a little bit ","ids":[33,352,288,323,806,126,254,785,460,76,271,44,88,384,277,742,261,1736,389,425,352,288,1103,462,65,370,1403,511,336,82,311,614,1766,657,922,632,264,326,354,315,1349,388,11,773,419,326,359,331,1678,358,270,283,70,426,358,527,67,912,264,326,1442,273,293,275,220],"ids_no_specials":[33,352,288,323,806,126,254,785,460,76,271,44,88,384,277,742,261,1736,389,425,352,288,1103,462,65,370,1403,511,336,82,311,614,1766,657,922,632,264,326,354,315,1349,388,11,773,419,326,359,331,1678,358,270,283,70,426,358,527,67,912,264,326,1442,273,293,275,220],"tokens":["B","ay","es","Ġand","Ġhis","Â","ł","The","ore","m","ĊĊ","M","y","Ġe","ar","li","er","Ġpost","Ġon","ĠB","ay","es","ian","Ġpro","b","ab","ility","Ġse","em","s","Ġto","Ġhave","Ġgener","ated","Ġqu","ite","Ġa","Ġl","ot","Ġof","Ġread","ers",",","Ġso","Ġthis","Ġl","un","ch","time","ĠI","Ġth","ou","g","ht","ĠI","âĢĻ","d","Ġadd","Ġa","Ġl","itt","le","Ġb","it","Ġ"],"offsets":[[0,1],[1,3],[3,5],[5,9],[9,13],[13,14],[13,14],[14,17],[17,20],[20,21],[21,23],[23,24],[24,25],[25,27],[27,29],[29,31],[31,33],[33,38],[38,41],[41,43],[43,45],[45,47],[47,50],[50,54],[54,55],[55,57],[57,62],[62,65],[65,67],[67,68],[68,71],[71,76],[76,82],[82,86],[86,89],[89,92],[92,94],[94,96],[96,98],[98,101],[101,106],[106,109],[109,110],[110,113],[113,118],[118,120],[120,122],[122,124],[124,128],[128,130],[130,133],[133,135],[135,136],[136,138],[138,140],[140,141],[141,142],[142,146],[146,148],[148,150],[150,153],[153,155],[155,157],[157,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,1,2,3,3,3,3,3,4,5,5,6,6,6,6,7,8,9,9,9,9,10,10,10,10,11,11,11,12,13,14,14,15,15,16,17,17,18,19,19,20,21,22,23,23,23,23,24,25,25,25,25,26,27,27,28,29,30,30,30,31,31,32],"decoded":"Bayes and his Theorem\n\nMy earlier post on Bayesian probability seems to have generated quite a lot of readers, so this lunchtime I thought I’d add a little bit ","decoded_with_specials":"Bayes and his Theorem\n\nMy earlier post on Bayesian probability seems to have generated quite a lot of readers, so this lunchtime I thought I’d add a little bit "} +{"kind":"pair","text":"What is the capital of France?","pair":"Paris is the capital of France.","ids":[1639,266,374,279,272,391,275,278,315,434,81,681,30,47,277,285,374,279,272,391,275,278,315,434,81,681,13],"tokens":["Wh","at","Ġis","Ġthe","Ġc","ap","it","al","Ġof","ĠF","r","ance","?","P","ar","is","Ġis","Ġthe","Ġc","ap","it","al","Ġof","ĠF","r","ance","."],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1],"sequence_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"offsets":[[0,2],[2,4],[4,7],[7,11],[11,13],[13,15],[15,17],[17,19],[19,22],[22,24],[24,25],[25,29],[29,30],[0,1],[1,3],[3,5],[5,8],[8,12],[12,14],[14,16],[16,18],[18,20],[20,23],[23,25],[25,26],[26,30],[30,31]]} +{"kind":"pair","text":"Question in English?","pair":"Ответ на русском языке, с цифрами 123.","ids":[48,361,267,290,304,468,968,1672,30,140,252,1792,140,110,1504,1792,1278,121,1478,220,141,222,141,225,141,223,141,223,140,118,1456,140,120,220,141,237,140,115,141,233,140,118,1504,11,220,141,223,220,141,228,1802,141,226,141,222,1478,140,120,1802,220,16,17,18,13],"tokens":["Q","ue","st","ion","Ġin","ĠE","ng","lish","?","Ð","ŀ","ÑĤ","Ð","²","е","ÑĤ","ĠÐ","½","а","Ġ","Ñ","Ģ","Ñ","ĥ","Ñ","ģ","Ñ","ģ","Ð","º","о","Ð","¼","Ġ","Ñ","ı","Ð","·","Ñ","ĭ","Ð","º","е",",","Ġ","Ñ","ģ","Ġ","Ñ","Ĩ","и","Ñ","Ħ","Ñ","Ģ","а","Ð","¼","и","Ġ","1","2","3","."],"type_ids":[0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],"sequence_ids":[0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"offsets":[[0,1],[1,3],[3,5],[5,8],[8,11],[11,13],[13,15],[15,19],[19,20],[0,1],[0,1],[1,2],[2,3],[2,3],[3,4],[4,5],[5,7],[6,7],[7,8],[8,9],[9,10],[9,10],[10,11],[10,11],[11,12],[11,12],[12,13],[12,13],[13,14],[13,14],[14,15],[15,16],[15,16],[16,17],[17,18],[17,18],[18,19],[18,19],[19,20],[19,20],[20,21],[20,21],[21,22],[22,23],[23,24],[24,25],[24,25],[25,26],[26,27],[26,27],[27,28],[28,29],[28,29],[29,30],[29,30],[30,31],[31,32],[31,32],[32,33],[33,34],[34,35],[35,36],[36,37],[37,38]]} +{"kind":"digest","file":"fixtures/lang/amh_Ethi.txt","cap_chars":200000,"text_sha256":"e353d32f45b449b76fb65ee48f1522086c02a6bffa607e73ed8c875c03c84908","n_tokens":410589,"ids_sha256":"94e81d1b6ac841f9239a293e251611d9bd837e560571fab41f201cb2d197b170"} +{"kind":"digest","file":"fixtures/lang/arb_Arab.txt","cap_chars":200000,"text_sha256":"529587117db0a7abaa2c8f31ab05d7b2f77e84a11068b46e26b2b37774d836c7","n_tokens":355681,"ids_sha256":"039bdac486f360bb7261ef6266085a6b703d02c5d630ce7f65275a8e8e9dfb7e"} +{"kind":"digest","file":"fixtures/lang/ben_Beng.txt","cap_chars":200000,"text_sha256":"915afd7dbbea8256bdb6112b16424c83aa47dae63c5cca0ebadfc6c227d2b89f","n_tokens":530179,"ids_sha256":"b7fd1aac486e3e1f0ed7dfda97f5ffb88deff0543fbb2ac784587b3554a2122d"} +{"kind":"digest","file":"fixtures/lang/cmn_Hani.txt","cap_chars":200000,"text_sha256":"1273c0c093a00739be9efea7804b10a295f902359d11663306d097211934c6df","n_tokens":513886,"ids_sha256":"940b43b90fdf2dff94028324216d26b5bb612abd8bab5182632b7a177545826b"} +{"kind":"digest","file":"fixtures/lang/ell_Grek.txt","cap_chars":200000,"text_sha256":"158c8d3e4933091099bf12a1e21f57c7b7ff3f5381d4b25901eb1fa7ca0926f4","n_tokens":359866,"ids_sha256":"321ef250e3cad536fa88e9160771e28146daeef27c0fdc4e6d753d54e7a8b259"} +{"kind":"digest","file":"fixtures/lang/eng_Latn.txt","cap_chars":200000,"text_sha256":"d998cb3153e3a56c2ecdf49293062e8bc6ca43365b6c91cfed147d73b84c19fb","n_tokens":83060,"ids_sha256":"b7c0d22a5b5c4d3cecf41723028cee31b9cfebc0a95d01dbc90a1b9c82a9aae5"} +{"kind":"digest","file":"fixtures/lang/heb_Hebr.txt","cap_chars":200000,"text_sha256":"386da8c0e0d416bffbe84b51baf895c67dbce6e0fd153be597933261794300fd","n_tokens":348415,"ids_sha256":"d2bb400fa8dd84ba405a4304c4080085c41fb1008e61e2414726f67a91134dc4"} +{"kind":"digest","file":"fixtures/lang/hin_Deva.txt","cap_chars":200000,"text_sha256":"2d114148c81035b1b63c94c7e9e805933c215d8aa9500d916923aec0a124ddde","n_tokens":504565,"ids_sha256":"da3baad15a4c8c01440ef1e4e84ee0840848d484e11a262b78d29140186ebf2a"} +{"kind":"digest","file":"fixtures/lang/jpn_Jpan.txt","cap_chars":200000,"text_sha256":"2e55c036f500d7720d020be2db45c87a49c17d3b7e63b30bada9cef4cb6aa158","n_tokens":534909,"ids_sha256":"a0962e7d49bce657d1e6ef026e000d54be3fdcdd128891e1881ecc052036e0d5"} +{"kind":"digest","file":"fixtures/lang/kat_Geor.txt","cap_chars":200000,"text_sha256":"c69a1aa8fb80b41459c98c15c756eb8f39a9151468b1d5b3f9bbffe4d82eb87a","n_tokens":527743,"ids_sha256":"4f7bf36895dc43ebf7fa8b35f8c6d125a52eb510768e6a82085da601440f74fe"} +{"kind":"digest","file":"fixtures/lang/kor_Hang.txt","cap_chars":200000,"text_sha256":"68659781c288136c1caffcb10eec9130406ae8a7b003c1e327b7e3601eba041d","n_tokens":448889,"ids_sha256":"b1f4cea840fc7a40834a94eb38d05a2edcadb08313fb41b42ec4a6a2fd392b2e"} +{"kind":"digest","file":"fixtures/lang/rus_Cyrl.txt","cap_chars":200000,"text_sha256":"5f967a82627afbd72c2f21c6e56183dfa5efc986ba34350c1cf9eea6c27a6132","n_tokens":278529,"ids_sha256":"ff8914566c3797925146a39e0cfe24603905a6dc90d35d7393d8f946e6f3bc4a"} +{"kind":"digest","file":"fixtures/lang/tam_Taml.txt","cap_chars":200000,"text_sha256":"bf09e1bca55aeab34162d5dc61afec7ac976b4e66eb5833ac5a037305911d165","n_tokens":537683,"ids_sha256":"6a9b67e85a6b40b8353a2594ad707d7fb588f77a3bfff3066d08a65aaff1d53b"} +{"kind":"digest","file":"fixtures/lang/tha_Thai.txt","cap_chars":200000,"text_sha256":"25209f4750b3d3f04440f98aeff50316a118cb982462a4763106e8747172b79c","n_tokens":555573,"ids_sha256":"db71b6f14a903f3f0fa6ad546381d861c101390b700176acdf048208d9956aef"} +{"kind":"digest","file":"fixtures/modalities/added_normalized_dense.txt","cap_chars":200000,"text_sha256":"9d49b3cacf40962c051a09f1350a0f5dc2fc210556889ed0f341cfe0cf0af4f9","n_tokens":51772,"ids_sha256":"3ca0ad0995208c6c31bdc2f1ccd973efe2298360793b82f144ca0f7362d3d75e"} +{"kind":"digest","file":"fixtures/modalities/added_normalized_sparse.txt","cap_chars":200000,"text_sha256":"55383f52b02a5d9686f9e4347729e5c08e61b9cff77913bd94ceb65c2fdf7fe4","n_tokens":38923,"ids_sha256":"84fa9bf9df62bb09f862f8ecd3c214e8f42fb7089b218128ded445a6347d2f17"} +{"kind":"digest","file":"fixtures/modalities/added_special_dense.txt","cap_chars":200000,"text_sha256":"472f90f6f2f5803626df5d7088f8ef12984de6e88fedf67723887511d35b64b2","n_tokens":62290,"ids_sha256":"099e71317aa5f6b38726897de80eec3ee8b92b52a7a44d046b02acd40d7998ac"} +{"kind":"digest","file":"fixtures/modalities/added_special_sparse.txt","cap_chars":200000,"text_sha256":"9c52602f7c8d009b11377743f95e5bfe9053694fc6b48c3dd6c1c6dc0681ab5a","n_tokens":41524,"ids_sha256":"776dc8e085461aaf6bf0358bebeab513718b1784360b5ccd1db23218a28414e7"} +{"kind":"digest","file":"fixtures/modalities/agentic-traces.txt","cap_chars":200000,"text_sha256":"ae9d63c1adc49f572d9af635ac1b0421461e4d1b92c5f35ea572980ff1e2480c","n_tokens":89543,"ids_sha256":"1eae26d5632e42b52febfc6d72478122a67b9221626c5f39ac43c80194160d64"} +{"kind":"digest","file":"fixtures/modalities/agentic_swe.txt","cap_chars":200000,"text_sha256":"b2d31e91ff91bb6c9a3aec3832d25acbecf9b58d5c0674e364d3b287182e7253","n_tokens":100599,"ids_sha256":"5006117d7f2444efde36fa91c66a83400c6d60591571bed9b244c7d27544a21c"} +{"kind":"digest","file":"fixtures/modalities/code_mixed.txt","cap_chars":200000,"text_sha256":"0fa7314727726db056433d36bc1bda021803be7f1d150cae83e362a3ff41821d","n_tokens":108739,"ids_sha256":"f55a6da417203450f48aeb91fdf1170ad21d9a3ce972f898021d24e199c10e90"} +{"kind":"digest","file":"fixtures/modalities/math_latex.txt","cap_chars":200000,"text_sha256":"2d17be8704f1f7b4f2174a054c0b85d6d22039e00be8e4a487927d27f3852e90","n_tokens":86437,"ids_sha256":"98378d7cc8572eed5ee200c8475e46511cc013239da030107264ab3095910607"} diff --git a/bindings/python/tests/golden/goldens/gpt-oss.jsonl b/bindings/python/tests/golden/goldens/gpt-oss.jsonl new file mode 100644 index 000000000..e4dd29dcb --- /dev/null +++ b/bindings/python/tests/golden/goldens/gpt-oss.jsonl @@ -0,0 +1,64 @@ +{"kind":"meta","model":"gpt-oss","tokenizer_file":"gpt-oss-slim.json","tokenizers_version":"0.23.1"} +{"kind":"sample","source":"edge:empty","text":"","ids":[],"ids_no_specials":[],"tokens":[],"offsets":[],"type_ids":[],"special_tokens_mask":[],"word_ids":[],"decoded":"","decoded_with_specials":""} +{"kind":"sample","source":"edge:spaces","text":" ","ids":[271],"ids_no_specials":[271],"tokens":["ĠĠĠ"],"offsets":[[0,3]],"type_ids":[0],"special_tokens_mask":[0],"word_ids":[0],"decoded":" ","decoded_with_specials":" "} +{"kind":"sample","source":"edge:hello","text":"Hello world","ids":[39,596,78,286,1733],"ids_no_specials":[39,596,78,286,1733],"tokens":["H","ell","o","Ġw","orld"],"offsets":[[0,1],[1,4],[4,5],[5,7],[7,11]],"type_ids":[0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0],"word_ids":[0,0,0,1,1],"decoded":"Hello world","decoded_with_specials":"Hello world"} +{"kind":"sample","source":"edge:punctuation","text":"Hello, world!! How's it going? (fine; thanks...)","ids":[39,596,78,11,286,1733,0,0,487,384,885,480,810,289,30,350,69,514,26,325,1104,82,1008,8],"ids_no_specials":[39,596,78,11,286,1733,0,0,487,384,885,480,810,289,30,350,69,514,26,325,1104,82,1008,8],"tokens":["H","ell","o",",","Ġw","orld","!","!","ĠH","ow","'s","Ġit","Ġgo","ing","?","Ġ(","f","ine",";","Ġth","ank","s","...",")"],"offsets":[[0,1],[1,4],[4,5],[5,6],[6,8],[8,12],[12,13],[13,14],[14,16],[16,18],[18,20],[20,23],[23,26],[26,29],[29,30],[30,32],[32,33],[33,36],[36,37],[37,40],[40,43],[43,44],[44,47],[47,48]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,1,2,2,3,3,4,4,4,5,6,6,7,8,9,9,10,11,11,11,12,12],"decoded":"Hello, world!! How's it going? (fine; thanks...)","decoded_with_specials":"Hello, world!! How's it going? (fine; thanks...)"} +{"kind":"sample","source":"edge:whitespace-mix","text":"line one\nline two\r\n\tindented\n trailing ","ids":[1137,1001,198,1137,1920,370,197,521,299,295,198,220,498,663,289,256],"ids_no_specials":[1137,1001,198,1137,1920,370,197,521,299,295,198,220,498,663,289,256],"tokens":["line","Ġone","Ċ","line","Ġtwo","čĊ","ĉ","ind","ent","ed","Ċ","Ġ","Ġtr","ail","ing","ĠĠ"],"offsets":[[0,4],[4,8],[8,9],[9,13],[13,17],[17,19],[19,20],[20,23],[23,26],[26,28],[28,29],[29,30],[30,33],[33,36],[36,39],[39,41]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,1,2,3,4,5,6,6,6,6,7,8,9,9,9,10],"decoded":"line one\nline two\r\n\tindented\n trailing ","decoded_with_specials":"line one\nline two\r\n\tindented\n trailing "} +{"kind":"sample","source":"edge:accents","text":"café naïve résumé — déjà vu","ids":[66,1553,377,898,127,107,737,428,1756,394,377,559,242,272,377,73,708,323,84],"ids_no_specials":[66,1553,377,898,127,107,737,428,1756,394,377,559,242,272,377,73,708,323,84],"tokens":["c","af","é","Ġna","Ã","¯","ve","Ġr","és","um","é","ĠâĢ","Ķ","Ġd","é","j","Ãł","Ġv","u"],"offsets":[[0,1],[1,3],[3,4],[4,7],[7,8],[7,8],[8,10],[10,12],[12,14],[14,16],[16,17],[17,19],[18,19],[19,21],[21,22],[22,23],[23,24],[24,26],[26,27]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,1,1,1,1,2,2,2,2,3,3,4,4,4,4,5,5],"decoded":"café naïve résumé — déjà vu","decoded_with_specials":"café naïve résumé — déjà vu"} +{"kind":"sample","source":"edge:accents-decomposed","text":"café naïve résumé","ids":[66,1553,68,136,223,898,72,136,230,737,322,136,223,82,394,68,136,223],"ids_no_specials":[66,1553,68,136,223,898,72,136,230,737,322,136,223,82,394,68,136,223],"tokens":["c","af","e","Ì","ģ","Ġna","i","Ì","Ī","ve","Ġre","Ì","ģ","s","um","e","Ì","ģ"],"offsets":[[0,1],[1,3],[3,4],[4,5],[4,5],[5,8],[8,9],[9,10],[9,10],[10,12],[12,15],[15,16],[15,16],[16,17],[17,19],[19,20],[20,21],[20,21]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,1,1,1,1,1,2,2,2,2,2,2,2,2],"decoded":"café naïve résumé","decoded_with_specials":"café naïve résumé"} +{"kind":"sample","source":"edge:emoji","text":"🤗 emoji, families 👨‍👩‍👧‍👦, flags 🇫🇷 and skin tones 👍🏽","ids":[172,253,97,245,863,78,73,72,11,2079,311,566,220,172,253,239,101,318,235,172,253,239,102,318,235,172,253,239,100,318,235,172,253,239,99,11,1548,348,82,220,172,253,229,104,172,253,229,115,326,1705,258,260,263,268,220,172,253,239,235,172,253,237,121],"ids_no_specials":[172,253,97,245,863,78,73,72,11,2079,311,566,220,172,253,239,101,318,235,172,253,239,102,318,235,172,253,239,100,318,235,172,253,239,99,11,1548,348,82,220,172,253,229,104,172,253,229,115,326,1705,258,260,263,268,220,172,253,239,235,172,253,237,121],"tokens":["ð","Ł","¤","Ĺ","Ġem","o","j","i",",","Ġfam","il","ies","Ġ","ð","Ł","ij","¨","âĢ","į","ð","Ł","ij","©","âĢ","į","ð","Ł","ij","§","âĢ","į","ð","Ł","ij","¦",",","Ġfl","ag","s","Ġ","ð","Ł","ĩ","«","ð","Ł","ĩ","·","Ġand","Ġsk","in","Ġt","on","es","Ġ","ð","Ł","ij","į","ð","Ł","ı","½"],"offsets":[[0,1],[0,1],[0,1],[0,1],[1,4],[4,5],[5,6],[6,7],[7,8],[8,12],[12,14],[14,17],[17,18],[18,19],[18,19],[18,19],[18,19],[19,20],[19,20],[20,21],[20,21],[20,21],[20,21],[21,22],[21,22],[22,23],[22,23],[22,23],[22,23],[23,24],[23,24],[24,25],[24,25],[24,25],[24,25],[25,26],[26,29],[29,31],[31,32],[32,33],[33,34],[33,34],[33,34],[33,34],[34,35],[34,35],[34,35],[34,35],[35,39],[39,42],[42,44],[44,46],[46,48],[48,50],[50,51],[51,52],[51,52],[51,52],[51,52],[52,53],[52,53],[52,53],[52,53]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,1,1,1,1,2,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,5,5,5,6,6,6,6,6,6,6,6,6,7,8,8,9,9,9,10,10,10,10,10,10,10,10,10],"decoded":"🤗 emoji, families 👨‍👩‍👧‍👦, flags 🇫🇷 and skin tones 👍🏽","decoded_with_specials":"🤗 emoji, families 👨‍👩‍👧‍👦, flags 🇫🇷 and skin tones 👍🏽"} +{"kind":"sample","source":"edge:cjk","text":"漢字とひらがなとカタカナが混ざった文章です。","ids":[162,120,95,161,255,245,605,101,605,110,845,231,605,234,605,103,605,101,845,104,845,123,845,104,769,232,605,234,162,115,115,605,244,605,96,605,253,1615,229,163,104,254,605,100,605,247,788],"ids_no_specials":[162,120,95,161,255,245,605,101,605,110,845,231,605,234,605,103,605,101,845,104,845,123,845,104,769,232,605,234,162,115,115,605,244,605,96,605,253,1615,229,163,104,254,605,100,605,247,788],"tokens":["æ","¼","¢","å","Ń","Ĺ","ãģ","¨","ãģ","²","ãĤ","ī","ãģ","Į","ãģ","ª","ãģ","¨","ãĤ","«","ãĤ","¿","ãĤ","«","ãĥ","Ĭ","ãģ","Į","æ","·","·","ãģ","ĸ","ãģ","£","ãģ","Ł","æĸ","ĩ","ç","«","ł","ãģ","§","ãģ","Ļ","ãĢĤ"],"offsets":[[0,1],[0,1],[0,1],[1,2],[1,2],[1,2],[2,3],[2,3],[3,4],[3,4],[4,5],[4,5],[5,6],[5,6],[6,7],[6,7],[7,8],[7,8],[8,9],[8,9],[9,10],[9,10],[10,11],[10,11],[11,12],[11,12],[12,13],[12,13],[13,14],[13,14],[13,14],[14,15],[14,15],[15,16],[15,16],[16,17],[16,17],[17,18],[17,18],[18,19],[18,19],[18,19],[19,20],[19,20],[20,21],[20,21],[21,22]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"decoded":"漢字とひらがなとカタカナが混ざった文章です。","decoded_with_specials":"漢字とひらがなとカタカナが混ざった文章です。"} +{"kind":"sample","source":"edge:korean","text":"한국어 텍스트 조각","ids":[1364,250,166,113,255,168,244,112,220,169,227,235,168,232,97,169,232,116,804,94,108,166,108,223],"ids_no_specials":[1364,250,166,113,255,168,244,112,220,169,227,235,168,232,97,169,232,116,804,94,108,166,108,223],"tokens":["íķ","ľ","ê","µ","Ń","ì","ĸ","´","Ġ","í","ħ","į","ì","Ĭ","¤","í","Ĭ","¸","Ġì","¡","°","ê","°","ģ"],"offsets":[[0,1],[0,1],[1,2],[1,2],[1,2],[2,3],[2,3],[2,3],[3,4],[4,5],[4,5],[4,5],[5,6],[5,6],[5,6],[6,7],[6,7],[6,7],[7,9],[8,9],[8,9],[9,10],[9,10],[9,10]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2],"decoded":"한국어 텍스트 조각","decoded_with_specials":"한국어 텍스트 조각"} +{"kind":"sample","source":"edge:rtl","text":"مرحبا بالعالم — שלום עולם","ids":[414,433,949,657,320,927,1115,715,1115,414,559,242,463,102,1192,790,251,463,95,790,250,147,251],"ids_no_specials":[414,433,949,657,320,927,1115,715,1115,414,559,242,463,102,1192,790,251,463,95,790,250,147,251],"tokens":["Ùħ","ر","ØŃ","ب","ا","Ġب","اÙĦ","ع","اÙĦ","Ùħ","ĠâĢ","Ķ","Ġ×","©","׾","×ķ×","Ŀ","Ġ×","¢","×ķ×","ľ","×","Ŀ"],"offsets":[[0,1],[1,2],[2,3],[3,4],[4,5],[5,7],[7,9],[9,10],[10,12],[12,13],[13,15],[14,15],[15,17],[16,17],[17,18],[18,20],[19,20],[20,22],[21,22],[22,24],[23,24],[24,25],[24,25]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,1,1,1,1,1,2,2,3,3,3,3,3,4,4,4,4,4,4],"decoded":"مرحبا بالعالم — שלום עולם","decoded_with_specials":"مرحبا بالعالم — שלום עולם"} +{"kind":"sample","source":"edge:numbers","text":"1234567890, 3.14159, 1,000,000th","ids":[899,18,19,20,21,22,23,24,15,11,220,18,13,1265,16,20,24,11,220,16,11,1302,11,1302,404],"ids_no_specials":[899,18,19,20,21,22,23,24,15,11,220,18,13,1265,16,20,24,11,220,16,11,1302,11,1302,404],"tokens":["12","3","4","5","6","7","8","9","0",",","Ġ","3",".","14","1","5","9",",","Ġ","1",",","000",",","000","th"],"offsets":[[0,2],[2,3],[3,4],[4,5],[5,6],[6,7],[7,8],[8,9],[9,10],[10,11],[11,12],[12,13],[13,14],[14,16],[16,17],[17,18],[18,19],[19,20],[20,21],[21,22],[22,23],[23,26],[26,27],[27,30],[30,32]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,1,1,1,2,2,2,3,4,5,6,7,8,8,9,9,10,11,12,13,14,15,16,17],"decoded":"1234567890, 3.14159, 1,000,000th","decoded_with_specials":"1234567890, 3.14159, 1,000,000th"} +{"kind":"sample","source":"edge:code","text":"def f(x):\n return x**2 # squared\nprint(f'{f(3)=}')","ids":[1314,285,7,87,1883,271,622,1215,410,17,220,1069,265,351,644,67,198,1598,1526,6,90,69,7,18,8,28,92,1542],"ids_no_specials":[1314,285,7,87,1883,271,622,1215,410,17,220,1069,265,351,644,67,198,1598,1526,6,90,69,7,18,8,28,92,1542],"tokens":["def","Ġf","(","x","):Ċ","ĠĠĠ","Ġreturn","Ġx","**","2","Ġ","Ġ#","Ġs","qu","are","d","Ċ","print","(f","'","{","f","(","3",")","=","}","')"],"offsets":[[0,3],[3,5],[5,6],[6,7],[7,10],[10,13],[13,20],[20,22],[22,24],[24,25],[25,26],[26,28],[28,30],[30,32],[32,35],[35,36],[36,37],[37,42],[42,44],[44,45],[45,46],[46,47],[47,48],[48,49],[49,50],[50,51],[51,52],[52,54]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,1,2,2,3,4,5,6,7,8,9,10,11,11,11,11,12,13,14,15,15,16,17,18,19,19,19,19],"decoded":"def f(x):\n return x**2 # squared\nprint(f'{f(3)=}')","decoded_with_specials":"def f(x):\n return x**2 # squared\nprint(f'{f(3)=}')"} +{"kind":"sample","source":"edge:long-word","text":"Donaudampfschifffahrtsgesellschaftskapitänsmützenabzeichen","ids":[35,263,64,527,1515,69,82,332,366,608,849,81,1561,70,268,596,82,332,64,826,82,74,403,278,450,77,82,76,572,83,89,262,378,1547,703,262],"ids_no_specials":[35,263,64,527,1515,69,82,332,366,608,849,81,1561,70,268,596,82,332,64,826,82,74,403,278,450,77,82,76,572,83,89,262,378,1547,703,262],"tokens":["D","on","a","ud","amp","f","s","ch","if","ff","ah","r","ts","g","es","ell","s","ch","a","ft","s","k","ap","it","ä","n","s","m","ü","t","z","en","ab","ze","ich","en"],"offsets":[[0,1],[1,3],[3,4],[4,6],[6,9],[9,10],[10,11],[11,13],[13,15],[15,17],[17,19],[19,20],[20,22],[22,23],[23,25],[25,28],[28,29],[29,31],[31,32],[32,34],[34,35],[35,36],[36,38],[38,40],[40,41],[41,42],[42,43],[43,44],[44,45],[45,46],[46,47],[47,49],[49,51],[51,53],[53,56],[56,58]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":"Donaudampfschifffahrtsgesellschaftskapitänsmützenabzeichen","decoded_with_specials":"Donaudampfschifffahrtsgesellschaftskapitänsmützenabzeichen"} +{"kind":"sample","source":"edge:url-email","text":"https://example.com/a/b?q=1&r=2#frag user.name+tag@example.co.uk","ids":[1998,947,1684,490,313,789,1136,14,64,14,65,30,80,28,16,5,81,28,17,2,1739,348,1825,13,897,10,83,348,31,490,313,789,10914,13,1160],"ids_no_specials":[1998,947,1684,490,313,789,1136,14,64,14,65,30,80,28,16,5,81,28,17,2,1739,348,1825,13,897,10,83,348,31,490,313,789,10914,13,1160],"tokens":["htt","ps","://","ex","am","ple",".com","/","a","/","b","?","q","=","1","&","r","=","2","#","fr","ag","Ġuser",".","name","+","t","ag","@","ex","am","ple",".co",".","uk"],"offsets":[[0,3],[3,5],[5,8],[8,10],[10,12],[12,15],[15,19],[19,20],[20,21],[21,22],[22,23],[23,24],[24,25],[25,26],[26,27],[27,28],[28,29],[29,30],[30,31],[31,32],[32,34],[34,36],[36,41],[41,42],[42,46],[46,47],[47,48],[48,50],[50,51],[51,53],[53,55],[55,58],[58,61],[61,62],[62,64]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,1,2,2,2,3,4,4,5,5,6,6,7,8,9,9,10,11,12,12,12,13,14,14,15,15,15,16,16,16,16,17,18,18],"decoded":"https://example.com/a/b?q=1&r=2#frag user.name+tag@example.co.uk","decoded_with_specials":"https://example.com/a/b?q=1&r=2#frag user.name+tag@example.co.uk"} +{"kind":"sample","source":"edge:unicode-spaces","text":"a b c　d","ids":[64,126,254,65,318,231,66,1397,67],"ids_no_specials":[64,126,254,65,318,231,66,1397,67],"tokens":["a","Â","ł","b","âĢ","ī","c","ãĢĢ","d"],"offsets":[[0,1],[1,2],[1,2],[2,3],[3,4],[3,4],[4,5],[5,6],[6,7]],"type_ids":[0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0],"word_ids":[0,1,1,1,2,2,2,3,3],"decoded":"a b c　d","decoded_with_specials":"a b c　d"} +{"kind":"sample","source":"edge:math","text":"∑ᵢ xᵢ² ≤ ∫₀^∞ e⁻ˣ dx ≈ 1","ids":[158,230,239,157,113,95,1215,157,113,95,126,110,220,158,231,97,220,158,230,104,158,224,222,61,158,230,252,319,158,223,119,135,96,272,87,220,158,231,230,220,16],"ids_no_specials":[158,230,239,157,113,95,1215,157,113,95,126,110,220,158,231,97,220,158,230,104,158,224,222,61,158,230,252,319,158,223,119,135,96,272,87,220,158,231,230,220,16],"tokens":["â","Ī","ij","á","µ","¢","Ġx","á","µ","¢","Â","²","Ġ","â","ī","¤","Ġ","â","Ī","«","â","Ĥ","Ģ","^","â","Ī","ŀ","Ġe","â","ģ","»","Ë","£","Ġd","x","Ġ","â","ī","Ī","Ġ","1"],"offsets":[[0,1],[0,1],[0,1],[1,2],[1,2],[1,2],[2,4],[4,5],[4,5],[4,5],[5,6],[5,6],[6,7],[7,8],[7,8],[7,8],[8,9],[9,10],[9,10],[9,10],[10,11],[10,11],[10,11],[11,12],[12,13],[12,13],[12,13],[13,15],[15,16],[15,16],[15,16],[16,17],[16,17],[17,19],[19,20],[20,21],[21,22],[21,22],[21,22],[22,23],[23,24]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,1,1,1,1,2,2,3,3,3,3,4,4,4,4,5,5,5,6,6,6,6,7,8,8,8,8,8,9,9,10,10,10,10,11,12],"decoded":"∑ᵢ xᵢ² ≤ ∫₀^∞ e⁻ˣ dx ≈ 1","decoded_with_specials":"∑ᵢ xᵢ² ≤ ∫₀^∞ e⁻ˣ dx ≈ 1"} +{"kind":"sample","source":"fixtures/lang/amh_Ethi.txt[:160]","text":"- ፍርድ ቤቱ በአቶ ዮናታን ተስፋዬ ላይ የ6 አመት ከ6 ወር ጽኑ የእስር ቅጣት አስተላለፈ\n- አንድነት ዶክተር ቴድሮስ ለዓለም ጤና ድርጅት ዋና ዳይሬክተርነት በመመረጣቸው ደስታውን ገለጸ\n- ህንድ መጠነ ሰፊ የድንጋይ ከሰል ሃይል ጣብያዎች የመገንባት እ","ids":[12,220,157,235,235,157,230,255,157,233,113,220,157,231,97,157,231,109,220,157,231,254,157,232,254,157,231,114,220,157,233,106,157,232,241,157,231,111,157,232,243,220,157,231,108,157,230,113,157,235,233,157,233,105,220,157,230,233,157,233,255,220,157,233,101,21,220,157,232,254,157,230,246,157,231,113,220,157,232,101,21,220,157,233,230,157,230,255,220,157,234,121,157,232,239,220,157,233,101,157,232,98,157,230,113,157,230,255,220,157,231,227,157,234,96,157,231,113,220,157,232,254,157,230,113,157,231,108,157,230,233,157,230,230,157,235,230,198,12,220,157,232,254,157,232,243,157,233,113,157,232,238,157,231,113,220,157,233,114,157,232,255,157,231,108,157,230,255,220,157,231,112,157,233,113,157,230,106,157,230,113,220,157,230,230,157,233,241,157,230,230,157,230,251,220,157,234,97,157,232,241,220,157,233,113,157,230,255,157,234,227,157,231,113,220,157,233,233,157,232,241,220,157,233,111,157,233,255,157,230,105,157,232,255,157,231,108,157,230,255,157,232,238,157,231,113,220,157,231,254,157,230,246,157,230,246,157,230,101,157,234,96,157,231,116,157,233,235,220,157,233,108,157,230,113,157,231,111,157,233,235,157,232,243,220,157,234,230,157,230,230,157,234,116,198,12,220,157,230,227,157,232,243,157,233,113,220,157,230,246,157,234,254,157,232,238,220,157,230,108,157,235,232,220,157,233,101,157,233,113,157,232,243,157,234,233,157,233,255,220,157,232,101,157,230,108,157,230,235,220,157,230,225,157,233,255,157,230,235,220,157,234,96,157,231,98,157,233,104,157,233,236,157,231,121,220,157,233,101,157,230,246,157,234,230,157,232,243,157,231,96,157,231,113,220,157,232,98],"ids_no_specials":[12,220,157,235,235,157,230,255,157,233,113,220,157,231,97,157,231,109,220,157,231,254,157,232,254,157,231,114,220,157,233,106,157,232,241,157,231,111,157,232,243,220,157,231,108,157,230,113,157,235,233,157,233,105,220,157,230,233,157,233,255,220,157,233,101,21,220,157,232,254,157,230,246,157,231,113,220,157,232,101,21,220,157,233,230,157,230,255,220,157,234,121,157,232,239,220,157,233,101,157,232,98,157,230,113,157,230,255,220,157,231,227,157,234,96,157,231,113,220,157,232,254,157,230,113,157,231,108,157,230,233,157,230,230,157,235,230,198,12,220,157,232,254,157,232,243,157,233,113,157,232,238,157,231,113,220,157,233,114,157,232,255,157,231,108,157,230,255,220,157,231,112,157,233,113,157,230,106,157,230,113,220,157,230,230,157,233,241,157,230,230,157,230,251,220,157,234,97,157,232,241,220,157,233,113,157,230,255,157,234,227,157,231,113,220,157,233,233,157,232,241,220,157,233,111,157,233,255,157,230,105,157,232,255,157,231,108,157,230,255,157,232,238,157,231,113,220,157,231,254,157,230,246,157,230,246,157,230,101,157,234,96,157,231,116,157,233,235,220,157,233,108,157,230,113,157,231,111,157,233,235,157,232,243,220,157,234,230,157,230,230,157,234,116,198,12,220,157,230,227,157,232,243,157,233,113,220,157,230,246,157,234,254,157,232,238,220,157,230,108,157,235,232,220,157,233,101,157,233,113,157,232,243,157,234,233,157,233,255,220,157,232,101,157,230,108,157,230,235,220,157,230,225,157,233,255,157,230,235,220,157,234,96,157,231,98,157,233,104,157,233,236,157,231,121,220,157,233,101,157,230,246,157,234,230,157,232,243,157,231,96,157,231,113,220,157,232,98],"tokens":["-","Ġ","á","į","į","á","Ī","Ń","á","ĭ","µ","Ġ","á","ī","¤","á","ī","±","Ġ","á","ī","ł","á","Ĭ","ł","á","ī","¶","Ġ","á","ĭ","®","á","Ĭ","ĵ","á","ī","³","á","Ĭ","ķ","Ġ","á","ī","°","á","Ī","µ","á","į","ĭ","á","ĭ","¬","Ġ","á","Ī","ĭ","á","ĭ","Ń","Ġ","á","ĭ","¨","6","Ġ","á","Ĭ","ł","á","Ī","ĺ","á","ī","µ","Ġ","á","Ĭ","¨","6","Ġ","á","ĭ","Ī","á","Ī","Ń","Ġ","á","Į","½","á","Ĭ","ij","Ġ","á","ĭ","¨","á","Ĭ","¥","á","Ī","µ","á","Ī","Ń","Ġ","á","ī","ħ","á","Į","£","á","ī","µ","Ġ","á","Ĭ","ł","á","Ī","µ","á","ī","°","á","Ī","ĭ","á","Ī","Ī","á","į","Ī","Ċ","-","Ġ","á","Ĭ","ł","á","Ĭ","ķ","á","ĭ","µ","á","Ĭ","IJ","á","ī","µ","Ġ","á","ĭ","¶","á","Ĭ","Ń","á","ī","°","á","Ī","Ń","Ġ","á","ī","´","á","ĭ","µ","á","Ī","®","á","Ī","µ","Ġ","á","Ī","Ī","á","ĭ","ĵ","á","Ī","Ī","á","Ī","Ŀ","Ġ","á","Į","¤","á","Ĭ","ĵ","Ġ","á","ĭ","µ","á","Ī","Ń","á","Į","ħ","á","ī","µ","Ġ","á","ĭ","ĭ","á","Ĭ","ĵ","Ġ","á","ĭ","³","á","ĭ","Ń","á","Ī","¬","á","Ĭ","Ń","á","ī","°","á","Ī","Ń","á","Ĭ","IJ","á","ī","µ","Ġ","á","ī","ł","á","Ī","ĺ","á","Ī","ĺ","á","Ī","¨","á","Į","£","á","ī","¸","á","ĭ","į","Ġ","á","ĭ","°","á","Ī","µ","á","ī","³","á","ĭ","į","á","Ĭ","ķ","Ġ","á","Į","Ī","á","Ī","Ī","á","Į","¸","Ċ","-","Ġ","á","Ī","ħ","á","Ĭ","ķ","á","ĭ","µ","Ġ","á","Ī","ĺ","á","Į","ł","á","Ĭ","IJ","Ġ","á","Ī","°","á","į","Ĭ","Ġ","á","ĭ","¨","á","ĭ","µ","á","Ĭ","ķ","á","Į","ĭ","á","ĭ","Ń","Ġ","á","Ĭ","¨","á","Ī","°","á","Ī","į","Ġ","á","Ī","ĥ","á","ĭ","Ń","á","Ī","į","Ġ","á","Į","£","á","ī","¥","á","ĭ","«","á","ĭ","İ","á","ī","½","Ġ","á","ĭ","¨","á","Ī","ĺ","á","Į","Ī","á","Ĭ","ķ","á","ī","£","á","ī","µ","Ġ","á","Ĭ","¥"],"offsets":[[0,1],[1,2],[2,3],[2,3],[2,3],[3,4],[3,4],[3,4],[4,5],[4,5],[4,5],[5,6],[6,7],[6,7],[6,7],[7,8],[7,8],[7,8],[8,9],[9,10],[9,10],[9,10],[10,11],[10,11],[10,11],[11,12],[11,12],[11,12],[12,13],[13,14],[13,14],[13,14],[14,15],[14,15],[14,15],[15,16],[15,16],[15,16],[16,17],[16,17],[16,17],[17,18],[18,19],[18,19],[18,19],[19,20],[19,20],[19,20],[20,21],[20,21],[20,21],[21,22],[21,22],[21,22],[22,23],[23,24],[23,24],[23,24],[24,25],[24,25],[24,25],[25,26],[26,27],[26,27],[26,27],[27,28],[28,29],[29,30],[29,30],[29,30],[30,31],[30,31],[30,31],[31,32],[31,32],[31,32],[32,33],[33,34],[33,34],[33,34],[34,35],[35,36],[36,37],[36,37],[36,37],[37,38],[37,38],[37,38],[38,39],[39,40],[39,40],[39,40],[40,41],[40,41],[40,41],[41,42],[42,43],[42,43],[42,43],[43,44],[43,44],[43,44],[44,45],[44,45],[44,45],[45,46],[45,46],[45,46],[46,47],[47,48],[47,48],[47,48],[48,49],[48,49],[48,49],[49,50],[49,50],[49,50],[50,51],[51,52],[51,52],[51,52],[52,53],[52,53],[52,53],[53,54],[53,54],[53,54],[54,55],[54,55],[54,55],[55,56],[55,56],[55,56],[56,57],[56,57],[56,57],[57,58],[58,59],[59,60],[60,61],[60,61],[60,61],[61,62],[61,62],[61,62],[62,63],[62,63],[62,63],[63,64],[63,64],[63,64],[64,65],[64,65],[64,65],[65,66],[66,67],[66,67],[66,67],[67,68],[67,68],[67,68],[68,69],[68,69],[68,69],[69,70],[69,70],[69,70],[70,71],[71,72],[71,72],[71,72],[72,73],[72,73],[72,73],[73,74],[73,74],[73,74],[74,75],[74,75],[74,75],[75,76],[76,77],[76,77],[76,77],[77,78],[77,78],[77,78],[78,79],[78,79],[78,79],[79,80],[79,80],[79,80],[80,81],[81,82],[81,82],[81,82],[82,83],[82,83],[82,83],[83,84],[84,85],[84,85],[84,85],[85,86],[85,86],[85,86],[86,87],[86,87],[86,87],[87,88],[87,88],[87,88],[88,89],[89,90],[89,90],[89,90],[90,91],[90,91],[90,91],[91,92],[92,93],[92,93],[92,93],[93,94],[93,94],[93,94],[94,95],[94,95],[94,95],[95,96],[95,96],[95,96],[96,97],[96,97],[96,97],[97,98],[97,98],[97,98],[98,99],[98,99],[98,99],[99,100],[99,100],[99,100],[100,101],[101,102],[101,102],[101,102],[102,103],[102,103],[102,103],[103,104],[103,104],[103,104],[104,105],[104,105],[104,105],[105,106],[105,106],[105,106],[106,107],[106,107],[106,107],[107,108],[107,108],[107,108],[108,109],[109,110],[109,110],[109,110],[110,111],[110,111],[110,111],[111,112],[111,112],[111,112],[112,113],[112,113],[112,113],[113,114],[113,114],[113,114],[114,115],[115,116],[115,116],[115,116],[116,117],[116,117],[116,117],[117,118],[117,118],[117,118],[118,119],[119,120],[120,121],[121,122],[121,122],[121,122],[122,123],[122,123],[122,123],[123,124],[123,124],[123,124],[124,125],[125,126],[125,126],[125,126],[126,127],[126,127],[126,127],[127,128],[127,128],[127,128],[128,129],[129,130],[129,130],[129,130],[130,131],[130,131],[130,131],[131,132],[132,133],[132,133],[132,133],[133,134],[133,134],[133,134],[134,135],[134,135],[134,135],[135,136],[135,136],[135,136],[136,137],[136,137],[136,137],[137,138],[138,139],[138,139],[138,139],[139,140],[139,140],[139,140],[140,141],[140,141],[140,141],[141,142],[142,143],[142,143],[142,143],[143,144],[143,144],[143,144],[144,145],[144,145],[144,145],[145,146],[146,147],[146,147],[146,147],[147,148],[147,148],[147,148],[148,149],[148,149],[148,149],[149,150],[149,150],[149,150],[150,151],[150,151],[150,151],[151,152],[152,153],[152,153],[152,153],[153,154],[153,154],[153,154],[154,155],[154,155],[154,155],[155,156],[155,156],[155,156],[156,157],[156,157],[156,157],[157,158],[157,158],[157,158],[158,159],[159,160],[159,160],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,7,7,7,7,8,9,9,9,9,9,9,9,9,9,9,10,10,10,10,11,12,12,12,12,12,12,12,13,13,13,13,13,13,13,14,14,14,14,14,14,14,14,14,14,14,14,14,15,15,15,15,15,15,15,15,15,15,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,17,18,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,20,20,20,20,20,20,20,20,20,20,20,20,20,21,21,21,21,21,21,21,21,21,21,21,21,21,22,22,22,22,22,22,22,22,22,22,22,22,22,23,23,23,23,23,23,23,24,24,24,24,24,24,24,24,24,24,24,24,24,25,25,25,25,25,25,25,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,29,29,29,29,29,29,29,29,29,29,30,31,32,32,32,32,32,32,32,32,32,32,33,33,33,33,33,33,33,33,33,33,34,34,34,34,34,34,34,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,36,36,36,36,36,36,36,36,36,36,37,37,37,37,37,37,37,37,37,37,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,40,40,40,40],"decoded":"- ፍርድ ቤቱ በአቶ ዮናታን ተስፋዬ ላይ የ6 አመት ከ6 ወር ጽኑ የእስር ቅጣት አስተላለፈ\n- አንድነት ዶክተር ቴድሮስ ለዓለም ጤና ድርጅት ዋና ዳይሬክተርነት በመመረጣቸው ደስታውን ገለጸ\n- ህንድ መጠነ ሰፊ የድንጋይ ከሰል ሃይል ጣብያዎች የመገንባት እ","decoded_with_specials":"- ፍርድ ቤቱ በአቶ ዮናታን ተስፋዬ ላይ የ6 አመት ከ6 ወር ጽኑ የእስር ቅጣት አስተላለፈ\n- አንድነት ዶክተር ቴድሮስ ለዓለም ጤና ድርጅት ዋና ዳይሬክተርነት በመመረጣቸው ደስታውን ገለጸ\n- ህንድ መጠነ ሰፊ የድንጋይ ከሰል ሃይል ጣብያዎች የመገንባት እ"} +{"kind":"sample","source":"fixtures/lang/arb_Arab.txt[:160]","text":"[بالصور] ترقوميا : الموت يطفىء باقة ورد قيد التفتح !!\nالخليل – دوت كوم – من غسان عبد الحميد - لم يخطر في بال أهالي \"ترقوميا\" التي انخلع قلبها وهي تودع التراب جث","ids":[58,657,1115,1321,1968,60,1155,433,782,417,414,443,320,712,1859,417,517,2221,1408,697,1274,148,94,927,320,782,713,888,433,547,220,782,443,547,589,517,697,517,949,1073,0,198,1115,1357,374,443,374,1127,1342,417,517,220,870,417,414,1127,2065,389,118,652,1195,1506,657,547,589,949,414,443,547,533,1358,414,2221,1357,1408,433,2162,927,1115,1718,539,1115,443,392,517,433,782,417,414,443,320,1,589,517,443,496,426,1357,374,715,220,782,374,657,539,320,888,539,443,1155,417,547,715,589,517,433,320,657,2045,148,104],"ids_no_specials":[58,657,1115,1321,1968,60,1155,433,782,417,414,443,320,712,1859,417,517,2221,1408,697,1274,148,94,927,320,782,713,888,433,547,220,782,443,547,589,517,697,517,949,1073,0,198,1115,1357,374,443,374,1127,1342,417,517,220,870,417,414,1127,2065,389,118,652,1195,1506,657,547,589,949,414,443,547,533,1358,414,2221,1357,1408,433,2162,927,1115,1718,539,1115,443,392,517,433,782,417,414,443,320,1,589,517,443,496,426,1357,374,715,220,782,374,657,539,320,888,539,443,1155,417,547,715,589,517,433,320,657,2045,148,104],"tokens":["[","ب","اÙĦ","ص","ÙĪر","]","Ġت","ر","ÙĤ","ÙĪ","Ùħ","ÙĬ","ا","Ġ:","ĠاÙĦÙħ","ÙĪ","ت","ĠÙĬ","Ø·","Ùģ","Ùī","Ø","¡","Ġب","ا","ÙĤ","Ø©","ĠÙĪ","ر","د","Ġ","ÙĤ","ÙĬ","د","ĠاÙĦ","ت","Ùģ","ت","ØŃ","Ġ!","!","Ċ","اÙĦ","Ø®","ÙĦ","ÙĬ","ÙĦ","ĠâĢĵ","Ġد","ÙĪ","ت","Ġ","Ùĥ","ÙĪ","Ùħ","ĠâĢĵ","ĠÙħÙĨ","ĠØ","º","س","اÙĨ","Ġع","ب","د","ĠاÙĦ","ØŃ","Ùħ","ÙĬ","د","Ġ-","ĠÙĦ","Ùħ","ĠÙĬ","Ø®","Ø·","ر","ĠÙģÙĬ","Ġب","اÙĦ","ĠØ£","Ùĩ","اÙĦ","ÙĬ","Ġ\"","ت","ر","ÙĤ","ÙĪ","Ùħ","ÙĬ","ا","\"","ĠاÙĦ","ت","ÙĬ","Ġا","ÙĨ","Ø®","ÙĦ","ع","Ġ","ÙĤ","ÙĦ","ب","Ùĩ","ا","ĠÙĪ","Ùĩ","ÙĬ","Ġت","ÙĪ","د","ع","ĠاÙĦ","ت","ر","ا","ب","Ġج","Ø","«"],"offsets":[[0,1],[1,2],[2,4],[4,5],[5,7],[7,8],[8,10],[10,11],[11,12],[12,13],[13,14],[14,15],[15,16],[16,18],[18,22],[22,23],[23,24],[24,26],[26,27],[27,28],[28,29],[29,30],[29,30],[30,32],[32,33],[33,34],[34,35],[35,37],[37,38],[38,39],[39,40],[40,41],[41,42],[42,43],[43,46],[46,47],[47,48],[48,49],[49,50],[50,52],[52,53],[53,54],[54,56],[56,57],[57,58],[58,59],[59,60],[60,62],[62,64],[64,65],[65,66],[66,67],[67,68],[68,69],[69,70],[70,72],[72,75],[75,77],[76,77],[77,78],[78,80],[80,82],[82,83],[83,84],[84,87],[87,88],[88,89],[89,90],[90,91],[91,93],[93,95],[95,96],[96,98],[98,99],[99,100],[100,101],[101,104],[104,106],[106,108],[108,110],[110,111],[111,113],[113,114],[114,116],[116,117],[117,118],[118,119],[119,120],[120,121],[121,122],[122,123],[123,124],[124,127],[127,128],[128,129],[129,131],[131,132],[132,133],[133,134],[134,135],[135,136],[136,137],[137,138],[138,139],[139,140],[140,141],[141,143],[143,144],[144,145],[145,147],[147,148],[148,149],[149,150],[150,153],[153,154],[154,155],[155,156],[156,157],[157,159],[159,160],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,1,2,2,2,2,2,2,2,3,4,4,4,5,5,5,5,5,5,6,6,6,6,7,7,7,8,8,8,8,9,9,9,9,9,10,10,10,11,11,11,11,11,12,13,13,13,14,14,14,14,15,16,17,17,17,17,18,18,18,19,19,19,19,19,20,21,21,22,22,22,22,23,24,24,25,25,25,25,26,27,27,27,27,27,27,27,28,29,29,29,30,30,30,30,30,31,31,31,31,31,31,32,32,32,33,33,33,33,34,34,34,34,34,35,35,35],"decoded":"[بالصور] ترقوميا : الموت يطفىء باقة ورد قيد التفتح !!\nالخليل – دوت كوم – من غسان عبد الحميد - لم يخطر في بال أهالي \"ترقوميا\" التي انخلع قلبها وهي تودع التراب جث","decoded_with_specials":"[بالصور] ترقوميا : الموت يطفىء باقة ورد قيد التفتح !!\nالخليل – دوت كوم – من غسان عبد الحميد - لم يخطر في بال أهالي \"ترقوميا\" التي انخلع قلبها وهي تودع التراب جث"} +{"kind":"sample","source":"fixtures/lang/ben_Beng.txt[:160]","text":"গ্রহ নীহারিকা\nগ্রহ নীহারিকা (ইংরেজি ভাষায়: Planetary nebula) এক বিশেষ ধরনের গ্যাসীয় নীহারিকা। যেসব তারার ভর কম, নির্দিষ্টভাবে বলতে গেলে যেসব তারার ভর সূর্যের ","ids":[360,245,1129,108,360,117,571,101,509,222,360,117,967,108,1466,243,797,198,360,245,1129,108,360,117,571,101,509,222,360,117,967,108,1466,243,797,350,360,229,360,224,2041,1457,250,1090,571,255,967,115,967,107,360,120,25,2064,270,292,815,453,65,361,64,8,571,237,360,243,571,105,1466,114,1457,115,571,100,2041,2150,1457,108,571,245,1129,107,967,116,509,222,360,107,360,120,571,101,509,222,360,117,967,108,1466,243,797,1670,571,107,1457,116,360,105,571,97,967,108,967,108,571,255,2041,571,243,360,106,11,571,101,1466,108,1129,99,1466,115,1129,253,360,255,967,105,917,571,105,360,110,360,97,917,571,245,1457,110,917,571,107,1457,116,360,105,571,97,967,108,967,108,571,255,2041,571,116,509,224,2041,1129,107,1457,108,220],"ids_no_specials":[360,245,1129,108,360,117,571,101,509,222,360,117,967,108,1466,243,797,198,360,245,1129,108,360,117,571,101,509,222,360,117,967,108,1466,243,797,350,360,229,360,224,2041,1457,250,1090,571,255,967,115,967,107,360,120,25,2064,270,292,815,453,65,361,64,8,571,237,360,243,571,105,1466,114,1457,115,571,100,2041,2150,1457,108,571,245,1129,107,967,116,509,222,360,107,360,120,571,101,509,222,360,117,967,108,1466,243,797,1670,571,107,1457,116,360,105,571,97,967,108,967,108,571,255,2041,571,243,360,106,11,571,101,1466,108,1129,99,1466,115,1129,253,360,255,967,105,917,571,105,360,110,360,97,917,571,245,1457,110,917,571,107,1457,116,360,105,571,97,967,108,967,108,571,255,2041,571,116,509,224,2041,1129,107,1457,108,220],"tokens":["à¦","Ĺ","à§įà¦","°","à¦","¹","Ġà¦","¨","à§","Ģ","à¦","¹","াà¦","°","িà¦","ķ","া","Ċ","à¦","Ĺ","à§įà¦","°","à¦","¹","Ġà¦","¨","à§","Ģ","à¦","¹","াà¦","°","িà¦","ķ","া","Ġ(","à¦","ĩ","à¦","Ĥ","র","à§ĩà¦","ľ","ি","Ġà¦","Ń","াà¦","·","াà¦","¯","à¦","¼",":","ĠPl","an","et","ary","Ġne","b","ul","a",")","Ġà¦","ı","à¦","ķ","Ġà¦","¬","িà¦","¶","à§ĩà¦","·","Ġà¦","§","র","ন","à§ĩà¦","°","Ġà¦","Ĺ","à§įà¦","¯","াà¦","¸","à§","Ģ","à¦","¯","à¦","¼","Ġà¦","¨","à§","Ģ","à¦","¹","াà¦","°","িà¦","ķ","া","।","Ġà¦","¯","à§ĩà¦","¸","à¦","¬","Ġà¦","¤","াà¦","°","াà¦","°","Ġà¦","Ń","র","Ġà¦","ķ","à¦","®",",","Ġà¦","¨","িà¦","°","à§įà¦","¦","িà¦","·","à§įà¦","Ł","à¦","Ń","াà¦","¬","à§ĩ","Ġà¦","¬","à¦","²","à¦","¤","à§ĩ","Ġà¦","Ĺ","à§ĩà¦","²","à§ĩ","Ġà¦","¯","à§ĩà¦","¸","à¦","¬","Ġà¦","¤","াà¦","°","াà¦","°","Ġà¦","Ń","র","Ġà¦","¸","à§","Ĥ","র","à§įà¦","¯","à§ĩà¦","°","Ġ"],"offsets":[[0,1],[0,1],[1,3],[2,3],[3,4],[3,4],[4,6],[5,6],[6,7],[6,7],[7,8],[7,8],[8,10],[9,10],[10,12],[11,12],[12,13],[13,14],[14,15],[14,15],[15,17],[16,17],[17,18],[17,18],[18,20],[19,20],[20,21],[20,21],[21,22],[21,22],[22,24],[23,24],[24,26],[25,26],[26,27],[27,29],[29,30],[29,30],[30,31],[30,31],[31,32],[32,34],[33,34],[34,35],[35,37],[36,37],[37,39],[38,39],[39,41],[40,41],[41,42],[41,42],[42,43],[43,46],[46,48],[48,50],[50,53],[53,56],[56,57],[57,59],[59,60],[60,61],[61,63],[62,63],[63,64],[63,64],[64,66],[65,66],[66,68],[67,68],[68,70],[69,70],[70,72],[71,72],[72,73],[73,74],[74,76],[75,76],[76,78],[77,78],[78,80],[79,80],[80,82],[81,82],[82,83],[82,83],[83,84],[83,84],[84,85],[84,85],[85,87],[86,87],[87,88],[87,88],[88,89],[88,89],[89,91],[90,91],[91,93],[92,93],[93,94],[94,95],[95,97],[96,97],[97,99],[98,99],[99,100],[99,100],[100,102],[101,102],[102,104],[103,104],[104,106],[105,106],[106,108],[107,108],[108,109],[109,111],[110,111],[111,112],[111,112],[112,113],[113,115],[114,115],[115,117],[116,117],[117,119],[118,119],[119,121],[120,121],[121,123],[122,123],[123,124],[123,124],[124,126],[125,126],[126,127],[127,129],[128,129],[129,130],[129,130],[130,131],[130,131],[131,132],[132,134],[133,134],[134,136],[135,136],[136,137],[137,139],[138,139],[139,141],[140,141],[141,142],[141,142],[142,144],[143,144],[144,146],[145,146],[146,148],[147,148],[148,150],[149,150],[150,151],[151,153],[152,153],[153,154],[153,154],[154,155],[155,157],[156,157],[157,159],[158,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,2,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,5,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,7,8,9,9,9,9,10,10,10,10,11,12,12,12,12,13,13,13,13,13,13,14,14,14,14,14,14,15,15,15,15,15,15,15,15,15,15,15,15,16,16,16,16,16,16,16,16,16,16,16,17,18,18,18,18,18,18,19,19,19,19,19,19,20,20,20,21,21,21,21,22,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,24,24,24,24,24,24,24,25,25,25,25,25,26,26,26,26,26,26,27,27,27,27,27,27,28,28,28,29,29,29,29,29,29,29,29,29,30],"decoded":"গ্রহ নীহারিকা\nগ্রহ নীহারিকা (ইংরেজি ভাষায়: Planetary nebula) এক বিশেষ ধরনের গ্যাসীয় নীহারিকা। যেসব তারার ভর কম, নির্দিষ্টভাবে বলতে গেলে যেসব তারার ভর সূর্যের ","decoded_with_specials":"গ্রহ নীহারিকা\nগ্রহ নীহারিকা (ইংরেজি ভাষায়: Planetary nebula) এক বিশেষ ধরনের গ্যাসীয় নীহারিকা। যেসব তারার ভর কম, নির্দিষ্টভাবে বলতে গেলে যেসব তারার ভর সূর্যের "} +{"kind":"sample","source":"fixtures/lang/cmn_Hani.txt[:160]","text":"強力建議大家未來盡量避免英航阿~~~\n班機原訂航程\n6/9 台北→香港\n6/9 香港→倫敦 (原訂23:15起飛，4:50am抵達)\n6/10 倫敦→斯德哥爾摩 (原訂7:40am起飛，11:05am抵達)\n就在第二段，英航no.25班機在香港機場兩度離開閘口又兩度返航\n第一次因有旅客身體不適 (好，不怪他，消耗時間也","ids":[2159,115,161,232,249,161,119,118,164,255,108,1640,1376,114,985,103,160,122,228,163,249,94,2213,237,165,223,123,1106,235,164,233,109,164,230,103,165,246,123,93,93,93,198,163,237,255,162,102,253,161,236,253,164,101,224,164,230,103,163,101,233,198,21,14,24,220,948,108,161,234,245,158,228,240,165,99,247,162,116,107,198,21,14,24,220,165,99,247,162,116,107,158,228,240,161,222,104,162,243,99,350,161,236,253,164,101,224,1860,25,1055,1897,115,165,96,249,979,19,25,1434,313,162,232,113,165,223,242,446,21,14,702,1222,222,104,162,243,99,158,228,240,1615,107,161,122,115,161,241,98,163,230,122,162,239,102,350,161,236,253,164,101,224,22,25,1723,313,1897,115,165,96,249,979,994,25,15,20,313,162,232,113,165,223,242,446,161,108,109,2178,163,105,105,774,234,162,106,113,979,164,233,109,164,230,103,1750,13,1161,163,237,255,162,102,253,2178,165,99,247,162,116,107,162,102,253,161,254,112,1106,102,161,118,99,165,249,95,165,244,233,165,244,246,948,96,948,230,1106,102,161,118,99,2206,242,164,230,103,198,163,105,105,624,222,162,105,94,1749,254,985,231,1024,227,1376,95,164,118,104,165,104,242,624,235,165,223,102,350,2077,121,979,624,235,162,222,103,1467,244,979,162,114,230,164,222,245,162,247,224,165,244,241,1140,253],"ids_no_specials":[2159,115,161,232,249,161,119,118,164,255,108,1640,1376,114,985,103,160,122,228,163,249,94,2213,237,165,223,123,1106,235,164,233,109,164,230,103,165,246,123,93,93,93,198,163,237,255,162,102,253,161,236,253,164,101,224,164,230,103,163,101,233,198,21,14,24,220,948,108,161,234,245,158,228,240,165,99,247,162,116,107,198,21,14,24,220,165,99,247,162,116,107,158,228,240,161,222,104,162,243,99,350,161,236,253,164,101,224,1860,25,1055,1897,115,165,96,249,979,19,25,1434,313,162,232,113,165,223,242,446,21,14,702,1222,222,104,162,243,99,158,228,240,1615,107,161,122,115,161,241,98,163,230,122,162,239,102,350,161,236,253,164,101,224,22,25,1723,313,1897,115,165,96,249,979,994,25,15,20,313,162,232,113,165,223,242,446,161,108,109,2178,163,105,105,774,234,162,106,113,979,164,233,109,164,230,103,1750,13,1161,163,237,255,162,102,253,2178,165,99,247,162,116,107,162,102,253,161,254,112,1106,102,161,118,99,165,249,95,165,244,233,165,244,246,948,96,948,230,1106,102,161,118,99,2206,242,164,230,103,198,163,105,105,624,222,162,105,94,1749,254,985,231,1024,227,1376,95,164,118,104,165,104,242,624,235,165,223,102,350,2077,121,979,624,235,162,222,103,1467,244,979,162,114,230,164,222,245,162,247,224,165,244,241,1140,253],"tokens":["å¼","·","å","Ĭ","Ľ","å","»","º","è","Ń","°","大","å®","¶","æľ","ª","ä","¾","Ĩ","ç","Ľ","¡","éĩ","ı","é","ģ","¿","åħ","į","è","ĭ","±","è","Ī","ª","é","ĺ","¿","~","~","~","Ċ","ç","ı","Ń","æ","©","Ł","å","İ","Ł","è","¨","Ĥ","è","Ī","ª","ç","¨","ĭ","Ċ","6","/","9","Ġ","åı","°","å","Į","Ĺ","â","Ĩ","Ĵ","é","¦","Ļ","æ","¸","¯","Ċ","6","/","9","Ġ","é","¦","Ļ","æ","¸","¯","â","Ĩ","Ĵ","å","Ģ","«","æ","ķ","¦","Ġ(","å","İ","Ł","è","¨","Ĥ","23",":","15","èµ","·","é","£","Ľ","ï¼Į","4",":","50","am","æ","Ĭ","µ","é","ģ","Ķ",")Ċ","6","/","10","Ġå","Ģ","«","æ","ķ","¦","â","Ĩ","Ĵ","æĸ","¯","å","¾","·","å","ĵ","¥","ç","Ī","¾","æ","ij","©","Ġ(","å","İ","Ł","è","¨","Ĥ","7",":","40","am","èµ","·","é","£","Ľ","ï¼Į","11",":","0","5","am","æ","Ĭ","µ","é","ģ","Ķ",")Ċ","å","°","±","åľ¨","ç","¬","¬","äº","Į","æ","®","µ","ï¼Į","è","ĭ","±","è","Ī","ª","no",".","25","ç","ı","Ń","æ","©","Ł","åľ¨","é","¦","Ļ","æ","¸","¯","æ","©","Ł","å","ł","´","åħ","©","å","º","¦","é","Ľ","¢","é","ĸ","ĭ","é","ĸ","ĺ","åı","£","åı","Ī","åħ","©","å","º","¦","è¿","Ķ","è","Ī","ª","Ċ","ç","¬","¬","ä¸","Ģ","æ","¬","¡","åĽ","ł","æľ","ī","æĹ","ħ","å®","¢","è","º","«","é","«","Ķ","ä¸","į","é","ģ","©","Ġ(","å¥","½","ï¼Į","ä¸","į","æ","Ģ","ª","ä»","ĸ","ï¼Į","æ","¶","Ī","è","Ģ","Ĺ","æ","Ļ","Ĥ","é","ĸ","ĵ","ä¹","Ł"],"offsets":[[0,1],[0,1],[1,2],[1,2],[1,2],[2,3],[2,3],[2,3],[3,4],[3,4],[3,4],[4,5],[5,6],[5,6],[6,7],[6,7],[7,8],[7,8],[7,8],[8,9],[8,9],[8,9],[9,10],[9,10],[10,11],[10,11],[10,11],[11,12],[11,12],[12,13],[12,13],[12,13],[13,14],[13,14],[13,14],[14,15],[14,15],[14,15],[15,16],[16,17],[17,18],[18,19],[19,20],[19,20],[19,20],[20,21],[20,21],[20,21],[21,22],[21,22],[21,22],[22,23],[22,23],[22,23],[23,24],[23,24],[23,24],[24,25],[24,25],[24,25],[25,26],[26,27],[27,28],[28,29],[29,30],[30,31],[30,31],[31,32],[31,32],[31,32],[32,33],[32,33],[32,33],[33,34],[33,34],[33,34],[34,35],[34,35],[34,35],[35,36],[36,37],[37,38],[38,39],[39,40],[40,41],[40,41],[40,41],[41,42],[41,42],[41,42],[42,43],[42,43],[42,43],[43,44],[43,44],[43,44],[44,45],[44,45],[44,45],[45,47],[47,48],[47,48],[47,48],[48,49],[48,49],[48,49],[49,51],[51,52],[52,54],[54,55],[54,55],[55,56],[55,56],[55,56],[56,57],[57,58],[58,59],[59,61],[61,63],[63,64],[63,64],[63,64],[64,65],[64,65],[64,65],[65,67],[67,68],[68,69],[69,71],[71,73],[72,73],[72,73],[73,74],[73,74],[73,74],[74,75],[74,75],[74,75],[75,76],[75,76],[76,77],[76,77],[76,77],[77,78],[77,78],[77,78],[78,79],[78,79],[78,79],[79,80],[79,80],[79,80],[80,82],[82,83],[82,83],[82,83],[83,84],[83,84],[83,84],[84,85],[85,86],[86,88],[88,90],[90,91],[90,91],[91,92],[91,92],[91,92],[92,93],[93,95],[95,96],[96,97],[97,98],[98,100],[100,101],[100,101],[100,101],[101,102],[101,102],[101,102],[102,104],[104,105],[104,105],[104,105],[105,106],[106,107],[106,107],[106,107],[107,108],[107,108],[108,109],[108,109],[108,109],[109,110],[110,111],[110,111],[110,111],[111,112],[111,112],[111,112],[112,114],[114,115],[115,117],[117,118],[117,118],[117,118],[118,119],[118,119],[118,119],[119,120],[120,121],[120,121],[120,121],[121,122],[121,122],[121,122],[122,123],[122,123],[122,123],[123,124],[123,124],[123,124],[124,125],[124,125],[125,126],[125,126],[125,126],[126,127],[126,127],[126,127],[127,128],[127,128],[127,128],[128,129],[128,129],[128,129],[129,130],[129,130],[130,131],[130,131],[131,132],[131,132],[132,133],[132,133],[132,133],[133,134],[133,134],[134,135],[134,135],[134,135],[135,136],[136,137],[136,137],[136,137],[137,138],[137,138],[138,139],[138,139],[138,139],[139,140],[139,140],[140,141],[140,141],[141,142],[141,142],[142,143],[142,143],[143,144],[143,144],[143,144],[144,145],[144,145],[144,145],[145,146],[145,146],[146,147],[146,147],[146,147],[147,149],[149,150],[149,150],[150,151],[151,152],[151,152],[152,153],[152,153],[152,153],[153,154],[153,154],[154,155],[155,156],[155,156],[155,156],[156,157],[156,157],[156,157],[157,158],[157,158],[157,158],[158,159],[158,159],[158,159],[159,160],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,3,4,5,6,7,7,7,7,7,7,8,8,8,8,8,8,8,8,8,9,10,11,12,13,13,13,13,13,13,13,14,14,14,14,14,14,14,14,14,15,16,16,16,16,16,16,17,18,19,20,20,20,20,20,21,22,23,24,25,25,25,25,25,25,25,26,27,28,29,30,30,30,30,30,30,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,32,33,33,33,33,33,33,34,35,36,37,37,37,37,37,37,38,39,40,41,41,42,42,42,42,42,42,42,43,44,44,44,44,44,44,44,44,44,44,44,44,45,45,45,45,45,45,45,45,46,47,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,49,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,51,52,52,53,53,53,53,53,53,53,53,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54],"decoded":"強力建議大家未來盡量避免英航阿~~~\n班機原訂航程\n6/9 台北→香港\n6/9 香港→倫敦 (原訂23:15起飛，4:50am抵達)\n6/10 倫敦→斯德哥爾摩 (原訂7:40am起飛，11:05am抵達)\n就在第二段，英航no.25班機在香港機場兩度離開閘口又兩度返航\n第一次因有旅客身體不適 (好，不怪他，消耗時間也","decoded_with_specials":"強力建議大家未來盡量避免英航阿~~~\n班機原訂航程\n6/9 台北→香港\n6/9 香港→倫敦 (原訂23:15起飛，4:50am抵達)\n6/10 倫敦→斯德哥爾摩 (原訂7:40am起飛，11:05am抵達)\n就在第二段，英航no.25班機在香港機場兩度離開閘口又兩度返航\n第一次因有旅客身體不適 (好，不怪他，消耗時間也"} +{"kind":"sample","source":"fixtures/lang/ell_Grek.txt[:160]","text":"Θυμήσου με\nΗ πόλη προσφέρει πολλές ευκαιρίες - πήγαινε στο εμπορικό κέντρο, στο σαλόνι ομορφιάς , σε κλάμπ και διασκέδασε με την ψυχή σου.\nΔημιούργησε μαγευτικά","ids":[138,246,1482,1722,138,106,1168,822,1482,1091,120,891,198,138,245,220,1345,2097,1727,1425,220,1345,1184,822,1168,139,228,138,255,1184,891,874,220,1345,822,1727,1727,138,255,1433,220,891,1482,1664,727,874,1184,1865,891,1433,533,220,1345,138,106,138,111,727,874,954,891,220,1168,824,822,220,891,1722,1345,822,1184,874,1664,2097,1091,118,138,255,954,824,1184,822,11,220,1168,824,822,220,1168,727,1727,2097,954,874,220,822,1722,822,1184,139,228,874,2132,1433,1366,220,1168,891,1091,118,1727,2132,1722,1345,1091,118,727,874,1091,112,874,727,1168,1664,138,255,138,112,727,1168,891,1091,120,891,220,824,1425,954,220,139,230,1482,139,229,138,106,220,1168,822,1482,558,138,242,1425,1722,874,822,139,235,1184,138,111,1425,1168,891,1091,120,727,138,111,891,1482,824,874,1664,2132],"ids_no_specials":[138,246,1482,1722,138,106,1168,822,1482,1091,120,891,198,138,245,220,1345,2097,1727,1425,220,1345,1184,822,1168,139,228,138,255,1184,891,874,220,1345,822,1727,1727,138,255,1433,220,891,1482,1664,727,874,1184,1865,891,1433,533,220,1345,138,106,138,111,727,874,954,891,220,1168,824,822,220,891,1722,1345,822,1184,874,1664,2097,1091,118,138,255,954,824,1184,822,11,220,1168,824,822,220,1168,727,1727,2097,954,874,220,822,1722,822,1184,139,228,874,2132,1433,1366,220,1168,891,1091,118,1727,2132,1722,1345,1091,118,727,874,1091,112,874,727,1168,1664,138,255,138,112,727,1168,891,1091,120,891,220,824,1425,954,220,139,230,1482,139,229,138,106,220,1168,822,1482,558,138,242,1425,1722,874,822,139,235,1184,138,111,1425,1168,891,1091,120,727,138,111,891,1482,824,874,1664,2132],"tokens":["Î","ĺ","Ïħ","μ","Î","®","Ïĥ","ο","Ïħ","ĠÎ","¼","ε","Ċ","Î","Ĺ","Ġ","ÏĢ","ÏĮ","λ","η","Ġ","ÏĢ","Ïģ","ο","Ïĥ","Ï","Ĩ","Î","Ń","Ïģ","ε","ι","Ġ","ÏĢ","ο","λ","λ","Î","Ń","ÏĤ","Ġ","ε","Ïħ","κ","α","ι","Ïģ","ί","ε","ÏĤ","Ġ-","Ġ","ÏĢ","Î","®","Î","³","α","ι","ν","ε","Ġ","Ïĥ","ÏĦ","ο","Ġ","ε","μ","ÏĢ","ο","Ïģ","ι","κ","ÏĮ","ĠÎ","º","Î","Ń","ν","ÏĦ","Ïģ","ο",",","Ġ","Ïĥ","ÏĦ","ο","Ġ","Ïĥ","α","λ","ÏĮ","ν","ι","Ġ","ο","μ","ο","Ïģ","Ï","Ĩ","ι","ά","ÏĤ","Ġ,","Ġ","Ïĥ","ε","ĠÎ","º","λ","ά","μ","ÏĢ","ĠÎ","º","α","ι","ĠÎ","´","ι","α","Ïĥ","κ","Î","Ń","Î","´","α","Ïĥ","ε","ĠÎ","¼","ε","Ġ","ÏĦ","η","ν","Ġ","Ï","Ī","Ïħ","Ï","ĩ","Î","®","Ġ","Ïĥ","ο","Ïħ",".Ċ","Î","Ķ","η","μ","ι","ο","Ï","į","Ïģ","Î","³","η","Ïĥ","ε","ĠÎ","¼","α","Î","³","ε","Ïħ","ÏĦ","ι","κ","ά"],"offsets":[[0,1],[0,1],[1,2],[2,3],[3,4],[3,4],[4,5],[5,6],[6,7],[7,9],[8,9],[9,10],[10,11],[11,12],[11,12],[12,13],[13,14],[14,15],[15,16],[16,17],[17,18],[18,19],[19,20],[20,21],[21,22],[22,23],[22,23],[23,24],[23,24],[24,25],[25,26],[26,27],[27,28],[28,29],[29,30],[30,31],[31,32],[32,33],[32,33],[33,34],[34,35],[35,36],[36,37],[37,38],[38,39],[39,40],[40,41],[41,42],[42,43],[43,44],[44,46],[46,47],[47,48],[48,49],[48,49],[49,50],[49,50],[50,51],[51,52],[52,53],[53,54],[54,55],[55,56],[56,57],[57,58],[58,59],[59,60],[60,61],[61,62],[62,63],[63,64],[64,65],[65,66],[66,67],[67,69],[68,69],[69,70],[69,70],[70,71],[71,72],[72,73],[73,74],[74,75],[75,76],[76,77],[77,78],[78,79],[79,80],[80,81],[81,82],[82,83],[83,84],[84,85],[85,86],[86,87],[87,88],[88,89],[89,90],[90,91],[91,92],[91,92],[92,93],[93,94],[94,95],[95,97],[97,98],[98,99],[99,100],[100,102],[101,102],[102,103],[103,104],[104,105],[105,106],[106,108],[107,108],[108,109],[109,110],[110,112],[111,112],[112,113],[113,114],[114,115],[115,116],[116,117],[116,117],[117,118],[117,118],[118,119],[119,120],[120,121],[121,123],[122,123],[123,124],[124,125],[125,126],[126,127],[127,128],[128,129],[129,130],[129,130],[130,131],[131,132],[131,132],[132,133],[132,133],[133,134],[134,135],[135,136],[136,137],[137,139],[139,140],[139,140],[140,141],[141,142],[142,143],[143,144],[144,145],[144,145],[145,146],[146,147],[146,147],[147,148],[148,149],[149,150],[150,152],[151,152],[152,153],[153,154],[153,154],[154,155],[155,156],[156,157],[157,158],[158,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,0,0,1,1,1,2,3,3,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,7,7,7,8,9,9,9,9,9,9,9,9,9,9,10,10,10,10,11,11,11,11,11,11,11,11,11,12,12,12,12,12,12,12,12,13,14,14,14,14,15,15,15,15,15,15,15,16,16,16,16,16,16,16,16,16,16,17,18,18,18,19,19,19,19,19,19,20,20,20,20,21,21,21,21,21,21,21,21,21,21,21,21,21,22,22,22,23,23,23,23,24,24,24,24,24,24,24,24,25,25,25,25,26,27,27,27,27,27,27,27,27,27,27,27,27,27,27,28,28,28,28,28,28,28,28,28,28,28],"decoded":"Θυμήσου με\nΗ πόλη προσφέρει πολλές ευκαιρίες - πήγαινε στο εμπορικό κέντρο, στο σαλόνι ομορφιάς , σε κλάμπ και διασκέδασε με την ψυχή σου.\nΔημιούργησε μαγευτικά","decoded_with_specials":"Θυμήσου με\nΗ πόλη προσφέρει πολλές ευκαιρίες - πήγαινε στο εμπορικό κέντρο, στο σαλόνι ομορφιάς , σε κλάμπ και διασκέδασε με την ψυχή σου.\nΔημιούργησε μαγευτικά"} +{"kind":"sample","source":"fixtures/lang/eng_Latn.txt[:160]","text":"|Viewing Single Post From: Spoilers for the Week of February 11th|\n|Lil||Feb 1 2013, 09:58 AM|\nDon't care about Chloe/Taniel/Jen-Jen. Don't care about Sami, rea","ids":[91,1097,289,336,289,282,398,564,454,511,25,336,79,78,311,409,395,290,1416,886,328,454,1113,1942,815,220,994,404,91,198,91,43,311,91,91,37,1113,220,16,220,667,18,11,220,15,24,25,20,23,355,44,91,198,35,263,1507,274,644,1078,1036,746,68,14,51,270,930,14,41,262,12,41,262,13,415,263,1507,274,644,1078,336,313,72,11,109943],"ids_no_specials":[91,1097,289,336,289,282,398,564,454,511,25,336,79,78,311,409,395,290,1416,886,328,454,1113,1942,815,220,994,404,91,198,91,43,311,91,91,37,1113,220,16,220,667,18,11,220,15,24,25,20,23,355,44,91,198,35,263,1507,274,644,1078,1036,746,68,14,51,270,930,14,41,262,12,41,262,13,415,263,1507,274,644,1078,336,313,72,11,109943],"tokens":["|","View","ing","ĠS","ing","le","ĠP","ost","ĠF","rom",":","ĠS","p","o","il","ers","Ġfor","Ġthe","ĠWe","ek","Ġof","ĠF","eb","ru","ary","Ġ","11","th","|","Ċ","|","L","il","|","|","F","eb","Ġ","1","Ġ","201","3",",","Ġ","0","9",":","5","8","ĠA","M","|","Ċ","D","on","'t","Ġc","are","Ġabout","ĠCh","lo","e","/","T","an","iel","/","J","en","-","J","en",".","ĠD","on","'t","Ġc","are","Ġabout","ĠS","am","i",",","Ġrea"],"offsets":[[0,1],[1,5],[5,8],[8,10],[10,13],[13,15],[15,17],[17,20],[20,22],[22,25],[25,26],[26,28],[28,29],[29,30],[30,32],[32,35],[35,39],[39,43],[43,46],[46,48],[48,51],[51,53],[53,55],[55,57],[57,60],[60,61],[61,63],[63,65],[65,66],[66,67],[67,68],[68,69],[69,71],[71,72],[72,73],[73,74],[74,76],[76,77],[77,78],[78,79],[79,82],[82,83],[83,84],[84,85],[85,86],[86,87],[87,88],[88,89],[89,90],[90,92],[92,93],[93,94],[94,95],[95,96],[96,98],[98,100],[100,102],[102,105],[105,111],[111,114],[114,116],[116,117],[117,118],[118,119],[119,121],[121,124],[124,125],[125,126],[126,128],[128,129],[129,130],[130,132],[132,133],[133,135],[135,137],[137,139],[139,141],[141,144],[144,150],[150,152],[152,154],[154,155],[155,156],[156,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,1,1,1,2,2,3,3,4,5,5,5,5,5,6,7,8,8,9,10,10,10,10,11,12,13,14,14,15,15,15,16,16,17,17,18,19,20,21,22,23,24,25,25,26,27,27,28,28,29,29,30,30,30,31,31,32,33,33,33,34,34,34,34,35,35,35,36,36,36,37,38,38,38,39,39,40,41,41,41,42,43],"decoded":"|Viewing Single Post From: Spoilers for the Week of February 11th|\n|Lil||Feb 1 2013, 09:58 AM|\nDon't care about Chloe/Taniel/Jen-Jen. Don't care about Sami, rea","decoded_with_specials":"|Viewing Single Post From: Spoilers for the Week of February 11th|\n|Lil||Feb 1 2013, 09:58 AM|\nDon't care about Chloe/Taniel/Jen-Jen. Don't care about Sami, rea"} +{"kind":"sample","source":"fixtures/lang/heb_Hebr.txt[:160]","text":"סיכונים – אלמנט מרכזי, חיוני לפעילות מסחרית. להבין מה הסיכון הוא מאוד חשוב. החוויה האנושית עולה כי מי יודע איך לקחת סיכונים בזמן, היא לנצח גדולים. לזכור פוליטיק","ids":[147,94,760,249,790,254,760,251,1127,1525,1192,1570,1766,2014,1997,1015,147,249,147,244,598,11,463,245,598,790,254,598,2069,147,97,1250,760,250,790,103,1997,147,94,147,245,1015,760,103,13,2069,1198,1726,760,253,1997,1198,1469,147,94,760,249,790,253,1469,790,238,1997,1158,790,241,463,245,1731,790,239,13,1469,147,245,653,653,760,242,1469,1158,1766,790,102,760,103,463,95,790,250,1198,463,249,598,1997,598,463,247,790,241,1250,1525,760,248,2069,147,100,147,245,1567,463,94,760,249,790,254,760,251,2095,147,244,1570,147,253,11,1469,760,238,2069,1766,147,99,147,245,463,240,2089,790,250,760,251,13,2069,147,244,147,249,790,101,463,97,790,250,760,246,760,100],"ids_no_specials":[147,94,760,249,790,254,760,251,1127,1525,1192,1570,1766,2014,1997,1015,147,249,147,244,598,11,463,245,598,790,254,598,2069,147,97,1250,760,250,790,103,1997,147,94,147,245,1015,760,103,13,2069,1198,1726,760,253,1997,1198,1469,147,94,760,249,790,253,1469,790,238,1997,1158,790,241,463,245,1731,790,239,13,1469,147,245,653,653,760,242,1469,1158,1766,790,102,760,103,463,95,790,250,1198,463,249,598,1997,598,463,247,790,241,1250,1525,760,248,2069,147,100,147,245,1567,463,94,760,249,790,254,760,251,2095,147,244,1570,147,253,11,1469,760,238,2069,1766,147,99,147,245,463,240,2089,790,250,760,251,13,2069,147,244,147,249,790,101,463,97,790,250,760,246,760,100],"tokens":["×","¡","×Ļ×","Ľ","×ķ×","ł","×Ļ×","Ŀ","ĠâĢĵ","Ġ×IJ","׾","×ŀ","׳","×ĺ","Ġ×ŀ","ר","×","Ľ","×","ĸ","×Ļ",",","Ġ×","Ĺ","×Ļ","×ķ×","ł","×Ļ","Ġ׾","×","¤","×¢","×Ļ×","ľ","×ķ×","ª","Ġ×ŀ","×","¡","×","Ĺ","ר","×Ļ×","ª",".","Ġ׾","×Ķ","×ij","×Ļ×","Ł","Ġ×ŀ","×Ķ","Ġ×Ķ","×","¡","×Ļ×","Ľ","×ķ×","Ł","Ġ×Ķ","×ķ×","IJ","Ġ×ŀ","×IJ","×ķ×","ĵ","Ġ×","Ĺ","ש","×ķ×","ij",".","Ġ×Ķ","×","Ĺ","×ķ","×ķ","×Ļ×","Ķ","Ġ×Ķ","×IJ","׳","×ķ×","©","×Ļ×","ª","Ġ×","¢","×ķ×","ľ","×Ķ","Ġ×","Ľ","×Ļ","Ġ×ŀ","×Ļ","Ġ×","Ļ","×ķ×","ĵ","×¢","Ġ×IJ","×Ļ×","ļ","Ġ׾","×","§","×","Ĺ","ת","Ġ×","¡","×Ļ×","Ľ","×ķ×","ł","×Ļ×","Ŀ","Ġ×ij","×","ĸ","×ŀ","×","Ł",",","Ġ×Ķ","×Ļ×","IJ","Ġ׾","׳","×","¦","×","Ĺ","Ġ×","Ĵ","×ĵ","×ķ×","ľ","×Ļ×","Ŀ",".","Ġ׾","×","ĸ","×","Ľ","×ķ×","¨","Ġ×","¤","×ķ×","ľ","×Ļ×","ĺ","×Ļ×","§"],"offsets":[[0,1],[0,1],[1,3],[2,3],[3,5],[4,5],[5,7],[6,7],[7,9],[9,11],[11,12],[12,13],[13,14],[14,15],[15,17],[17,18],[18,19],[18,19],[19,20],[19,20],[20,21],[21,22],[22,24],[23,24],[24,25],[25,27],[26,27],[27,28],[28,30],[30,31],[30,31],[31,32],[32,34],[33,34],[34,36],[35,36],[36,38],[38,39],[38,39],[39,40],[39,40],[40,41],[41,43],[42,43],[43,44],[44,46],[46,47],[47,48],[48,50],[49,50],[50,52],[52,53],[53,55],[55,56],[55,56],[56,58],[57,58],[58,60],[59,60],[60,62],[62,64],[63,64],[64,66],[66,67],[67,69],[68,69],[69,71],[70,71],[71,72],[72,74],[73,74],[74,75],[75,77],[77,78],[77,78],[78,79],[79,80],[80,82],[81,82],[82,84],[84,85],[85,86],[86,88],[87,88],[88,90],[89,90],[90,92],[91,92],[92,94],[93,94],[94,95],[95,97],[96,97],[97,98],[98,100],[100,101],[101,103],[102,103],[103,105],[104,105],[105,106],[106,108],[108,110],[109,110],[110,112],[112,113],[112,113],[113,114],[113,114],[114,115],[115,117],[116,117],[117,119],[118,119],[119,121],[120,121],[121,123],[122,123],[123,125],[125,126],[125,126],[126,127],[127,128],[127,128],[128,129],[129,131],[131,133],[132,133],[133,135],[135,136],[136,137],[136,137],[137,138],[137,138],[138,140],[139,140],[140,141],[141,143],[142,143],[143,145],[144,145],[145,146],[146,148],[148,149],[148,149],[149,150],[149,150],[150,152],[151,152],[152,154],[153,154],[154,156],[155,156],[156,158],[157,158],[158,160],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,0,1,2,2,2,2,2,3,3,3,3,3,3,3,4,5,5,5,5,5,5,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,7,8,9,9,9,9,9,10,10,11,11,11,11,11,11,11,12,12,12,13,13,13,13,14,14,14,14,14,15,16,16,16,16,16,16,16,17,17,17,17,17,17,17,18,18,18,18,18,19,19,19,20,20,21,21,21,21,21,22,22,22,23,23,23,23,23,23,24,24,24,24,24,24,24,24,25,25,25,25,25,25,26,27,27,27,28,28,28,28,28,28,29,29,29,29,29,29,29,30,31,31,31,31,31,31,31,32,32,32,32,32,32,32,32],"decoded":"סיכונים – אלמנט מרכזי, חיוני לפעילות מסחרית. להבין מה הסיכון הוא מאוד חשוב. החוויה האנושית עולה כי מי יודע איך לקחת סיכונים בזמן, היא לנצח גדולים. לזכור פוליטיק","decoded_with_specials":"סיכונים – אלמנט מרכזי, חיוני לפעילות מסחרית. להבין מה הסיכון הוא מאוד חשוב. החוויה האנושית עולה כי מי יודע איך לקחת סיכונים בזמן, היא לנצח גדולים. לזכור פוליטיק"} +{"kind":"sample","source":"fixtures/lang/hin_Deva.txt[:160]","text":"PHOTOS: न्यूज पढ़ते-पढ़ते अचानक ये क्या करने लगी एंकर!\nकुछ समय पहले एक टीवी चैनल पर एंकर खबर पढ़ रही थी और पीछे की स्क्रीन पर 10 मिनिट का पोर्न वीडियो चलता रहा,","ids":[47,39,46,51,46,50,25,367,101,811,107,358,224,281,250,1583,281,95,281,120,1329,628,12,2033,281,95,281,120,1329,628,367,227,281,248,721,101,1016,367,107,628,848,811,107,519,848,632,998,628,367,110,281,245,840,367,237,1004,1016,632,4175,1016,1533,281,249,1263,1637,281,107,1583,1554,1455,628,367,237,1016,367,253,840,281,113,840,367,248,1690,998,1455,1583,632,367,237,1004,1016,632,367,244,281,105,632,1583,281,95,281,120,367,108,1554,840,367,98,840,367,242,632,1583,840,281,249,628,848,840,1263,811,243,2062,840,998,1583,632,220,702,1559,911,101,911,253,848,519,1583,952,632,811,101,367,113,840,281,94,911,107,952,367,248,1455,1329,519,367,108,1554,519,11],"ids_no_specials":[47,39,46,51,46,50,25,367,101,811,107,358,224,281,250,1583,281,95,281,120,1329,628,12,2033,281,95,281,120,1329,628,367,227,281,248,721,101,1016,367,107,628,848,811,107,519,848,632,998,628,367,110,281,245,840,367,237,1004,1016,632,4175,1016,1533,281,249,1263,1637,281,107,1583,1554,1455,628,367,237,1016,367,253,840,281,113,840,367,248,1690,998,1455,1583,632,367,237,1004,1016,632,367,244,281,105,632,1583,281,95,281,120,367,108,1554,840,367,98,840,367,242,632,1583,840,281,249,628,848,840,1263,811,243,2062,840,998,1583,632,220,702,1559,911,101,911,253,848,519,1583,952,632,811,101,367,113,840,281,94,911,107,952,367,248,1455,1329,519,367,108,1554,519,11],"tokens":["P","H","O","T","O","S",":","Ġà¤","¨","à¥įà¤","¯","à¥","Ĥ","à¤","ľ","Ġप","à¤","¢","à¤","¼","त","à¥ĩ","-","प","à¤","¢","à¤","¼","त","à¥ĩ","Ġà¤","ħ","à¤","ļ","ाà¤","¨","à¤ķ","Ġà¤","¯","à¥ĩ","Ġà¤ķ","à¥įà¤","¯","ा","Ġà¤ķ","र","न","à¥ĩ","Ġà¤","²","à¤","Ĺ","à¥Ģ","Ġà¤","ı","à¤Ĥ","à¤ķ","र","!Ċ","à¤ķ","à¥ģ","à¤","Ľ","Ġस","म","à¤","¯","Ġप","ह","ल","à¥ĩ","Ġà¤","ı","à¤ķ","Ġà¤","Ł","à¥Ģ","à¤","µ","à¥Ģ","Ġà¤","ļ","à¥Ī","न","ल","Ġप","र","Ġà¤","ı","à¤Ĥ","à¤ķ","र","Ġà¤","ĸ","à¤","¬","र","Ġप","à¤","¢","à¤","¼","Ġà¤","°","ह","à¥Ģ","Ġà¤","¥","à¥Ģ","Ġà¤","Ķ","र","Ġप","à¥Ģ","à¤","Ľ","à¥ĩ","Ġà¤ķ","à¥Ģ","Ġस","à¥įà¤","ķ","à¥įर","à¥Ģ","न","Ġप","र","Ġ","10","Ġम","िà¤","¨","िà¤","Ł","Ġà¤ķ","ा","Ġप","à¥ĭ","र","à¥įà¤","¨","Ġà¤","µ","à¥Ģ","à¤","¡","िà¤","¯","à¥ĭ","Ġà¤","ļ","ल","त","ा","Ġà¤","°","ह","ा",","],"offsets":[[0,1],[1,2],[2,3],[3,4],[4,5],[5,6],[6,7],[7,9],[8,9],[9,11],[10,11],[11,12],[11,12],[12,13],[12,13],[13,15],[15,16],[15,16],[16,17],[16,17],[17,18],[18,19],[19,20],[20,21],[21,22],[21,22],[22,23],[22,23],[23,24],[24,25],[25,27],[26,27],[27,28],[27,28],[28,30],[29,30],[30,31],[31,33],[32,33],[33,34],[34,36],[36,38],[37,38],[38,39],[39,41],[41,42],[42,43],[43,44],[44,46],[45,46],[46,47],[46,47],[47,48],[48,50],[49,50],[50,51],[51,52],[52,53],[53,55],[55,56],[56,57],[57,58],[57,58],[58,60],[60,61],[61,62],[61,62],[62,64],[64,65],[65,66],[66,67],[67,69],[68,69],[69,70],[70,72],[71,72],[72,73],[73,74],[73,74],[74,75],[75,77],[76,77],[77,78],[78,79],[79,80],[80,82],[82,83],[83,85],[84,85],[85,86],[86,87],[87,88],[88,90],[89,90],[90,91],[90,91],[91,92],[92,94],[94,95],[94,95],[95,96],[95,96],[96,98],[97,98],[98,99],[99,100],[100,102],[101,102],[102,103],[103,105],[104,105],[105,106],[106,108],[108,109],[109,110],[109,110],[110,111],[111,113],[113,114],[114,116],[116,118],[117,118],[118,120],[120,121],[121,122],[122,124],[124,125],[125,126],[126,128],[128,130],[130,132],[131,132],[132,134],[133,134],[134,136],[136,137],[137,139],[139,140],[140,141],[141,143],[142,143],[143,145],[144,145],[145,146],[146,147],[146,147],[147,149],[148,149],[149,150],[150,152],[151,152],[152,153],[153,154],[154,155],[155,157],[156,157],[157,158],[158,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,1,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,6,6,6,7,7,7,7,8,8,8,8,9,9,9,9,9,10,10,10,10,10,11,12,12,12,12,13,13,13,13,14,14,14,14,15,15,15,16,16,16,16,16,16,17,17,17,17,17,18,18,19,19,19,19,19,20,20,20,20,20,21,21,21,21,21,22,22,22,22,23,23,23,24,24,24,25,25,25,25,25,26,26,27,27,27,27,27,27,28,28,29,30,31,31,31,31,31,32,32,33,33,33,33,33,34,34,34,34,34,34,34,34,35,35,35,35,35,36,36,36,36,37],"decoded":"PHOTOS: न्यूज पढ़ते-पढ़ते अचानक ये क्या करने लगी एंकर!\nकुछ समय पहले एक टीवी चैनल पर एंकर खबर पढ़ रही थी और पीछे की स्क्रीन पर 10 मिनिट का पोर्न वीडियो चलता रहा,","decoded_with_specials":"PHOTOS: न्यूज पढ़ते-पढ़ते अचानक ये क्या करने लगी एंकर!\nकुछ समय पहले एक टीवी चैनल पर एंकर खबर पढ़ रही थी और पीछे की स्क्रीन पर 10 मिनिट का पोर्न वीडियो चलता रहा,"} +{"kind":"sample","source":"fixtures/lang/jpn_Jpan.txt[:160]","text":"Omni Dallas Parkwest Hotelでは4ツ星ホテルで、アイアンホース・ゴルフコース、Love Field Airport とテキサス・スタジアムから7.8kmかかるのところにあります。 優れたホテルにオープンし、ダラスにある古代の建築の象徴です。\n部屋\n快適なゲストルームには、モダンな設備を備えたプレ","ids":[46,76,1906,415,586,288,398,941,86,376,487,346,296,605,100,605,107,19,769,226,2181,253,769,249,769,228,769,104,605,100,1395,845,95,845,97,845,95,769,111,769,249,769,120,845,117,769,119,845,112,769,104,769,243,845,111,769,120,845,117,1395,43,1048,454,1174,355,380,447,220,605,101,769,228,845,255,845,113,845,117,769,119,845,117,845,123,845,116,845,95,769,254,605,233,845,231,22,13,23,74,76,605,233,605,233,845,233,605,106,605,101,605,241,845,235,605,104,605,224,845,232,605,122,605,247,788,1222,226,103,845,234,605,253,769,249,769,228,769,104,605,104,845,103,769,120,769,245,769,111,605,245,1395,769,222,769,102,845,117,605,104,605,224,845,233,948,97,1467,96,605,106,161,119,118,163,107,231,605,106,164,109,94,161,122,112,605,100,605,247,3414,165,225,101,161,109,233,198,161,123,104,165,223,102,605,103,845,110,845,117,769,230,769,104,769,120,769,254,605,104,605,107,1395,769,95,769,222,769,111,605,103,164,101,255,161,224,247,845,240,161,224,247,605,230,605,253,769,245,769,105],"ids_no_specials":[46,76,1906,415,586,288,398,941,86,376,487,346,296,605,100,605,107,19,769,226,2181,253,769,249,769,228,769,104,605,100,1395,845,95,845,97,845,95,769,111,769,249,769,120,845,117,769,119,845,112,769,104,769,243,845,111,769,120,845,117,1395,43,1048,454,1174,355,380,447,220,605,101,769,228,845,255,845,113,845,117,769,119,845,117,845,123,845,116,845,95,769,254,605,233,845,231,22,13,23,74,76,605,233,605,233,845,233,605,106,605,101,605,241,845,235,605,104,605,224,845,232,605,122,605,247,788,1222,226,103,845,234,605,253,769,249,769,228,769,104,605,104,845,103,769,120,769,245,769,111,605,245,1395,769,222,769,102,845,117,605,104,605,224,845,233,948,97,1467,96,605,106,161,119,118,163,107,231,605,106,164,109,94,161,122,112,605,100,605,247,3414,165,225,101,161,109,233,198,161,123,104,165,223,102,605,103,845,110,845,117,769,230,769,104,769,120,769,254,605,104,605,107,1395,769,95,769,222,769,111,605,103,164,101,255,161,224,247,845,240,161,224,247,605,230,605,253,769,245,769,105],"tokens":["O","m","ni","ĠD","all","as","ĠP","ark","w","est","ĠH","ot","el","ãģ","§","ãģ","¯","4","ãĥ","Ħ","æĺ","Ł","ãĥ","Ľ","ãĥ","Ĩ","ãĥ","«","ãģ","§","ãĢģ","ãĤ","¢","ãĤ","¤","ãĤ","¢","ãĥ","³","ãĥ","Ľ","ãĥ","¼","ãĤ","¹","ãĥ","»","ãĤ","´","ãĥ","«","ãĥ","ķ","ãĤ","³","ãĥ","¼","ãĤ","¹","ãĢģ","L","ove","ĠF","ield","ĠA","ir","port","Ġ","ãģ","¨","ãĥ","Ĩ","ãĤ","Ń","ãĤ","µ","ãĤ","¹","ãĥ","»","ãĤ","¹","ãĤ","¿","ãĤ","¸","ãĤ","¢","ãĥ","ł","ãģ","ĭ","ãĤ","ī","7",".","8","k","m","ãģ","ĭ","ãģ","ĭ","ãĤ","ĭ","ãģ","®","ãģ","¨","ãģ","ĵ","ãĤ","į","ãģ","«","ãģ","Ĥ","ãĤ","Ĭ","ãģ","¾","ãģ","Ļ","ãĢĤ","Ġå","Ħ","ª","ãĤ","Į","ãģ","Ł","ãĥ","Ľ","ãĥ","Ĩ","ãĥ","«","ãģ","«","ãĤ","ª","ãĥ","¼","ãĥ","Ĺ","ãĥ","³","ãģ","Ĺ","ãĢģ","ãĥ","Ģ","ãĥ","©","ãĤ","¹","ãģ","«","ãģ","Ĥ","ãĤ","ĭ","åı","¤","ä»","£","ãģ","®","å","»","º","ç","¯","ī","ãģ","®","è","±","¡","å","¾","´","ãģ","§","ãģ","Ļ","ãĢĤĊ","é","ĥ","¨","å","±","ĭ","Ċ","å","¿","«","é","ģ","©","ãģ","ª","ãĤ","²","ãĤ","¹","ãĥ","Ī","ãĥ","«","ãĥ","¼","ãĥ","ł","ãģ","«","ãģ","¯","ãĢģ","ãĥ","¢","ãĥ","Ģ","ãĥ","³","ãģ","ª","è","¨","Ń","å","Ĥ","Ļ","ãĤ","Ĵ","å","Ĥ","Ļ","ãģ","Ī","ãģ","Ł","ãĥ","Ĺ","ãĥ","¬"],"offsets":[[0,1],[1,2],[2,4],[4,6],[6,9],[9,11],[11,13],[13,16],[16,17],[17,20],[20,22],[22,24],[24,26],[26,27],[26,27],[27,28],[27,28],[28,29],[29,30],[29,30],[30,31],[30,31],[31,32],[31,32],[32,33],[32,33],[33,34],[33,34],[34,35],[34,35],[35,36],[36,37],[36,37],[37,38],[37,38],[38,39],[38,39],[39,40],[39,40],[40,41],[40,41],[41,42],[41,42],[42,43],[42,43],[43,44],[43,44],[44,45],[44,45],[45,46],[45,46],[46,47],[46,47],[47,48],[47,48],[48,49],[48,49],[49,50],[49,50],[50,51],[51,52],[52,55],[55,57],[57,61],[61,63],[63,65],[65,69],[69,70],[70,71],[70,71],[71,72],[71,72],[72,73],[72,73],[73,74],[73,74],[74,75],[74,75],[75,76],[75,76],[76,77],[76,77],[77,78],[77,78],[78,79],[78,79],[79,80],[79,80],[80,81],[80,81],[81,82],[81,82],[82,83],[82,83],[83,84],[84,85],[85,86],[86,87],[87,88],[88,89],[88,89],[89,90],[89,90],[90,91],[90,91],[91,92],[91,92],[92,93],[92,93],[93,94],[93,94],[94,95],[94,95],[95,96],[95,96],[96,97],[96,97],[97,98],[97,98],[98,99],[98,99],[99,100],[99,100],[100,101],[101,103],[102,103],[102,103],[103,104],[103,104],[104,105],[104,105],[105,106],[105,106],[106,107],[106,107],[107,108],[107,108],[108,109],[108,109],[109,110],[109,110],[110,111],[110,111],[111,112],[111,112],[112,113],[112,113],[113,114],[113,114],[114,115],[115,116],[115,116],[116,117],[116,117],[117,118],[117,118],[118,119],[118,119],[119,120],[119,120],[120,121],[120,121],[121,122],[121,122],[122,123],[122,123],[123,124],[123,124],[124,125],[124,125],[124,125],[125,126],[125,126],[125,126],[126,127],[126,127],[127,128],[127,128],[127,128],[128,129],[128,129],[128,129],[129,130],[129,130],[130,131],[130,131],[131,133],[133,134],[133,134],[133,134],[134,135],[134,135],[134,135],[135,136],[136,137],[136,137],[136,137],[137,138],[137,138],[137,138],[138,139],[138,139],[139,140],[139,140],[140,141],[140,141],[141,142],[141,142],[142,143],[142,143],[143,144],[143,144],[144,145],[144,145],[145,146],[145,146],[146,147],[146,147],[147,148],[148,149],[148,149],[149,150],[149,150],[150,151],[150,151],[151,152],[151,152],[152,153],[152,153],[152,153],[153,154],[153,154],[153,154],[154,155],[154,155],[155,156],[155,156],[155,156],[156,157],[156,157],[157,158],[157,158],[158,159],[158,159],[159,160],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,1,1,1,2,2,2,2,3,3,3,3,3,3,3,4,5,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,7,7,7,7,7,7,7,8,8,8,9,9,10,10,10,11,11,11,11,11,11,11,11,11,11,11,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,13,14,15,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,17,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,20,21,21,21,21,21,21,22,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24],"decoded":"Omni Dallas Parkwest Hotelでは4ツ星ホテルで、アイアンホース・ゴルフコース、Love Field Airport とテキサス・スタジアムから7.8kmかかるのところにあります。 優れたホテルにオープンし、ダラスにある古代の建築の象徴です。\n部屋\n快適なゲストルームには、モダンな設備を備えたプレ","decoded_with_specials":"Omni Dallas Parkwest Hotelでは4ツ星ホテルで、アイアンホース・ゴルフコース、Love Field Airport とテキサス・スタジアムから7.8kmかかるのところにあります。 優れたホテルにオープンし、ダラスにある古代の建築の象徴です。\n部屋\n快適なゲストルームには、モダンな設備を備えたプレ"} +{"kind":"sample","source":"fixtures/lang/kat_Geor.txt[:160]","text":"ჩვენ მსოფლიოს სხვადასხვა ქვეყანაში ვცხოვრობთ და სხვადასხვა ენაზე ვსაუბრობთ, მაგრამ საერთო მიზანი გვაერთიანებს. ჩვენი მთავარი მიზანი სამყაროს შემოქმედისა და ბიბლ","ids":[303,102,303,243,856,250,636,249,303,94,1326,97,303,248,928,251,303,94,636,94,303,106,303,243,671,241,671,94,303,106,303,243,587,636,98,303,243,856,100,671,250,671,101,688,636,243,303,103,303,106,1326,243,303,254,1326,239,303,245,636,241,587,636,94,303,106,303,243,671,241,671,94,303,106,303,243,587,636,242,303,250,671,244,794,636,243,303,94,671,1857,239,303,254,1326,239,303,245,11,636,249,671,240,303,254,671,249,636,94,671,242,303,2121,245,1149,636,249,928,244,671,250,688,636,240,303,243,671,242,303,2121,245,688,671,250,1894,303,94,13,636,102,303,243,856,250,688,636,249,303,245,671,243,671,254,688,636,249,928,244,671,250,688,636,94,671,249,303,100,671,254,1326,94,636,101,856,249,1326,98,303,249,856,241,1863,587,636,241,587,636,239,928,239,303,248],"ids_no_specials":[303,102,303,243,856,250,636,249,303,94,1326,97,303,248,928,251,303,94,636,94,303,106,303,243,671,241,671,94,303,106,303,243,587,636,98,303,243,856,100,671,250,671,101,688,636,243,303,103,303,106,1326,243,303,254,1326,239,303,245,636,241,587,636,94,303,106,303,243,671,241,671,94,303,106,303,243,587,636,242,303,250,671,244,794,636,243,303,94,671,1857,239,303,254,1326,239,303,245,11,636,249,671,240,303,254,671,249,636,94,671,242,303,2121,245,1149,636,249,928,244,671,250,688,636,240,303,243,671,242,303,2121,245,688,671,250,1894,303,94,13,636,102,303,243,856,250,688,636,249,303,245,671,243,671,254,688,636,249,928,244,671,250,688,636,94,671,249,303,100,671,254,1326,94,636,101,856,249,1326,98,303,249,856,241,1863,587,636,241,587,636,239,928,239,303,248],"tokens":["áĥ","©","áĥ","ķ","áĥĶáĥ","ľ","Ġáĥ","Ľ","áĥ","¡","áĥĿáĥ","¤","áĥ","ļ","áĥĺáĥ","Ŀ","áĥ","¡","Ġáĥ","¡","áĥ","®","áĥ","ķ","áĥIJáĥ","ĵ","áĥIJáĥ","¡","áĥ","®","áĥ","ķ","áĥIJ","Ġáĥ","¥","áĥ","ķ","áĥĶáĥ","§","áĥIJáĥ","ľ","áĥIJáĥ","¨","áĥĺ","Ġáĥ","ķ","áĥ","ª","áĥ","®","áĥĿáĥ","ķ","áĥ","ł","áĥĿáĥ","ij","áĥ","Ĺ","Ġáĥ","ĵ","áĥIJ","Ġáĥ","¡","áĥ","®","áĥ","ķ","áĥIJáĥ","ĵ","áĥIJáĥ","¡","áĥ","®","áĥ","ķ","áĥIJ","Ġáĥ","Ķ","áĥ","ľ","áĥIJáĥ","ĸ","áĥĶ","Ġáĥ","ķ","áĥ","¡","áĥIJáĥ","£áĥ","ij","áĥ","ł","áĥĿáĥ","ij","áĥ","Ĺ",",","Ġáĥ","Ľ","áĥIJáĥ","Ĵ","áĥ","ł","áĥIJáĥ","Ľ","Ġáĥ","¡","áĥIJáĥ","Ķ","áĥ","łáĥ","Ĺ","áĥĿ","Ġáĥ","Ľ","áĥĺáĥ","ĸ","áĥIJáĥ","ľ","áĥĺ","Ġáĥ","Ĵ","áĥ","ķ","áĥIJáĥ","Ķ","áĥ","łáĥ","Ĺ","áĥĺ","áĥIJáĥ","ľ","áĥĶáĥij","áĥ","¡",".","Ġáĥ","©","áĥ","ķ","áĥĶáĥ","ľ","áĥĺ","Ġáĥ","Ľ","áĥ","Ĺ","áĥIJáĥ","ķ","áĥIJáĥ","ł","áĥĺ","Ġáĥ","Ľ","áĥĺáĥ","ĸ","áĥIJáĥ","ľ","áĥĺ","Ġáĥ","¡","áĥIJáĥ","Ľ","áĥ","§","áĥIJáĥ","ł","áĥĿáĥ","¡","Ġáĥ","¨","áĥĶáĥ","Ľ","áĥĿáĥ","¥","áĥ","Ľ","áĥĶáĥ","ĵ","áĥĺáĥ¡","áĥIJ","Ġáĥ","ĵ","áĥIJ","Ġáĥ","ij","áĥĺáĥ","ij","áĥ","ļ"],"offsets":[[0,1],[0,1],[1,2],[1,2],[2,4],[3,4],[4,6],[5,6],[6,7],[6,7],[7,9],[8,9],[9,10],[9,10],[10,12],[11,12],[12,13],[12,13],[13,15],[14,15],[15,16],[15,16],[16,17],[16,17],[17,19],[18,19],[19,21],[20,21],[21,22],[21,22],[22,23],[22,23],[23,24],[24,26],[25,26],[26,27],[26,27],[27,29],[28,29],[29,31],[30,31],[31,33],[32,33],[33,34],[34,36],[35,36],[36,37],[36,37],[37,38],[37,38],[38,40],[39,40],[40,41],[40,41],[41,43],[42,43],[43,44],[43,44],[44,46],[45,46],[46,47],[47,49],[48,49],[49,50],[49,50],[50,51],[50,51],[51,53],[52,53],[53,55],[54,55],[55,56],[55,56],[56,57],[56,57],[57,58],[58,60],[59,60],[60,61],[60,61],[61,63],[62,63],[63,64],[64,66],[65,66],[66,67],[66,67],[67,69],[68,70],[69,70],[70,71],[70,71],[71,73],[72,73],[73,74],[73,74],[74,75],[75,77],[76,77],[77,79],[78,79],[79,80],[79,80],[80,82],[81,82],[82,84],[83,84],[84,86],[85,86],[86,87],[86,88],[87,88],[88,89],[89,91],[90,91],[91,93],[92,93],[93,95],[94,95],[95,96],[96,98],[97,98],[98,99],[98,99],[99,101],[100,101],[101,102],[101,103],[102,103],[103,104],[104,106],[105,106],[106,108],[108,109],[108,109],[109,110],[110,112],[111,112],[112,113],[112,113],[113,115],[114,115],[115,116],[116,118],[117,118],[118,119],[118,119],[119,121],[120,121],[121,123],[122,123],[123,124],[124,126],[125,126],[126,128],[127,128],[128,130],[129,130],[130,131],[131,133],[132,133],[133,135],[134,135],[135,136],[135,136],[136,138],[137,138],[138,140],[139,140],[140,142],[141,142],[142,144],[143,144],[144,146],[145,146],[146,147],[146,147],[147,149],[148,149],[149,151],[151,152],[152,154],[153,154],[154,155],[155,157],[156,157],[157,159],[158,159],[159,160],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,4,4,5,5,5,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,8,8,8,8,8,8,8,8,8,8,8,8,8,9,10,10,10,10,10,10,10,10,11,11,11,11,11,11,11,11,12,12,12,12,12,12,12,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,14,15,15,15,15,15,15,15,16,16,16,16,16,16,16,16,16,17,17,17,17,17,17,17,18,18,18,18,18,18,18,18,18,18,19,19,19,19,19,19,19,19,19,19,19,19,20,20,20,21,21,21,21,21,21],"decoded":"ჩვენ მსოფლიოს სხვადასხვა ქვეყანაში ვცხოვრობთ და სხვადასხვა ენაზე ვსაუბრობთ, მაგრამ საერთო მიზანი გვაერთიანებს. ჩვენი მთავარი მიზანი სამყაროს შემოქმედისა და ბიბლ","decoded_with_specials":"ჩვენ მსოფლიოს სხვადასხვა ქვეყანაში ვცხოვრობთ და სხვადასხვა ენაზე ვსაუბრობთ, მაგრამ საერთო მიზანი გვაერთიანებს. ჩვენი მთავარი მიზანი სამყაროს შემოქმედისა და ბიბლ"} +{"kind":"sample","source":"fixtures/lang/kor_Hang.txt[:160]","text":"전화번호:\n위치: 뉴질랜드 > 남섬 > 말버러 > 블레넘\n가격대 (1박): 117,886 원 - 140,244 원\n4.5성급 — Lugano Motor Lodge 4.5*\n- 예약 옵션:\n- 트립어드바이저는 호텔스닷컴, Expedia, Agoda, Asia Web Direct 및 Boo","ids":[168,254,226,169,247,242,167,110,230,169,246,116,734,168,250,226,168,117,246,25,1112,231,112,168,100,230,167,252,250,167,241,250,1424,1112,224,101,168,226,105,1424,1112,100,238,167,110,226,167,253,105,1424,1112,116,242,167,254,230,167,226,246,198,166,108,222,166,110,102,167,234,222,350,16,167,108,243,3127,220,994,22,11,23,23,21,804,249,238,533,220,1265,15,11,1494,19,804,249,238,198,19,13,20,168,226,109,166,116,231,559,242,451,846,270,78,391,346,267,451,368,684,220,19,13,20,6658,12,804,246,230,168,243,121,804,246,113,168,227,246,734,12,220,169,232,116,167,99,121,168,244,112,167,241,250,167,108,242,2186,168,254,222,167,232,242,220,169,246,116,169,227,242,168,232,97,1528,115,168,119,112,11,1771,79,295,535,11,355,70,368,64,11,1877,535,486,1113,415,1681,1112,108,237,418,78,78],"ids_no_specials":[168,254,226,169,247,242,167,110,230,169,246,116,734,168,250,226,168,117,246,25,1112,231,112,168,100,230,167,252,250,167,241,250,1424,1112,224,101,168,226,105,1424,1112,100,238,167,110,226,167,253,105,1424,1112,116,242,167,254,230,167,226,246,198,166,108,222,166,110,102,167,234,222,350,16,167,108,243,3127,220,994,22,11,23,23,21,804,249,238,533,220,1265,15,11,1494,19,804,249,238,198,19,13,20,168,226,109,166,116,231,559,242,451,846,270,78,391,346,267,451,368,684,220,19,13,20,6658,12,804,246,230,168,243,121,804,246,113,168,227,246,734,12,220,169,232,116,167,99,121,168,244,112,167,241,250,167,108,242,2186,168,254,222,167,232,242,220,169,246,116,169,227,242,168,232,97,1528,115,168,119,112,11,1771,79,295,535,11,355,70,368,64,11,1877,535,486,1113,415,1681,1112,108,237,418,78,78],"tokens":["ì","ł","Ħ","í","Ļ","Ķ","ë","²","Ī","í","ĺ","¸",":Ċ","ì","ľ","Ħ","ì","¹","ĺ",":","Ġë","ī","´","ì","§","Ī","ë","ŀ","ľ","ë","ĵ","ľ","Ġ>","Ġë","Ĥ","¨","ì","Ħ","¬","Ġ>","Ġë","§","IJ","ë","²","Ħ","ë","Ł","¬","Ġ>","Ġë","¸","Ķ","ë","ł","Ī","ë","Ħ","ĺ","Ċ","ê","°","Ģ","ê","²","©","ë","Į","Ģ","Ġ(","1","ë","°","ķ","):","Ġ","11","7",",","8","8","6","Ġì","Ľ","IJ","Ġ-","Ġ","14","0",",","24","4","Ġì","Ľ","IJ","Ċ","4",".","5","ì","Ħ","±","ê","¸","ī","ĠâĢ","Ķ","ĠL","ug","an","o","ĠM","ot","or","ĠL","od","ge","Ġ","4",".","5","*Ċ","-","Ġì","ĺ","Ī","ì","ķ","½","Ġì","ĺ","µ","ì","ħ","ĺ",":Ċ","-","Ġ","í","Ĭ","¸","ë","¦","½","ì","ĸ","´","ë","ĵ","ľ","ë","°","Ķ","ìĿ´","ì","ł","Ģ","ë","Ĭ","Ķ","Ġ","í","ĺ","¸","í","ħ","Ķ","ì","Ĭ","¤","ëĭ","·","ì","»","´",",","ĠEx","p","ed","ia",",","ĠA","g","od","a",",","ĠAs","ia","ĠW","eb","ĠD","irect","Ġë","°","ı","ĠB","o","o"],"offsets":[[0,1],[0,1],[0,1],[1,2],[1,2],[1,2],[2,3],[2,3],[2,3],[3,4],[3,4],[3,4],[4,6],[6,7],[6,7],[6,7],[7,8],[7,8],[7,8],[8,9],[9,11],[10,11],[10,11],[11,12],[11,12],[11,12],[12,13],[12,13],[12,13],[13,14],[13,14],[13,14],[14,16],[16,18],[17,18],[17,18],[18,19],[18,19],[18,19],[19,21],[21,23],[22,23],[22,23],[23,24],[23,24],[23,24],[24,25],[24,25],[24,25],[25,27],[27,29],[28,29],[28,29],[29,30],[29,30],[29,30],[30,31],[30,31],[30,31],[31,32],[32,33],[32,33],[32,33],[33,34],[33,34],[33,34],[34,35],[34,35],[34,35],[35,37],[37,38],[38,39],[38,39],[38,39],[39,41],[41,42],[42,44],[44,45],[45,46],[46,47],[47,48],[48,49],[49,51],[50,51],[50,51],[51,53],[53,54],[54,56],[56,57],[57,58],[58,60],[60,61],[61,63],[62,63],[62,63],[63,64],[64,65],[65,66],[66,67],[67,68],[67,68],[67,68],[68,69],[68,69],[68,69],[69,71],[70,71],[71,73],[73,75],[75,77],[77,78],[78,80],[80,82],[82,84],[84,86],[86,88],[88,90],[90,91],[91,92],[92,93],[93,94],[94,96],[96,97],[97,99],[98,99],[98,99],[99,100],[99,100],[99,100],[100,102],[101,102],[101,102],[102,103],[102,103],[102,103],[103,105],[105,106],[106,107],[107,108],[107,108],[107,108],[108,109],[108,109],[108,109],[109,110],[109,110],[109,110],[110,111],[110,111],[110,111],[111,112],[111,112],[111,112],[112,113],[113,114],[113,114],[113,114],[114,115],[114,115],[114,115],[115,116],[116,117],[116,117],[116,117],[117,118],[117,118],[117,118],[118,119],[118,119],[118,119],[119,120],[119,120],[120,121],[120,121],[120,121],[121,122],[122,125],[125,126],[126,128],[128,130],[130,131],[131,133],[133,134],[134,136],[136,137],[137,138],[138,141],[141,143],[143,145],[145,147],[147,149],[149,154],[154,156],[155,156],[155,156],[156,158],[158,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,0,0,0,0,0,1,2,2,2,2,2,2,3,4,4,4,4,4,4,4,4,4,4,4,4,5,6,6,6,6,6,6,7,8,8,8,8,8,8,8,8,8,9,10,10,10,10,10,10,10,10,10,11,12,12,12,12,12,12,12,12,12,13,14,15,15,15,16,17,18,18,19,20,20,20,21,21,21,22,23,24,24,25,26,26,27,27,27,28,29,30,31,32,32,32,32,32,32,33,33,34,34,34,34,35,35,35,36,36,36,37,38,39,40,41,42,43,43,43,43,43,43,44,44,44,44,44,44,45,46,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,49,50,50,50,50,51,52,52,52,52,53,54,54,55,55,56,56,57,57,57,58,58,58],"decoded":"전화번호:\n위치: 뉴질랜드 > 남섬 > 말버러 > 블레넘\n가격대 (1박): 117,886 원 - 140,244 원\n4.5성급 — Lugano Motor Lodge 4.5*\n- 예약 옵션:\n- 트립어드바이저는 호텔스닷컴, Expedia, Agoda, Asia Web Direct 및 Boo","decoded_with_specials":"전화번호:\n위치: 뉴질랜드 > 남섬 > 말버러 > 블레넘\n가격대 (1박): 117,886 원 - 140,244 원\n4.5성급 — Lugano Motor Lodge 4.5*\n- 예약 옵션:\n- 트립어드바이저는 호텔스닷컴, Expedia, Agoda, Asia Web Direct 및 Boo"} +{"kind":"sample","source":"fixtures/lang/rus_Cyrl.txt[:160]","text":"Покупая продукты в супермаркетах, сегодня уже мало кто верит в их качество. Как не сделать свое меню экстремальным и на что следует обращать внимание – читайте!","ids":[140,253,439,118,442,666,304,544,1266,1267,442,462,338,465,743,669,442,666,1025,500,969,462,1041,304,724,11,669,327,683,1267,341,544,1449,1065,327,946,1019,317,828,2192,743,1025,1108,743,816,724,828,304,664,327,749,520,317,13,300,248,1945,1967,669,489,1260,1035,615,669,520,317,327,946,776,1086,220,1226,462,749,345,1574,1019,615,1208,500,816,1235,1848,2192,669,383,1677,442,1041,913,757,714,1717,1035,615,743,1287,500,841,331,327,1127,1848,1108,304,719,338,327,0],"ids_no_specials":[140,253,439,118,442,666,304,544,1266,1267,442,462,338,465,743,669,442,666,1025,500,969,462,1041,304,724,11,669,327,683,1267,341,544,1449,1065,327,946,1019,317,828,2192,743,1025,1108,743,816,724,828,304,664,327,749,520,317,13,300,248,1945,1967,669,489,1260,1035,615,669,520,317,327,946,776,1086,220,1226,462,749,345,1574,1019,615,1208,500,816,1235,1848,2192,669,383,1677,442,1041,913,757,714,1717,1035,615,743,1287,500,841,331,327,1127,1848,1108,304,719,338,327,0],"tokens":["Ð","Ł","оÐ","º","Ñĥ","п","а","Ñı","ĠпÑĢ","од","Ñĥ","к","ÑĤ","Ñĭ","Ġв","ĠÑģ","Ñĥ","п","еÑĢ","м","аÑĢ","к","еÑĤ","а","Ñħ",",","ĠÑģ","е","г","од","н","Ñı","ĠÑĥ","ж","е","Ġм","ал","о","Ġк","ÑĤо","Ġв","еÑĢ","иÑĤ","Ġв","Ġи","Ñħ","Ġк","а","Ñĩ","е","ÑģÑĤ","в","о",".","ĠÐ","ļ","ак","Ġне","ĠÑģ","д","ел","аÑĤ","ÑĮ","ĠÑģ","в","о","е","Ġм","ен","Ñİ","Ġ","Ñį","к","ÑģÑĤ","ÑĢ","ем","ал","ÑĮ","нÑĭ","м","Ġи","Ġна","ĠÑĩ","ÑĤо","ĠÑģ","л","ед","Ñĥ","еÑĤ","Ġо","б","ÑĢа","Ñī","аÑĤ","ÑĮ","Ġв","ни","м","ан","и","е","ĠâĢĵ","ĠÑĩ","иÑĤ","а","й","ÑĤ","е","!"],"offsets":[[0,1],[0,1],[1,3],[2,3],[3,4],[4,5],[5,6],[6,7],[7,10],[10,12],[12,13],[13,14],[14,15],[15,16],[16,18],[18,20],[20,21],[21,22],[22,24],[24,25],[25,27],[27,28],[28,30],[30,31],[31,32],[32,33],[33,35],[35,36],[36,37],[37,39],[39,40],[40,41],[41,43],[43,44],[44,45],[45,47],[47,49],[49,50],[50,52],[52,54],[54,56],[56,58],[58,60],[60,62],[62,64],[64,65],[65,67],[67,68],[68,69],[69,70],[70,72],[72,73],[73,74],[74,75],[75,77],[76,77],[77,79],[79,82],[82,84],[84,85],[85,87],[87,89],[89,90],[90,92],[92,93],[93,94],[94,95],[95,97],[97,99],[99,100],[100,101],[101,102],[102,103],[103,105],[105,106],[106,108],[108,110],[110,111],[111,113],[113,114],[114,116],[116,119],[119,121],[121,123],[123,125],[125,126],[126,128],[128,129],[129,131],[131,133],[133,134],[134,136],[136,137],[137,139],[139,140],[140,142],[142,144],[144,145],[145,147],[147,148],[148,149],[149,151],[151,153],[153,155],[155,156],[156,157],[157,158],[158,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,0,1,1,1,1,1,1,2,3,3,3,3,3,3,3,3,3,3,4,5,5,5,5,5,5,6,6,6,7,7,7,8,8,9,9,9,10,11,11,12,12,12,12,12,12,12,13,14,14,14,15,16,16,16,16,16,17,17,17,17,18,18,18,19,19,19,19,19,19,19,19,19,19,20,21,22,22,23,23,23,23,23,24,24,24,24,24,24,25,25,25,25,25,25,26,27,27,27,27,27,27,28],"decoded":"Покупая продукты в супермаркетах, сегодня уже мало кто верит в их качество. Как не сделать свое меню экстремальным и на что следует обращать внимание – читайте!","decoded_with_specials":"Покупая продукты в супермаркетах, сегодня уже мало кто верит в их качество. Как не сделать свое меню экстремальным и на что следует обращать внимание – читайте!"} +{"kind":"sample","source":"fixtures/lang/tam_Taml.txt[:160]","text":"‘ஹலோ கலெக்டர் சாரா… சரக்கு வேணும்… கடை எப்ப திறப்பீங்க?’\nஒரு படத்தில் ஒயின்ஷாப்புக்குள் திருடப் போன வடிவேலு நன்றாக சரக்கடித்து விட்டு, ஒயின்ஷாப் ஓனருக்கு போன் ப","ids":[318,246,473,117,473,110,647,233,1060,243,473,110,647,228,473,243,1257,253,473,108,902,1060,248,473,122,473,108,473,122,1131,1060,248,473,108,473,243,1257,243,1694,1060,113,647,229,473,96,1694,473,106,902,1131,1060,243,473,253,647,230,1060,236,473,103,1257,103,1060,97,1856,109,473,103,1257,103,647,222,473,247,1257,243,30,438,198,473,240,473,108,1694,1060,103,473,253,473,97,1257,97,1856,110,902,1060,240,473,107,1856,102,1257,115,473,122,473,103,1257,103,1694,473,243,1257,243,1694,473,111,902,1060,97,1856,108,1694,473,253,473,103,902,1060,103,647,233,473,102,1060,113,473,253,1856,113,647,229,473,110,1694,1060,101,473,102,1257,109,473,122,473,243,1060,248,473,108,473,243,1257,243,473,253,1856,97,1257,97,1694,1060,113,1856,253,1257,253,1694,11,1060,240,473,107,1856,102,1257,115,473,122,473,103,902,1060,241,473,102,473,108,1694,473,243,1257,243,1694,1060,103,647,233,473,102,902,1060,103],"ids_no_specials":[318,246,473,117,473,110,647,233,1060,243,473,110,647,228,473,243,1257,253,473,108,902,1060,248,473,122,473,108,473,122,1131,1060,248,473,108,473,243,1257,243,1694,1060,113,647,229,473,96,1694,473,106,902,1131,1060,243,473,253,647,230,1060,236,473,103,1257,103,1060,97,1856,109,473,103,1257,103,647,222,473,247,1257,243,30,438,198,473,240,473,108,1694,1060,103,473,253,473,97,1257,97,1856,110,902,1060,240,473,107,1856,102,1257,115,473,122,473,103,1257,103,1694,473,243,1257,243,1694,473,111,902,1060,97,1856,108,1694,473,253,473,103,902,1060,103,647,233,473,102,1060,113,473,253,1856,113,647,229,473,110,1694,1060,101,473,102,1257,109,473,122,473,243,1060,248,473,108,473,243,1257,243,473,253,1856,97,1257,97,1694,1060,113,1856,253,1257,253,1694,11,1060,240,473,107,1856,102,1257,115,473,122,473,103,902,1060,241,473,102,473,108,1694,473,243,1257,243,1694,1060,103,647,233,473,102,902,1060,103],"tokens":["âĢ","ĺ","à®","¹","à®","²","à¯","ĭ","Ġà®","ķ","à®","²","à¯","Ĩ","à®","ķ","à¯įà®","Ł","à®","°","à¯į","Ġà®","ļ","à®","¾","à®","°","à®","¾","âĢ¦","Ġà®","ļ","à®","°","à®","ķ","à¯įà®","ķ","à¯ģ","Ġà®","µ","à¯","ĩ","à®","£","à¯ģ","à®","®","à¯į","âĢ¦","Ġà®","ķ","à®","Ł","à¯","Ī","Ġà®","İ","à®","ª","à¯įà®","ª","Ġà®","¤","ிà®","±","à®","ª","à¯įà®","ª","à¯","Ģ","à®","Ļ","à¯įà®","ķ","?","âĢĻ","Ċ","à®","Ĵ","à®","°","à¯ģ","Ġà®","ª","à®","Ł","à®","¤","à¯įà®","¤","ிà®","²","à¯į","Ġà®","Ĵ","à®","¯","ிà®","©","à¯įà®","·","à®","¾","à®","ª","à¯įà®","ª","à¯ģ","à®","ķ","à¯įà®","ķ","à¯ģ","à®","³","à¯į","Ġà®","¤","ிà®","°","à¯ģ","à®","Ł","à®","ª","à¯į","Ġà®","ª","à¯","ĭ","à®","©","Ġà®","µ","à®","Ł","ிà®","µ","à¯","ĩ","à®","²","à¯ģ","Ġà®","¨","à®","©","à¯įà®","±","à®","¾","à®","ķ","Ġà®","ļ","à®","°","à®","ķ","à¯įà®","ķ","à®","Ł","ிà®","¤","à¯įà®","¤","à¯ģ","Ġà®","µ","ிà®","Ł","à¯įà®","Ł","à¯ģ",",","Ġà®","Ĵ","à®","¯","ிà®","©","à¯įà®","·","à®","¾","à®","ª","à¯į","Ġà®","ĵ","à®","©","à®","°","à¯ģ","à®","ķ","à¯įà®","ķ","à¯ģ","Ġà®","ª","à¯","ĭ","à®","©","à¯į","Ġà®","ª"],"offsets":[[0,1],[0,1],[1,2],[1,2],[2,3],[2,3],[3,4],[3,4],[4,6],[5,6],[6,7],[6,7],[7,8],[7,8],[8,9],[8,9],[9,11],[10,11],[11,12],[11,12],[12,13],[13,15],[14,15],[15,16],[15,16],[16,17],[16,17],[17,18],[17,18],[18,19],[19,21],[20,21],[21,22],[21,22],[22,23],[22,23],[23,25],[24,25],[25,26],[26,28],[27,28],[28,29],[28,29],[29,30],[29,30],[30,31],[31,32],[31,32],[32,33],[33,34],[34,36],[35,36],[36,37],[36,37],[37,38],[37,38],[38,40],[39,40],[40,41],[40,41],[41,43],[42,43],[43,45],[44,45],[45,47],[46,47],[47,48],[47,48],[48,50],[49,50],[50,51],[50,51],[51,52],[51,52],[52,54],[53,54],[54,55],[55,56],[56,57],[57,58],[57,58],[58,59],[58,59],[59,60],[60,62],[61,62],[62,63],[62,63],[63,64],[63,64],[64,66],[65,66],[66,68],[67,68],[68,69],[69,71],[70,71],[71,72],[71,72],[72,74],[73,74],[74,76],[75,76],[76,77],[76,77],[77,78],[77,78],[78,80],[79,80],[80,81],[81,82],[81,82],[82,84],[83,84],[84,85],[85,86],[85,86],[86,87],[87,89],[88,89],[89,91],[90,91],[91,92],[92,93],[92,93],[93,94],[93,94],[94,95],[95,97],[96,97],[97,98],[97,98],[98,99],[98,99],[99,101],[100,101],[101,102],[101,102],[102,104],[103,104],[104,105],[104,105],[105,106],[105,106],[106,107],[107,109],[108,109],[109,110],[109,110],[110,112],[111,112],[112,113],[112,113],[113,114],[113,114],[114,116],[115,116],[116,117],[116,117],[117,118],[117,118],[118,120],[119,120],[120,121],[120,121],[121,123],[122,123],[123,125],[124,125],[125,126],[126,128],[127,128],[128,130],[129,130],[130,132],[131,132],[132,133],[133,134],[134,136],[135,136],[136,137],[136,137],[137,139],[138,139],[139,141],[140,141],[141,142],[141,142],[142,143],[142,143],[143,144],[144,146],[145,146],[146,147],[146,147],[147,148],[147,148],[148,149],[149,150],[149,150],[150,152],[151,152],[152,153],[153,155],[154,155],[155,156],[155,156],[156,157],[156,157],[157,158],[158,160],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,3,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,6,7,7,7,7,7,7,8,8,8,8,8,8,9,9,9,9,9,9,9,9,9,9,9,9,9,9,10,10,10,11,11,11,11,11,12,12,12,12,12,12,12,12,12,12,12,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,14,14,14,14,14,14,14,14,14,14,15,15,15,15,15,15,16,16,16,16,16,16,16,16,16,16,16,17,17,17,17,17,17,17,17,17,17,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,19,19,19,19,19,19,19,20,21,21,21,21,21,21,21,21,21,21,21,21,21,22,22,22,22,22,22,22,22,22,22,22,22,23,23,23,23,23,23,23,24,24],"decoded":"‘ஹலோ கலெக்டர் சாரா… சரக்கு வேணும்… கடை எப்ப திறப்பீங்க?’\nஒரு படத்தில் ஒயின்ஷாப்புக்குள் திருடப் போன வடிவேலு நன்றாக சரக்கடித்து விட்டு, ஒயின்ஷாப் ஓனருக்கு போன் ப","decoded_with_specials":"‘ஹலோ கலெக்டர் சாரா… சரக்கு வேணும்… கடை எப்ப திறப்பீங்க?’\nஒரு படத்தில் ஒயின்ஷாப்புக்குள் திருடப் போன வடிவேலு நன்றாக சரக்கடித்து விட்டு, ஒயின்ஷாப் ஓனருக்கு போன் ப"} +{"kind":"sample","source":"fixtures/lang/tha_Thai.txt[:160]","text":"นับเจดีย์ดูนะครับว่าครบยี่สิบองค์รึเปล่า คำว่าซาว ทางเหนือแปลว่่ายี่สิบครับ ผมนับดูแล้วก็ครบนะ ลองดู เสร็จแล้วก็เข้าเยี่ยมชมพิพธภัณฑ์ เดินเข้าไปด้านในหน่อยก็จะม","ids":[1435,381,109,381,248,795,1965,230,381,242,381,113,381,95,795,234,381,242,381,117,1435,381,108,381,226,1619,381,109,381,248,381,100,2055,2084,226,1619,381,248,381,95,381,113,2055,381,103,381,112,381,248,1732,381,229,381,226,795,234,1619,381,114,795,1965,249,381,98,2055,1384,1210,226,381,111,381,100,2055,2084,233,2084,100,1210,245,2084,229,795,1965,104,1435,381,115,1732,795,223,381,249,381,98,381,100,2055,2055,2084,95,381,113,2055,381,103,381,112,381,248,381,226,1619,381,109,381,248,1210,250,381,94,1435,381,109,381,248,381,242,381,117,795,223,381,98,795,231,381,100,381,223,795,229,381,226,1619,381,248,1435,381,108,1210,98,1732,381,229,381,242,381,117,333,117,1965,103,1619,795,229,381,230,795,223,381,98,795,231,381,100,381,223,795,229,795,1965,224,795,231,1384,795,1965,95,381,113,2055,381,95,381,94,381,232,381,94,381,252,381,112,381,252,381,246,381,254,381,109,381,241,381,239,795,234,333,117,1965,242,381,112,1435,795,1965,224,795,231,1384,795,226,381,249,381,242,795,231,1384,1435,795,225,1435,381,104,1435,2055,1732,381,95,381,223,795,229,381,230,381,108,381,94],"ids_no_specials":[1435,381,109,381,248,795,1965,230,381,242,381,113,381,95,795,234,381,242,381,117,1435,381,108,381,226,1619,381,109,381,248,381,100,2055,2084,226,1619,381,248,381,95,381,113,2055,381,103,381,112,381,248,1732,381,229,381,226,795,234,1619,381,114,795,1965,249,381,98,2055,1384,1210,226,381,111,381,100,2055,2084,233,2084,100,1210,245,2084,229,795,1965,104,1435,381,115,1732,795,223,381,249,381,98,381,100,2055,2055,2084,95,381,113,2055,381,103,381,112,381,248,381,226,1619,381,109,381,248,1210,250,381,94,1435,381,109,381,248,381,242,381,117,795,223,381,98,795,231,381,100,381,223,795,229,381,226,1619,381,248,1435,381,108,1210,98,1732,381,229,381,242,381,117,333,117,1965,103,1619,795,229,381,230,795,223,381,98,795,231,381,100,381,223,795,229,795,1965,224,795,231,1384,795,1965,95,381,113,2055,381,95,381,94,381,232,381,94,381,252,381,112,381,252,381,246,381,254,381,109,381,241,381,239,795,234,333,117,1965,242,381,112,1435,795,1965,224,795,231,1384,795,226,381,249,381,242,795,231,1384,1435,795,225,1435,381,104,1435,2055,1732,381,95,381,223,795,229,381,230,381,108,381,94],"tokens":["à¸Ļ","à¸","±","à¸","ļ","à¹","Ģà¸","Ī","à¸","Ķ","à¸","µ","à¸","¢","à¹","Į","à¸","Ķ","à¸","¹","à¸Ļ","à¸","°","à¸","Ħ","ร","à¸","±","à¸","ļ","à¸","§","à¹Ī","าà¸","Ħ","ร","à¸","ļ","à¸","¢","à¸","µ","à¹Ī","à¸","ª","à¸","´","à¸","ļ","à¸Ń","à¸","ĩ","à¸","Ħ","à¹","Į","ร","à¸","¶","à¹","Ģà¸","Ľ","à¸","¥","à¹Ī","า","Ġà¸","Ħ","à¸","³","à¸","§","à¹Ī","าà¸","ĭ","าà¸","§","Ġà¸","Ĺ","าà¸","ĩ","à¹","Ģà¸","«","à¸Ļ","à¸","·","à¸Ń","à¹","ģ","à¸","Ľ","à¸","¥","à¸","§","à¹Ī","à¹Ī","าà¸","¢","à¸","µ","à¹Ī","à¸","ª","à¸","´","à¸","ļ","à¸","Ħ","ร","à¸","±","à¸","ļ","Ġà¸","ľ","à¸","¡","à¸Ļ","à¸","±","à¸","ļ","à¸","Ķ","à¸","¹","à¹","ģ","à¸","¥","à¹","ī","à¸","§","à¸","ģ","à¹","ĩ","à¸","Ħ","ร","à¸","ļ","à¸Ļ","à¸","°","Ġà¸","¥","à¸Ń","à¸","ĩ","à¸","Ķ","à¸","¹","Ġà","¹","Ģà¸","ª","ร","à¹","ĩ","à¸","Ī","à¹","ģ","à¸","¥","à¹","ī","à¸","§","à¸","ģ","à¹","ĩ","à¹","Ģà¸","Ĥ","à¹","ī","า","à¹","Ģà¸","¢","à¸","µ","à¹Ī","à¸","¢","à¸","¡","à¸","Ĭ","à¸","¡","à¸","ŀ","à¸","´","à¸","ŀ","à¸","ĺ","à¸","ł","à¸","±","à¸","ĵ","à¸","ij","à¹","Į","Ġà","¹","Ģà¸","Ķ","à¸","´","à¸Ļ","à¹","Ģà¸","Ĥ","à¹","ī","า","à¹","Ħ","à¸","Ľ","à¸","Ķ","à¹","ī","า","à¸Ļ","à¹","ĥ","à¸Ļ","à¸","«","à¸Ļ","à¹Ī","à¸Ń","à¸","¢","à¸","ģ","à¹","ĩ","à¸","Ī","à¸","°","à¸","¡"],"offsets":[[0,1],[1,2],[1,2],[2,3],[2,3],[3,4],[3,5],[4,5],[5,6],[5,6],[6,7],[6,7],[7,8],[7,8],[8,9],[8,9],[9,10],[9,10],[10,11],[10,11],[11,12],[12,13],[12,13],[13,14],[13,14],[14,15],[15,16],[15,16],[16,17],[16,17],[17,18],[17,18],[18,19],[19,21],[20,21],[21,22],[22,23],[22,23],[23,24],[23,24],[24,25],[24,25],[25,26],[26,27],[26,27],[27,28],[27,28],[28,29],[28,29],[29,30],[30,31],[30,31],[31,32],[31,32],[32,33],[32,33],[33,34],[34,35],[34,35],[35,36],[35,37],[36,37],[37,38],[37,38],[38,39],[39,40],[40,42],[41,42],[42,43],[42,43],[43,44],[43,44],[44,45],[45,47],[46,47],[47,49],[48,49],[49,51],[50,51],[51,53],[52,53],[53,54],[53,55],[54,55],[55,56],[56,57],[56,57],[57,58],[58,59],[58,59],[59,60],[59,60],[60,61],[60,61],[61,62],[61,62],[62,63],[63,64],[64,66],[65,66],[66,67],[66,67],[67,68],[68,69],[68,69],[69,70],[69,70],[70,71],[70,71],[71,72],[71,72],[72,73],[73,74],[73,74],[74,75],[74,75],[75,77],[76,77],[77,78],[77,78],[78,79],[79,80],[79,80],[80,81],[80,81],[81,82],[81,82],[82,83],[82,83],[83,84],[83,84],[84,85],[84,85],[85,86],[85,86],[86,87],[86,87],[87,88],[87,88],[88,89],[88,89],[89,90],[89,90],[90,91],[91,92],[91,92],[92,93],[93,94],[93,94],[94,96],[95,96],[96,97],[97,98],[97,98],[98,99],[98,99],[99,100],[99,100],[100,102],[101,102],[101,103],[102,103],[103,104],[104,105],[104,105],[105,106],[105,106],[106,107],[106,107],[107,108],[107,108],[108,109],[108,109],[109,110],[109,110],[110,111],[110,111],[111,112],[111,112],[112,113],[112,114],[113,114],[114,115],[114,115],[115,116],[116,117],[116,118],[117,118],[118,119],[118,119],[119,120],[120,121],[120,121],[121,122],[121,122],[122,123],[122,123],[123,124],[123,124],[124,125],[124,125],[125,126],[125,126],[126,127],[126,127],[127,128],[127,128],[128,129],[128,129],[129,130],[129,130],[130,131],[130,131],[131,132],[131,132],[132,133],[132,133],[133,135],[134,135],[134,136],[135,136],[136,137],[136,137],[137,138],[138,139],[138,140],[139,140],[140,141],[140,141],[141,142],[142,143],[142,143],[143,144],[143,144],[144,145],[144,145],[145,146],[145,146],[146,147],[147,148],[148,149],[148,149],[149,150],[150,151],[150,151],[151,152],[152,153],[153,154],[154,155],[154,155],[155,156],[155,156],[156,157],[156,157],[157,158],[157,158],[158,159],[158,159],[159,160],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6],"decoded":"นับเจดีย์ดูนะครับว่าครบยี่สิบองค์รึเปล่า คำว่าซาว ทางเหนือแปลว่่ายี่สิบครับ ผมนับดูแล้วก็ครบนะ ลองดู เสร็จแล้วก็เข้าเยี่ยมชมพิพธภัณฑ์ เดินเข้าไปด้านในหน่อยก็จะม","decoded_with_specials":"นับเจดีย์ดูนะครับว่าครบยี่สิบองค์รึเปล่า คำว่าซาว ทางเหนือแปลว่่ายี่สิบครับ ผมนับดูแล้วก็ครบนะ ลองดู เสร็จแล้วก็เข้าเยี่ยมชมพิพธภัณฑ์ เดินเข้าไปด้านในหน่อยก็จะม"} +{"kind":"sample","source":"fixtures/modalities/added_normalized_dense.txt[:160]","text":"FLIBBERJAST CRUNGLEDORF FLIBBERJAST SNORLAXIAN fast BLORPTRONIC ZORPTASTIC split WIDGETRON split stage SNORLAXIAN BLORPTRONIC BLORPTRONIC \n QUIBBLENAUT CRUNGLED","ids":[37,43,40,33,33,866,41,32,1117,363,49,52,45,38,1400,35,1301,37,454,43,40,33,33,866,41,32,1117,336,45,1301,43,32,55,40,1499,285,629,418,43,1301,47,51,49,975,2022,1489,1301,47,51,32,1117,2022,265,528,278,486,1240,38,2175,49,975,265,528,278,420,477,336,45,1301,43,32,55,40,1499,418,43,1301,47,51,49,975,2022,418,43,1301,47,51,49,975,2022,793,1486,52,40,33,33,43,1262,32,52,51,363,49,52,45,38,1400,35],"ids_no_specials":[37,43,40,33,33,866,41,32,1117,363,49,52,45,38,1400,35,1301,37,454,43,40,33,33,866,41,32,1117,336,45,1301,43,32,55,40,1499,285,629,418,43,1301,47,51,49,975,2022,1489,1301,47,51,32,1117,2022,265,528,278,486,1240,38,2175,49,975,265,528,278,420,477,336,45,1301,43,32,55,40,1499,418,43,1301,47,51,49,975,2022,418,43,1301,47,51,49,975,2022,793,1486,52,40,33,33,43,1262,32,52,51,363,49,52,45,38,1400,35],"tokens":["F","L","I","B","B","ER","J","A","ST","ĠC","R","U","N","G","LE","D","OR","F","ĠF","L","I","B","B","ER","J","A","ST","ĠS","N","OR","L","A","X","I","AN","Ġf","ast","ĠB","L","OR","P","T","R","ON","IC","ĠZ","OR","P","T","A","ST","IC","Ġs","pl","it","ĠW","ID","G","ET","R","ON","Ġs","pl","it","Ġst","age","ĠS","N","OR","L","A","X","I","AN","ĠB","L","OR","P","T","R","ON","IC","ĠB","L","OR","P","T","R","ON","IC","ĠĊ","ĠQ","U","I","B","B","L","EN","A","U","T","ĠC","R","U","N","G","LE","D"],"offsets":[[0,1],[1,2],[2,3],[3,4],[4,5],[5,7],[7,8],[8,9],[9,11],[11,13],[13,14],[14,15],[15,16],[16,17],[17,19],[19,20],[20,22],[22,23],[23,25],[25,26],[26,27],[27,28],[28,29],[29,31],[31,32],[32,33],[33,35],[35,37],[37,38],[38,40],[40,41],[41,42],[42,43],[43,44],[44,46],[46,48],[48,51],[51,53],[53,54],[54,56],[56,57],[57,58],[58,59],[59,61],[61,63],[63,65],[65,67],[67,68],[68,69],[69,70],[70,72],[72,74],[74,76],[76,78],[78,80],[80,82],[82,84],[84,85],[85,87],[87,88],[88,90],[90,92],[92,94],[94,96],[96,99],[99,102],[102,104],[104,105],[105,107],[107,108],[108,109],[109,110],[110,111],[111,113],[113,115],[115,116],[116,118],[118,119],[119,120],[120,121],[121,123],[123,125],[125,127],[127,128],[128,130],[130,131],[131,132],[132,133],[133,135],[135,137],[137,139],[139,141],[141,142],[142,143],[143,144],[144,145],[145,146],[146,148],[148,149],[149,150],[150,151],[151,153],[153,154],[154,155],[155,156],[156,157],[157,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,4,4,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,7,7,7,8,8,8,8,8,8,9,9,9,10,10,11,11,11,11,11,11,11,11,12,12,12,12,12,12,12,12,13,13,13,13,13,13,13,13,14,15,15,15,15,15,15,15,15,15,15,16,16,16,16,16,16,16],"decoded":"FLIBBERJAST CRUNGLEDORF FLIBBERJAST SNORLAXIAN fast BLORPTRONIC ZORPTASTIC split WIDGETRON split stage SNORLAXIAN BLORPTRONIC BLORPTRONIC \n QUIBBLENAUT CRUNGLED","decoded_with_specials":"FLIBBERJAST CRUNGLEDORF FLIBBERJAST SNORLAXIAN fast BLORPTRONIC ZORPTASTIC split WIDGETRON split stage SNORLAXIAN BLORPTRONIC BLORPTRONIC \n QUIBBLENAUT CRUNGLED"} +{"kind":"sample","source":"fixtures/modalities/added_normalized_sparse.txt[:160]","text":"ZORPTASTIC for we for decode CRUNGLEDORF here through ZORPTASTIC and the we WIDGETRON model \n normalize bytes and back flows text CRUNGLEDORF WUZZLEFANG chunk b","ids":[57,1301,47,51,32,1117,2022,395,581,395,1885,676,363,49,52,45,38,1400,35,1301,37,2105,1819,1489,1301,47,51,32,1117,2022,326,290,581,486,1240,38,2175,49,975,284,959,793,297,551,280,750,656,83,268,326,1602,1548,1513,2201,363,49,52,45,38,1400,35,1301,37,486,52,57,57,1400,37,1499,38,549,373,74,287],"ids_no_specials":[57,1301,47,51,32,1117,2022,395,581,395,1885,676,363,49,52,45,38,1400,35,1301,37,2105,1819,1489,1301,47,51,32,1117,2022,326,290,581,486,1240,38,2175,49,975,284,959,793,297,551,280,750,656,83,268,326,1602,1548,1513,2201,363,49,52,45,38,1400,35,1301,37,486,52,57,57,1400,37,1499,38,549,373,74,287],"tokens":["Z","OR","P","T","A","ST","IC","Ġfor","Ġwe","Ġfor","Ġdec","ode","ĠC","R","U","N","G","LE","D","OR","F","Ġhere","Ġthrough","ĠZ","OR","P","T","A","ST","IC","Ġand","Ġthe","Ġwe","ĠW","ID","G","ET","R","ON","Ġm","odel","ĠĊ","Ġn","orm","al","ize","Ġby","t","es","Ġand","Ġback","Ġfl","ows","Ġtext","ĠC","R","U","N","G","LE","D","OR","F","ĠW","U","Z","Z","LE","F","AN","G","Ġch","un","k","Ġb"],"offsets":[[0,1],[1,3],[3,4],[4,5],[5,6],[6,8],[8,10],[10,14],[14,17],[17,21],[21,25],[25,28],[28,30],[30,31],[31,32],[32,33],[33,34],[34,36],[36,37],[37,39],[39,40],[40,45],[45,53],[53,55],[55,57],[57,58],[58,59],[59,60],[60,62],[62,64],[64,68],[68,72],[72,75],[75,77],[77,79],[79,80],[80,82],[82,83],[83,85],[85,87],[87,91],[91,93],[93,95],[95,98],[98,100],[100,103],[103,106],[106,107],[107,109],[109,113],[113,118],[118,121],[121,124],[124,129],[129,131],[131,132],[132,133],[133,134],[134,135],[135,137],[137,138],[138,140],[140,141],[141,143],[143,144],[144,145],[145,146],[146,148],[148,149],[149,151],[151,152],[152,155],[155,157],[157,158],[158,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,1,2,3,4,4,5,5,5,5,5,5,5,5,5,6,7,8,8,8,8,8,8,8,9,10,11,12,12,12,12,12,12,13,13,14,15,15,15,15,16,16,16,17,18,19,19,20,21,21,21,21,21,21,21,21,21,22,22,22,22,22,22,22,22,23,23,23,24],"decoded":"ZORPTASTIC for we for decode CRUNGLEDORF here through ZORPTASTIC and the we WIDGETRON model \n normalize bytes and back flows text CRUNGLEDORF WUZZLEFANG chunk b","decoded_with_specials":"ZORPTASTIC for we for decode CRUNGLEDORF here through ZORPTASTIC and the we WIDGETRON model \n normalize bytes and back flows text CRUNGLEDORF WUZZLEFANG chunk b"} +{"kind":"sample","source":"fixtures/modalities/added_special_dense.txt[:160]","text":"fast <|xs2|> chunk <|xs1|> <|xs4|> <|xs3|> <|xs4|> modality language <|xs0|> reads <|xs2|> and and \n <|xs1|> <|xs0|> <|xs1|> <|xs4|> <|xs4|> <|xs1|> back and re","ids":[69,629,464,91,87,82,17,91,29,549,373,74,464,91,87,82,16,91,29,464,91,87,82,19,91,29,464,91,87,82,18,91,29,464,91,87,82,19,91,29,1812,280,536,305,516,84,477,464,91,87,82,15,91,29,1729,82,464,91,87,82,17,91,29,326,326,793,464,91,87,82,16,91,29,464,91,87,82,15,91,29,464,91,87,82,16,91,29,464,91,87,82,19,91,29,464,91,87,82,19,91,29,464,91,87,82,16,91,29,1602,326,322],"ids_no_specials":[69,629,464,91,87,82,17,91,29,549,373,74,464,91,87,82,16,91,29,464,91,87,82,19,91,29,464,91,87,82,18,91,29,464,91,87,82,19,91,29,1812,280,536,305,516,84,477,464,91,87,82,15,91,29,1729,82,464,91,87,82,17,91,29,326,326,793,464,91,87,82,16,91,29,464,91,87,82,15,91,29,464,91,87,82,16,91,29,464,91,87,82,19,91,29,464,91,87,82,19,91,29,464,91,87,82,16,91,29,1602,326,322],"tokens":["f","ast","Ġ<","|","x","s","2","|",">","Ġch","un","k","Ġ<","|","x","s","1","|",">","Ġ<","|","x","s","4","|",">","Ġ<","|","x","s","3","|",">","Ġ<","|","x","s","4","|",">","Ġmod","al","ity","Ġl","ang","u","age","Ġ<","|","x","s","0","|",">","Ġread","s","Ġ<","|","x","s","2","|",">","Ġand","Ġand","ĠĊ","Ġ<","|","x","s","1","|",">","Ġ<","|","x","s","0","|",">","Ġ<","|","x","s","1","|",">","Ġ<","|","x","s","4","|",">","Ġ<","|","x","s","4","|",">","Ġ<","|","x","s","1","|",">","Ġback","Ġand","Ġre"],"offsets":[[0,1],[1,4],[4,6],[6,7],[7,8],[8,9],[9,10],[10,11],[11,12],[12,15],[15,17],[17,18],[18,20],[20,21],[21,22],[22,23],[23,24],[24,25],[25,26],[26,28],[28,29],[29,30],[30,31],[31,32],[32,33],[33,34],[34,36],[36,37],[37,38],[38,39],[39,40],[40,41],[41,42],[42,44],[44,45],[45,46],[46,47],[47,48],[48,49],[49,50],[50,54],[54,56],[56,59],[59,61],[61,64],[64,65],[65,68],[68,70],[70,71],[71,72],[72,73],[73,74],[74,75],[75,76],[76,81],[81,82],[82,84],[84,85],[85,86],[86,87],[87,88],[88,89],[89,90],[90,94],[94,98],[98,100],[100,102],[102,103],[103,104],[104,105],[105,106],[106,107],[107,108],[108,110],[110,111],[111,112],[112,113],[113,114],[114,115],[115,116],[116,118],[118,119],[119,120],[120,121],[121,122],[122,123],[123,124],[124,126],[126,127],[127,128],[128,129],[129,130],[130,131],[131,132],[132,134],[134,135],[135,136],[136,137],[137,138],[138,139],[139,140],[140,142],[142,143],[143,144],[144,145],[145,146],[146,147],[147,148],[148,153],[153,157],[157,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,1,1,2,2,3,4,4,5,5,5,6,6,7,7,8,9,9,10,10,11,11,12,13,13,14,14,15,15,16,17,17,18,18,19,19,20,21,21,22,22,22,23,23,23,23,24,24,25,25,26,27,27,28,28,29,29,30,30,31,32,32,33,34,35,36,36,37,37,38,39,39,40,40,41,41,42,43,43,44,44,45,45,46,47,47,48,48,49,49,50,51,51,52,52,53,53,54,55,55,56,56,57,57,58,59,59,60,61,62],"decoded":"fast <|xs2|> chunk <|xs1|> <|xs4|> <|xs3|> <|xs4|> modality language <|xs0|> reads <|xs2|> and and \n <|xs1|> <|xs0|> <|xs1|> <|xs4|> <|xs4|> <|xs1|> back and re","decoded_with_specials":"fast <|xs2|> chunk <|xs1|> <|xs4|> <|xs3|> <|xs4|> modality language <|xs0|> reads <|xs2|> and and \n <|xs1|> <|xs0|> <|xs1|> <|xs4|> <|xs4|> <|xs1|> back and re"} +{"kind":"sample","source":"fixtures/modalities/added_special_sparse.txt[:160]","text":"<|xs0|> every for encode <|xs0|> bytes the normalize decode text model <|xs4|> <|xs3|> and \n and <|xs3|> here <|xs1|> again model here text <|xs2|> <|xs2|> lang","ids":[27,91,87,82,15,91,29,1753,395,469,66,676,464,91,87,82,15,91,29,656,83,268,290,297,551,280,750,1885,676,2201,284,959,464,91,87,82,19,91,29,464,91,87,82,18,91,29,326,793,326,464,91,87,82,18,91,29,2105,464,91,87,82,16,91,29,1017,524,284,959,2105,2201,464,91,87,82,17,91,29,464,91,87,82,17,91,29,305,516],"ids_no_specials":[27,91,87,82,15,91,29,1753,395,469,66,676,464,91,87,82,15,91,29,656,83,268,290,297,551,280,750,1885,676,2201,284,959,464,91,87,82,19,91,29,464,91,87,82,18,91,29,326,793,326,464,91,87,82,18,91,29,2105,464,91,87,82,16,91,29,1017,524,284,959,2105,2201,464,91,87,82,17,91,29,464,91,87,82,17,91,29,305,516],"tokens":["<","|","x","s","0","|",">","Ġevery","Ġfor","Ġen","c","ode","Ġ<","|","x","s","0","|",">","Ġby","t","es","Ġthe","Ġn","orm","al","ize","Ġdec","ode","Ġtext","Ġm","odel","Ġ<","|","x","s","4","|",">","Ġ<","|","x","s","3","|",">","Ġand","ĠĊ","Ġand","Ġ<","|","x","s","3","|",">","Ġhere","Ġ<","|","x","s","1","|",">","Ġag","ain","Ġm","odel","Ġhere","Ġtext","Ġ<","|","x","s","2","|",">","Ġ<","|","x","s","2","|",">","Ġl","ang"],"offsets":[[0,1],[1,2],[2,3],[3,4],[4,5],[5,6],[6,7],[7,13],[13,17],[17,20],[20,21],[21,24],[24,26],[26,27],[27,28],[28,29],[29,30],[30,31],[31,32],[32,35],[35,36],[36,38],[38,42],[42,44],[44,47],[47,49],[49,52],[52,56],[56,59],[59,64],[64,66],[66,70],[70,72],[72,73],[73,74],[74,75],[75,76],[76,77],[77,78],[78,80],[80,81],[81,82],[82,83],[83,84],[84,85],[85,86],[86,90],[90,92],[92,96],[96,98],[98,99],[99,100],[100,101],[101,102],[102,103],[103,104],[104,109],[109,111],[111,112],[112,113],[113,114],[114,115],[115,116],[116,117],[117,120],[120,123],[123,125],[125,129],[129,134],[134,139],[139,141],[141,142],[142,143],[143,144],[144,145],[145,146],[146,147],[147,149],[149,150],[150,151],[151,152],[152,153],[153,154],[154,155],[155,157],[157,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,1,1,2,3,3,4,5,6,6,6,7,7,8,8,9,10,10,11,11,11,12,13,13,13,13,14,14,15,16,16,17,17,18,18,19,20,20,21,21,22,22,23,24,24,25,26,27,28,28,29,29,30,31,31,32,33,33,34,34,35,36,36,37,37,38,38,39,40,41,41,42,42,43,44,44,45,45,46,46,47,48,48,49,49],"decoded":"<|xs0|> every for encode <|xs0|> bytes the normalize decode text model <|xs4|> <|xs3|> and \n and <|xs3|> here <|xs1|> again model here text <|xs2|> <|xs2|> lang","decoded_with_specials":"<|xs0|> every for encode <|xs0|> bytes the normalize decode text model <|xs4|> <|xs3|> and \n and <|xs3|> here <|xs1|> again model here text <|xs2|> <|xs2|> lang"} +{"kind":"sample","source":"fixtures/modalities/agentic-traces.txt[:160]","text":"Analyze this codebase and summarize what it is and how it works.\nI need to explore the codebase structure by reading the main entry points and configuration fil","ids":[2223,280,88,1547,495,274,676,65,618,326,265,2177,277,750,1412,480,382,326,1495,480,1101,82,558,40,1309,316,513,528,510,290,274,676,65,618,989,1350,627,656,1729,289,290,284,524,1121,1102,275,1120,82,326,406,1269,330,387,1620],"ids_no_specials":[2223,280,88,1547,495,274,676,65,618,326,265,2177,277,750,1412,480,382,326,1495,480,1101,82,558,40,1309,316,513,528,510,290,274,676,65,618,989,1350,627,656,1729,289,290,284,524,1121,1102,275,1120,82,326,406,1269,330,387,1620],"tokens":["An","al","y","ze","Ġthis","Ġc","ode","b","ase","Ġand","Ġs","umm","ar","ize","Ġwhat","Ġit","Ġis","Ġand","Ġhow","Ġit","Ġwork","s",".Ċ","I","Ġneed","Ġto","Ġex","pl","ore","Ġthe","Ġc","ode","b","ase","Ġstr","uct","ure","Ġby","Ġread","ing","Ġthe","Ġm","ain","Ġent","ry","Ġp","oint","s","Ġand","Ġcon","fig","ur","ation","Ġfil"],"offsets":[[0,2],[2,4],[4,5],[5,7],[7,12],[12,14],[14,17],[17,18],[18,21],[21,25],[25,27],[27,30],[30,32],[32,35],[35,40],[40,43],[43,46],[46,50],[50,54],[54,57],[57,62],[62,63],[63,65],[65,66],[66,71],[71,74],[74,77],[77,79],[79,82],[82,86],[86,88],[88,91],[91,92],[92,95],[95,99],[99,102],[102,105],[105,108],[108,113],[113,116],[116,120],[120,122],[122,125],[125,129],[129,131],[131,133],[133,137],[137,138],[138,142],[142,146],[146,149],[149,151],[151,156],[156,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,1,2,2,2,2,3,4,4,4,4,5,6,7,8,9,10,11,11,12,13,14,15,16,16,16,17,18,18,18,18,19,19,19,20,21,21,22,23,23,24,24,25,25,25,26,27,27,27,27,28],"decoded":"Analyze this codebase and summarize what it is and how it works.\nI need to explore the codebase structure by reading the main entry points and configuration fil","decoded_with_specials":"Analyze this codebase and summarize what it is and how it works.\nI need to explore the codebase structure by reading the main entry points and configuration fil"} +{"kind":"sample","source":"fixtures/modalities/agentic_swe.txt[:160]","text":"[system]\nYou are a helpful assistant that can interact with a computer to solve tasks.\n\n[user]\n\n/testbed\n\nI've uploaded a pytho","ids":[58,82,839,1592,3575,553,261,1652,1500,1680,421,493,484,665,993,588,483,261,452,772,259,316,1887,737,260,1447,82,364,58,1428,1592,27,84,528,78,324,295,1337,311,268,523,14,83,376,65,295,198,808,84,528,78,324,295,1337,311,268,523,40,6,737,337,528,78,324,295,261,275,88,404,78],"ids_no_specials":[58,82,839,1592,3575,553,261,1652,1500,1680,421,493,484,665,993,588,483,261,452,772,259,316,1887,737,260,1447,82,364,58,1428,1592,27,84,528,78,324,295,1337,311,268,523,14,83,376,65,295,198,808,84,528,78,324,295,1337,311,268,523,40,6,737,337,528,78,324,295,261,275,88,404,78],"tokens":["[","s","ystem","]Ċ","You","Ġare","Ġa","Ġhelp","ful","Ġass","ist","ant","Ġthat","Ġcan","Ġinter","act","Ġwith","Ġa","Ġcom","put","er","Ġto","Ġsol","ve","Ġt","ask","s",".ĊĊ","[","user","]Ċ","<","u","pl","o","ad","ed","_f","il","es",">Ċ","/","t","est","b","ed","Ċ","Ċ","I","'","ve","Ġu","pl","o","ad","ed","Ġa","Ġp","y","th","o"],"offsets":[[0,1],[1,2],[2,7],[7,9],[9,12],[12,16],[16,18],[18,23],[23,26],[26,30],[30,33],[33,36],[36,41],[41,45],[45,51],[51,54],[54,59],[59,61],[61,65],[65,68],[68,70],[70,73],[73,77],[77,79],[79,81],[81,84],[84,85],[85,88],[88,89],[89,93],[93,95],[95,96],[96,97],[97,99],[99,100],[100,102],[102,104],[104,106],[106,108],[108,110],[110,112],[112,113],[113,114],[114,117],[117,118],[118,120],[120,121],[121,123],[123,124],[124,126],[126,127],[127,129],[129,131],[131,133],[133,135],[135,137],[137,139],[139,140],[140,141],[141,143],[143,145],[145,147],[147,148],[148,150],[150,152],[152,154],[154,156],[156,157],[157,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,1,2,3,4,5,5,6,6,6,7,8,9,9,10,11,12,12,12,13,14,14,15,15,15,16,17,17,18,19,19,19,19,19,19,20,20,20,21,21,22,22,22,22,23,24,25,25,25,25,25,26,26,26,27,28,28,28,29,29,29,29,29,30,31,31,31,31],"decoded":"[system]\nYou are a helpful assistant that can interact with a computer to solve tasks.\n\n[user]\n\n/testbed\n\nI've uploaded a pytho","decoded_with_specials":"[system]\nYou are a helpful assistant that can interact with a computer to solve tasks.\n\n[user]\n\n/testbed\n\nI've uploaded a pytho"} +{"kind":"sample","source":"fixtures/modalities/code_mixed.txt[:160]","text":"// django-f3f9601cff03b38942e70ce8042bdfdec6002449/django/__init__.py\nfrom django.utils.version import get_version\n\nVERSION = (6, 2, 0, \"alpha\", 0)\n\n__version__","ids":[393,272,73,516,78,12,69,18,69,24,1910,16,66,608,15,18,65,18,23,24,19,17,68,22,15,400,2241,19,17,65,67,69,613,66,21,504,1494,19,24,14,67,73,516,78,14,771,258,278,771,13,5823,198,2845,272,73,516,78,13,2056,82,13,85,1977,1588,717,62,85,1977,279,53,866,50,2044,314,350,21,11,220,17,11,220,15,11,392,280,961,64,672,220,15,1029,771,85,1977,771],"ids_no_specials":[393,272,73,516,78,12,69,18,69,24,1910,16,66,608,15,18,65,18,23,24,19,17,68,22,15,400,2241,19,17,65,67,69,613,66,21,504,1494,19,24,14,67,73,516,78,14,771,258,278,771,13,5823,198,2845,272,73,516,78,13,2056,82,13,85,1977,1588,717,62,85,1977,279,53,866,50,2044,314,350,21,11,220,17,11,220,15,11,392,280,961,64,672,220,15,1029,771,85,1977,771],"tokens":["//","Ġd","j","ang","o","-","f","3","f","9","60","1","c","ff","0","3","b","3","8","9","4","2","e","7","0","ce","80","4","2","b","d","f","de","c","6","00","24","4","9","/","d","j","ang","o","/","__","in","it","__",".","py","Ċ","from","Ġd","j","ang","o",".","util","s",".","v","ersion","Ġimport","Ġget","_","v","ersion","ĊĊ","V","ER","S","ION","Ġ=","Ġ(","6",",","Ġ","2",",","Ġ","0",",","Ġ\"","al","ph","a","\",","Ġ","0",")ĊĊ","__","v","ersion","__"],"offsets":[[0,2],[2,4],[4,5],[5,8],[8,9],[9,10],[10,11],[11,12],[12,13],[13,14],[14,16],[16,17],[17,18],[18,20],[20,21],[21,22],[22,23],[23,24],[24,25],[25,26],[26,27],[27,28],[28,29],[29,30],[30,31],[31,33],[33,35],[35,36],[36,37],[37,38],[38,39],[39,40],[40,42],[42,43],[43,44],[44,46],[46,48],[48,49],[49,50],[50,51],[51,52],[52,53],[53,56],[56,57],[57,58],[58,60],[60,62],[62,64],[64,66],[66,67],[67,69],[69,70],[70,74],[74,76],[76,77],[77,80],[80,81],[81,82],[82,86],[86,87],[87,88],[88,89],[89,95],[95,102],[102,106],[106,107],[107,108],[108,114],[114,116],[116,117],[117,119],[119,120],[120,123],[123,125],[125,127],[127,128],[128,129],[129,130],[130,131],[131,132],[132,133],[133,134],[134,135],[135,137],[137,139],[139,141],[141,142],[142,144],[144,145],[145,146],[146,149],[149,151],[151,152],[152,158],[158,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,1,1,1,1,2,2,3,4,5,5,6,7,7,8,8,9,10,10,10,11,11,12,13,13,14,15,15,16,17,17,17,17,17,18,18,19,19,20,21,21,21,21,21,22,22,23,23,24,24,25,26,27,28,28,28,28,29,29,29,30,30,30,31,32,33,33,33,34,35,35,35,35,36,37,38,39,40,41,42,43,44,45,46,47,47,47,48,49,50,51,52,53,53,54],"decoded":"// django-f3f9601cff03b38942e70ce8042bdfdec6002449/django/__init__.py\nfrom django.utils.version import get_version\n\nVERSION = (6, 2, 0, \"alpha\", 0)\n\n__version__","decoded_with_specials":"// django-f3f9601cff03b38942e70ce8042bdfdec6002449/django/__init__.py\nfrom django.utils.version import get_version\n\nVERSION = (6, 2, 0, \"alpha\", 0)\n\n__version__"} +{"kind":"sample","source":"fixtures/modalities/math_latex.txt[:160]","text":"Bayes and his Theorem\n\nMy earlier post on Bayesian probability seems to have generated quite a lot of readers, so this lunchtime I thought I’d add a little bit ","ids":[33,356,268,326,1232,126,254,976,510,76,279,44,88,319,277,75,905,1926,402,418,356,268,1200,440,65,378,1763,458,347,82,316,679,2217,780,474,651,261,305,346,328,1729,409,11,813,495,305,373,332,83,642,357,325,283,70,470,357,438,67,1147,261,305,1387,282,287,278,220],"ids_no_specials":[33,356,268,326,1232,126,254,976,510,76,279,44,88,319,277,75,905,1926,402,418,356,268,1200,440,65,378,1763,458,347,82,316,679,2217,780,474,651,261,305,346,328,1729,409,11,813,495,305,373,332,83,642,357,325,283,70,470,357,438,67,1147,261,305,1387,282,287,278,220],"tokens":["B","ay","es","Ġand","Ġhis","Â","ł","The","ore","m","ĊĊ","M","y","Ġe","ar","l","ier","Ġpost","Ġon","ĠB","ay","es","ian","Ġpro","b","ab","ility","Ġse","em","s","Ġto","Ġhave","Ġgener","ated","Ġqu","ite","Ġa","Ġl","ot","Ġof","Ġread","ers",",","Ġso","Ġthis","Ġl","un","ch","t","ime","ĠI","Ġth","ou","g","ht","ĠI","âĢĻ","d","Ġadd","Ġa","Ġl","itt","le","Ġb","it","Ġ"],"offsets":[[0,1],[1,3],[3,5],[5,9],[9,13],[13,14],[13,14],[14,17],[17,20],[20,21],[21,23],[23,24],[24,25],[25,27],[27,29],[29,30],[30,33],[33,38],[38,41],[41,43],[43,45],[45,47],[47,50],[50,54],[54,55],[55,57],[57,62],[62,65],[65,67],[67,68],[68,71],[71,76],[76,82],[82,86],[86,89],[89,92],[92,94],[94,96],[96,98],[98,101],[101,106],[106,109],[109,110],[110,113],[113,118],[118,120],[120,122],[122,124],[124,125],[125,128],[128,130],[130,133],[133,135],[135,136],[136,138],[138,140],[140,141],[141,142],[142,146],[146,148],[148,150],[150,153],[153,155],[155,157],[157,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,1,2,3,3,3,3,3,4,5,5,6,6,6,6,7,8,9,9,9,9,10,10,10,10,11,11,11,12,13,14,14,15,15,16,17,17,18,19,19,20,21,22,23,23,23,23,23,24,25,25,25,25,26,27,27,28,29,30,30,30,31,31,32],"decoded":"Bayes and his Theorem\n\nMy earlier post on Bayesian probability seems to have generated quite a lot of readers, so this lunchtime I thought I’d add a little bit ","decoded_with_specials":"Bayes and his Theorem\n\nMy earlier post on Bayesian probability seems to have generated quite a lot of readers, so this lunchtime I thought I’d add a little bit "} +{"kind":"pair","text":"What is the capital of France?","pair":"Paris is the capital of France.","ids":[2073,266,382,290,274,403,278,280,328,454,81,766,30,1586,276,382,290,274,403,278,280,328,454,81,766,13],"tokens":["Wh","at","Ġis","Ġthe","Ġc","ap","it","al","Ġof","ĠF","r","ance","?","Par","is","Ġis","Ġthe","Ġc","ap","it","al","Ġof","ĠF","r","ance","."],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1],"sequence_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"offsets":[[0,2],[2,4],[4,7],[7,11],[11,13],[13,15],[15,17],[17,19],[19,22],[22,24],[24,25],[25,29],[29,30],[0,3],[3,5],[5,8],[8,12],[12,14],[14,16],[16,18],[18,20],[20,23],[23,25],[25,26],[26,30],[30,31]]} +{"kind":"pair","text":"Question in English?","pair":"Ответ на русском языке, с цифрами 123.","ids":[2231,376,294,306,457,892,75,1109,30,140,252,338,520,1041,1235,1839,442,369,369,462,1231,220,544,635,465,462,327,11,669,220,924,331,1470,714,500,331,220,899,18,13],"tokens":["Qu","est","ion","Ġin","ĠE","ng","l","ish","?","Ð","ŀ","ÑĤ","в","еÑĤ","Ġна","ĠÑĢ","Ñĥ","Ñģ","Ñģ","к","ом","Ġ","Ñı","з","Ñĭ","к","е",",","ĠÑģ","Ġ","ÑĨ","и","ÑĦ","ÑĢа","м","и","Ġ","12","3","."],"type_ids":[0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],"sequence_ids":[0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"offsets":[[0,2],[2,5],[5,8],[8,11],[11,13],[13,15],[15,16],[16,19],[19,20],[0,1],[0,1],[1,2],[2,3],[3,5],[5,8],[8,10],[10,11],[11,12],[12,13],[13,14],[14,16],[16,17],[17,18],[18,19],[19,20],[20,21],[21,22],[22,23],[23,25],[25,26],[26,27],[27,28],[28,29],[29,31],[31,32],[32,33],[33,34],[34,36],[36,37],[37,38]]} +{"kind":"digest","file":"fixtures/lang/amh_Ethi.txt","cap_chars":200000,"text_sha256":"e353d32f45b449b76fb65ee48f1522086c02a6bffa607e73ed8c875c03c84908","n_tokens":408940,"ids_sha256":"1cae81c403508193ca8ff64e63e7ea85bef0a3e022852fd6dfb74692af026b30"} +{"kind":"digest","file":"fixtures/lang/arb_Arab.txt","cap_chars":200000,"text_sha256":"529587117db0a7abaa2c8f31ab05d7b2f77e84a11068b46e26b2b37774d836c7","n_tokens":155016,"ids_sha256":"7da7f698b228a5d623b6755fdbf3041296725d617429cc36b09af2dfd4565945"} +{"kind":"digest","file":"fixtures/lang/ben_Beng.txt","cap_chars":200000,"text_sha256":"915afd7dbbea8256bdb6112b16424c83aa47dae63c5cca0ebadfc6c227d2b89f","n_tokens":237934,"ids_sha256":"609f51fb3b0a16d9cd38ae7251a5d8183b1b697505a0383e847026bfa7b60e9d"} +{"kind":"digest","file":"fixtures/lang/cmn_Hani.txt","cap_chars":200000,"text_sha256":"1273c0c093a00739be9efea7804b10a295f902359d11663306d097211934c6df","n_tokens":423151,"ids_sha256":"b66bf8ca6975f22116e617c93d2c3b2fad0130f47a7c651b10ab291ab014c33f"} +{"kind":"digest","file":"fixtures/lang/ell_Grek.txt","cap_chars":200000,"text_sha256":"158c8d3e4933091099bf12a1e21f57c7b7ff3f5381d4b25901eb1fa7ca0926f4","n_tokens":220863,"ids_sha256":"31776444c3a613bb72e48693d3a328cd47fe1ed9f3565631f7b39c5a0ec98e24"} +{"kind":"digest","file":"fixtures/lang/eng_Latn.txt","cap_chars":200000,"text_sha256":"d998cb3153e3a56c2ecdf49293062e8bc6ca43365b6c91cfed147d73b84c19fb","n_tokens":81502,"ids_sha256":"00799a5da7cc43dee695b1b12dafa093b6a5dba1ead25e79b390f4fa830c193e"} +{"kind":"digest","file":"fixtures/lang/heb_Hebr.txt","cap_chars":200000,"text_sha256":"386da8c0e0d416bffbe84b51baf895c67dbce6e0fd153be597933261794300fd","n_tokens":194493,"ids_sha256":"83007727b44d76577478f90db0e2484724caceabdd34c934e3f1bd3c230fb059"} +{"kind":"digest","file":"fixtures/lang/hin_Deva.txt","cap_chars":200000,"text_sha256":"2d114148c81035b1b63c94c7e9e805933c215d8aa9500d916923aec0a124ddde","n_tokens":190081,"ids_sha256":"c9f6a8e5316ae0aa42edfb205df3bfd87d1d2dd389f99b8158017bfa045d4a00"} +{"kind":"digest","file":"fixtures/lang/jpn_Jpan.txt","cap_chars":200000,"text_sha256":"2e55c036f500d7720d020be2db45c87a49c17d3b7e63b30bada9cef4cb6aa158","n_tokens":394999,"ids_sha256":"3ce44f9c3224bbde509b9d4ef0599a353f68bec9df93197b2d790b394daff749"} +{"kind":"digest","file":"fixtures/lang/kat_Geor.txt","cap_chars":200000,"text_sha256":"c69a1aa8fb80b41459c98c15c756eb8f39a9151468b1d5b3f9bbffe4d82eb87a","n_tokens":214781,"ids_sha256":"be74053cef24784813bc4651aaab63d3e30c127acd77c1a6c605d23d4a754f2c"} +{"kind":"digest","file":"fixtures/lang/kor_Hang.txt","cap_chars":200000,"text_sha256":"68659781c288136c1caffcb10eec9130406ae8a7b003c1e327b7e3601eba041d","n_tokens":386042,"ids_sha256":"dba68a44fe26c3d6095b616b82fb44c58edc5aff52b394ddaee847b988916f68"} +{"kind":"digest","file":"fixtures/lang/rus_Cyrl.txt","cap_chars":200000,"text_sha256":"5f967a82627afbd72c2f21c6e56183dfa5efc986ba34350c1cf9eea6c27a6132","n_tokens":142269,"ids_sha256":"9a00f62c00053dda1253aaa313a9cdf86eab281cc6d1734e45a7c2c3af176721"} +{"kind":"digest","file":"fixtures/lang/tam_Taml.txt","cap_chars":200000,"text_sha256":"bf09e1bca55aeab34162d5dc61afec7ac976b4e66eb5833ac5a037305911d165","n_tokens":266134,"ids_sha256":"eaa9772252847572a1bac0871bc3913e752c8305f01a98816b1a9efb3f273767"} +{"kind":"digest","file":"fixtures/lang/tha_Thai.txt","cap_chars":200000,"text_sha256":"25209f4750b3d3f04440f98aeff50316a118cb982462a4763106e8747172b79c","n_tokens":308172,"ids_sha256":"632a43cf18a7453efac7fb7624f54ef7926e65aceb023395a1694c4202b13c96"} +{"kind":"digest","file":"fixtures/modalities/added_normalized_dense.txt","cap_chars":200000,"text_sha256":"9d49b3cacf40962c051a09f1350a0f5dc2fc210556889ed0f341cfe0cf0af4f9","n_tokens":53448,"ids_sha256":"04bbcdbcf60e6a30fa5efb31df745759c4b7bb948fd052313c1192455179ed15"} +{"kind":"digest","file":"fixtures/modalities/added_normalized_sparse.txt","cap_chars":200000,"text_sha256":"55383f52b02a5d9686f9e4347729e5c08e61b9cff77913bd94ceb65c2fdf7fe4","n_tokens":40277,"ids_sha256":"006b9b7c4783c75e1de87fea1b543e323246afc8850a333082ecba1a9669df25"} +{"kind":"digest","file":"fixtures/modalities/added_special_dense.txt","cap_chars":200000,"text_sha256":"472f90f6f2f5803626df5d7088f8ef12984de6e88fedf67723887511d35b64b2","n_tokens":62545,"ids_sha256":"a10e4e23f8cc1e6bd70b554c4ccbd24877c1f43e45b5473b25e3cb3450a4fd1c"} +{"kind":"digest","file":"fixtures/modalities/added_special_sparse.txt","cap_chars":200000,"text_sha256":"9c52602f7c8d009b11377743f95e5bfe9053694fc6b48c3dd6c1c6dc0681ab5a","n_tokens":42285,"ids_sha256":"3d9361f3871dba89f662c19933c8b03c55ced98e391f338aca625fc794513f61"} +{"kind":"digest","file":"fixtures/modalities/agentic-traces.txt","cap_chars":200000,"text_sha256":"ae9d63c1adc49f572d9af635ac1b0421461e4d1b92c5f35ea572980ff1e2480c","n_tokens":91411,"ids_sha256":"39e14640f6aabeb911510d464148a94e1c233c26a280cef4d975752ed77f4624"} +{"kind":"digest","file":"fixtures/modalities/agentic_swe.txt","cap_chars":200000,"text_sha256":"b2d31e91ff91bb6c9a3aec3832d25acbecf9b58d5c0674e364d3b287182e7253","n_tokens":102381,"ids_sha256":"a7f8eed212d0a850e6a797aec6c1dfa6ad83679f1e4415c7621b1a4c1dffbe71"} +{"kind":"digest","file":"fixtures/modalities/code_mixed.txt","cap_chars":200000,"text_sha256":"0fa7314727726db056433d36bc1bda021803be7f1d150cae83e362a3ff41821d","n_tokens":107692,"ids_sha256":"2318f16dd1aeb0e8d1e512ef00edc50a727e389a87404f5192f043a6fc7b0711"} +{"kind":"digest","file":"fixtures/modalities/math_latex.txt","cap_chars":200000,"text_sha256":"2d17be8704f1f7b4f2174a054c0b85d6d22039e00be8e4a487927d27f3852e90","n_tokens":86248,"ids_sha256":"d14dd96df151dcb8059b8f44bbd5217894edb5ccd7e7cc71b58520ffb5a6e9f8"} diff --git a/bindings/python/tests/golden/goldens/gpt2.jsonl b/bindings/python/tests/golden/goldens/gpt2.jsonl new file mode 100644 index 000000000..1558c971e --- /dev/null +++ b/bindings/python/tests/golden/goldens/gpt2.jsonl @@ -0,0 +1,64 @@ +{"kind":"meta","model":"gpt2","tokenizer_file":"gpt2.json","tokenizers_version":"0.23.1"} +{"kind":"sample","source":"edge:empty","text":"","ids":[],"ids_no_specials":[],"tokens":[],"offsets":[],"type_ids":[],"special_tokens_mask":[],"word_ids":[],"decoded":"","decoded_with_specials":""} +{"kind":"sample","source":"edge:spaces","text":" ","ids":[220,220,220],"ids_no_specials":[220,220,220],"tokens":["Ġ","Ġ","Ġ"],"offsets":[[0,1],[1,2],[2,3]],"type_ids":[0,0,0],"special_tokens_mask":[0,0,0],"word_ids":[0,0,0],"decoded":" ","decoded_with_specials":" "} +{"kind":"sample","source":"edge:hello","text":"Hello world","ids":[15496,995],"ids_no_specials":[15496,995],"tokens":["Hello","Ġworld"],"offsets":[[0,5],[5,11]],"type_ids":[0,0],"special_tokens_mask":[0,0],"word_ids":[0,1],"decoded":"Hello world","decoded_with_specials":"Hello world"} +{"kind":"sample","source":"edge:punctuation","text":"Hello, world!! How's it going? (fine; thanks...)","ids":[15496,11,995,3228,1374,338,340,1016,30,357,38125,26,5176,23029],"ids_no_specials":[15496,11,995,3228,1374,338,340,1016,30,357,38125,26,5176,23029],"tokens":["Hello",",","Ġworld","!!","ĠHow","'s","Ġit","Ġgoing","?","Ġ(","fine",";","Ġthanks","...)"],"offsets":[[0,5],[5,6],[6,12],[12,14],[14,18],[18,20],[20,23],[23,29],[29,30],[30,32],[32,36],[36,37],[37,44],[44,48]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,1,2,3,4,5,6,7,8,9,10,11,12,13],"decoded":"Hello, world!! How's it going? (fine; thanks...)","decoded_with_specials":"Hello, world!! How's it going? (fine; thanks...)"} +{"kind":"sample","source":"edge:whitespace-mix","text":"line one\nline two\r\n\tindented\n trailing ","ids":[1370,530,198,1370,734,201,198,197,521,4714,198,220,25462,220,220],"ids_no_specials":[1370,530,198,1370,734,201,198,197,521,4714,198,220,25462,220,220],"tokens":["line","Ġone","Ċ","line","Ġtwo","č","Ċ","ĉ","ind","ented","Ċ","Ġ","Ġtrailing","Ġ","Ġ"],"offsets":[[0,4],[4,8],[8,9],[9,13],[13,17],[17,18],[18,19],[19,20],[20,23],[23,28],[28,29],[29,30],[30,39],[39,40],[40,41]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,1,2,3,4,5,5,6,7,7,8,8,9,10,10],"decoded":"line one\nline two\r\n\tindented\n trailing ","decoded_with_specials":"line one\nline two\r\n\tindented\n trailing "} +{"kind":"sample","source":"edge:accents","text":"café naïve résumé — déjà vu","ids":[66,1878,2634,41492,40560,16345,2634,851,39073,73,24247,410,84],"ids_no_specials":[66,1878,2634,41492,40560,16345,2634,851,39073,73,24247,410,84],"tokens":["c","af","é","Ġnaïve","Ġré","sum","é","ĠâĢĶ","Ġdé","j","Ãł","Ġv","u"],"offsets":[[0,1],[1,3],[3,4],[4,10],[10,13],[13,16],[16,17],[17,19],[19,22],[22,23],[23,24],[24,26],[26,27]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,1,2,2,2,3,4,4,4,5,5],"decoded":"café naïve résumé — déjà vu","decoded_with_specials":"café naïve résumé — déjà vu"} +{"kind":"sample","source":"edge:accents-decomposed","text":"café naïve résumé","ids":[66,8635,136,223,299,1872,136,230,303,302,136,223,82,2454,136,223],"ids_no_specials":[66,8635,136,223,299,1872,136,230,303,302,136,223,82,2454,136,223],"tokens":["c","afe","Ì","ģ","Ġn","ai","Ì","Ī","ve","Ġre","Ì","ģ","s","ume","Ì","ģ"],"offsets":[[0,1],[1,4],[4,5],[4,5],[5,7],[7,9],[9,10],[9,10],[10,12],[12,15],[15,16],[15,16],[16,17],[17,20],[20,21],[20,21]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,1,1,2,2,3,3,4,5,6,6,7,7,8,8],"decoded":"café naïve résumé","decoded_with_specials":"café naïve résumé"} +{"kind":"sample","source":"edge:emoji","text":"🤗 emoji, families 👨‍👩‍👧‍👦, flags 🇫🇷 and skin tones 👍🏽","ids":[8582,97,245,44805,11,4172,50169,101,447,235,41840,102,447,235,41840,100,447,235,41840,99,11,9701,12520,229,104,8582,229,115,290,4168,23755,50169,235,8582,237,121],"ids_no_specials":[8582,97,245,44805,11,4172,50169,101,447,235,41840,102,447,235,41840,100,447,235,41840,99,11,9701,12520,229,104,8582,229,115,290,4168,23755,50169,235,8582,237,121],"tokens":["ðŁ","¤","Ĺ","Ġemoji",",","Ġfamilies","ĠðŁij","¨","âĢ","į","ðŁij","©","âĢ","į","ðŁij","§","âĢ","į","ðŁij","¦",",","Ġflags","ĠðŁ","ĩ","«","ðŁ","ĩ","·","Ġand","Ġskin","Ġtones","ĠðŁij","į","ðŁ","ı","½"],"offsets":[[0,1],[0,1],[0,1],[1,7],[7,8],[8,17],[17,19],[18,19],[19,20],[19,20],[20,21],[20,21],[21,22],[21,22],[22,23],[22,23],[23,24],[23,24],[24,25],[24,25],[25,26],[26,32],[32,34],[33,34],[33,34],[34,35],[34,35],[34,35],[35,39],[39,44],[44,50],[50,52],[51,52],[52,53],[52,53],[52,53]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,1,2,3,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,5,6,6,6,6,6,6,7,8,9,10,10,10,10,10],"decoded":"🤗 emoji, families 👨‍👩‍👧‍👦, flags 🇫🇷 and skin tones 👍🏽","decoded_with_specials":"🤗 emoji, families 👨‍👩‍👧‍👦, flags 🇫🇷 and skin tones 👍🏽"} +{"kind":"sample","source":"edge:cjk","text":"漢字とひらがなとカタカナが混ざった文章です。","ids":[162,120,95,27764,245,30201,2515,110,36853,35585,26945,30201,21763,23376,21763,26229,35585,162,115,115,2515,244,33180,25224,23877,229,44165,254,30640,33623,16764],"ids_no_specials":[162,120,95,27764,245,30201,2515,110,36853,35585,26945,30201,21763,23376,21763,26229,35585,162,115,115,2515,244,33180,25224,23877,229,44165,254,30640,33623,16764],"tokens":["æ","¼","¢","åŃ","Ĺ","ãģ¨","ãģ","²","ãĤī","ãģĮ","ãģª","ãģ¨","ãĤ«","ãĤ¿","ãĤ«","ãĥĬ","ãģĮ","æ","·","·","ãģ","ĸ","ãģ£","ãģŁ","æĸ","ĩ","ç«","ł","ãģ§","ãģĻ","ãĢĤ"],"offsets":[[0,1],[0,1],[0,1],[1,2],[1,2],[2,3],[3,4],[3,4],[4,5],[5,6],[6,7],[7,8],[8,9],[9,10],[10,11],[11,12],[12,13],[13,14],[13,14],[13,14],[14,15],[14,15],[15,16],[16,17],[17,18],[17,18],[18,19],[18,19],[19,20],[20,21],[21,22]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"decoded":"漢字とひらがなとカタカナが混ざった文章です。","decoded_with_specials":"漢字とひらがなとカタカナが混ざった文章です。"} +{"kind":"sample","source":"edge:korean","text":"한국어 텍스트 조각","ids":[47991,250,166,113,255,168,244,112,220,169,227,235,168,232,97,169,232,116,23821,94,108,166,108,223],"ids_no_specials":[47991,250,166,113,255,168,244,112,220,169,227,235,168,232,97,169,232,116,23821,94,108,166,108,223],"tokens":["íķ","ľ","ê","µ","Ń","ì","ĸ","´","Ġ","í","ħ","į","ì","Ĭ","¤","í","Ĭ","¸","Ġì","¡","°","ê","°","ģ"],"offsets":[[0,1],[0,1],[1,2],[1,2],[1,2],[2,3],[2,3],[2,3],[3,4],[4,5],[4,5],[4,5],[5,6],[5,6],[5,6],[6,7],[6,7],[6,7],[7,9],[8,9],[8,9],[9,10],[9,10],[9,10]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2],"decoded":"한국어 텍스트 조각","decoded_with_specials":"한국어 텍스트 조각"} +{"kind":"sample","source":"edge:rtl","text":"مرحبا بالعالم — שלום עולם","ids":[25405,26897,148,255,39848,12919,17550,101,23525,44690,23525,25405,851,14360,102,40010,27072,147,251,14360,95,27072,40010,147,251],"ids_no_specials":[25405,26897,148,255,39848,12919,17550,101,23525,44690,23525,25405,851,14360,102,40010,27072,147,251,14360,95,27072,40010,147,251],"tokens":["Ùħ","ر","Ø","Ń","ب","ا","ĠØ","¨","اÙĦ","ع","اÙĦ","Ùħ","ĠâĢĶ","Ġ×","©","׾","×ķ","×","Ŀ","Ġ×","¢","×ķ","׾","×","Ŀ"],"offsets":[[0,1],[1,2],[2,3],[2,3],[3,4],[4,5],[5,7],[6,7],[7,9],[9,10],[10,12],[12,13],[13,15],[15,17],[16,17],[17,18],[18,19],[19,20],[19,20],[20,22],[21,22],[22,23],[23,24],[24,25],[24,25]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,1,1,1,1,1,1,2,3,3,3,3,3,3,4,4,4,4,4,4],"decoded":"مرحبا بالعالم — שלום עולם","decoded_with_specials":"مرحبا بالعالم — שלום עולם"} +{"kind":"sample","source":"edge:numbers","text":"1234567890, 3.14159, 1,000,000th","ids":[10163,2231,30924,3829,11,513,13,1415,19707,11,352,11,830,11,830,400],"ids_no_specials":[10163,2231,30924,3829,11,513,13,1415,19707,11,352,11,830,11,830,400],"tokens":["123","45","678","90",",","Ġ3",".","14","159",",","Ġ1",",","000",",","000","th"],"offsets":[[0,3],[3,5],[5,8],[8,10],[10,11],[11,13],[13,14],[14,16],[16,19],[19,20],[20,22],[22,23],[23,26],[26,27],[27,30],[30,32]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,1,2,3,4,4,5,6,7,8,9,10,11],"decoded":"1234567890, 3.14159, 1,000,000th","decoded_with_specials":"1234567890, 3.14159, 1,000,000th"} +{"kind":"sample","source":"edge:code","text":"def f(x):\n return x**2 # squared\nprint(f'{f(3)=}')","ids":[4299,277,7,87,2599,198,220,220,220,1441,2124,1174,17,220,1303,44345,198,4798,7,69,6,90,69,7,18,47505,92,11537],"ids_no_specials":[4299,277,7,87,2599,198,220,220,220,1441,2124,1174,17,220,1303,44345,198,4798,7,69,6,90,69,7,18,47505,92,11537],"tokens":["def","Ġf","(","x","):","Ċ","Ġ","Ġ","Ġ","Ġreturn","Ġx","**","2","Ġ","Ġ#","Ġsquared","Ċ","print","(","f","'","{","f","(","3",")=","}","')"],"offsets":[[0,3],[3,5],[5,6],[6,7],[7,9],[9,10],[10,11],[11,12],[12,13],[13,20],[20,22],[22,24],[24,25],[25,26],[26,28],[28,36],[36,37],[37,42],[42,43],[43,44],[44,45],[45,46],[46,47],[47,48],[48,49],[49,51],[51,52],[52,54]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,1,2,3,4,5,5,5,5,6,7,8,9,10,11,12,13,14,15,16,17,17,18,19,20,21,21,21],"decoded":"def f(x):\n return x**2 # squared\nprint(f'{f(3)=}')","decoded_with_specials":"def f(x):\n return x**2 # squared\nprint(f'{f(3)=}')"} +{"kind":"sample","source":"edge:long-word","text":"Donaudampfschifffahrtsgesellschaftskapitänsmützenabzeichen","ids":[3987,3885,696,9501,354,361,487,993,81,912,3212,19187,11693,701,8135,499,270,11033,77,5796,9116,83,4801,397,2736,41437],"ids_no_specials":[3987,3885,696,9501,354,361,487,993,81,912,3212,19187,11693,701,8135,499,270,11033,77,5796,9116,83,4801,397,2736,41437],"tokens":["Don","aud","amp","fs","ch","if","ff","ah","r","ts","ges","ells","cha","ft","sk","ap","it","ä","n","sm","ü","t","zen","ab","ze","ichen"],"offsets":[[0,3],[3,6],[6,9],[9,11],[11,13],[13,15],[15,17],[17,19],[19,20],[20,22],[22,25],[25,29],[29,32],[32,34],[34,36],[36,38],[38,40],[40,41],[41,42],[42,44],[44,45],[45,46],[46,49],[49,51],[51,53],[53,58]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":"Donaudampfschifffahrtsgesellschaftskapitänsmützenabzeichen","decoded_with_specials":"Donaudampfschifffahrtsgesellschaftskapitänsmützenabzeichen"} +{"kind":"sample","source":"edge:url-email","text":"https://example.com/a/b?q=1&r=2#frag user.name+tag@example.co.uk","ids":[5450,1378,20688,13,785,14,64,14,65,30,80,28,16,5,81,28,17,2,8310,363,2836,13,3672,10,12985,31,20688,13,1073,13,2724],"ids_no_specials":[5450,1378,20688,13,785,14,64,14,65,30,80,28,16,5,81,28,17,2,8310,363,2836,13,3672,10,12985,31,20688,13,1073,13,2724],"tokens":["https","://","example",".","com","/","a","/","b","?","q","=","1","&","r","=","2","#","fr","ag","Ġuser",".","name","+","tag","@","example",".","co",".","uk"],"offsets":[[0,5],[5,8],[8,15],[15,16],[16,19],[19,20],[20,21],[21,22],[22,23],[23,24],[24,25],[25,26],[26,27],[27,28],[28,29],[29,30],[30,31],[31,32],[32,34],[34,36],[36,41],[41,42],[42,46],[46,47],[47,50],[50,51],[51,58],[58,59],[59,61],[61,62],[62,64]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,18,19,20,21,22,23,24,25,26,27,28,29],"decoded":"https://example.com/a/b?q=1&r=2#frag user.name+tag@example.co.uk","decoded_with_specials":"https://example.com/a/b?q=1&r=2#frag user.name+tag@example.co.uk"} +{"kind":"sample","source":"edge:unicode-spaces","text":"a b c　d","ids":[64,1849,65,447,231,66,5099,222,67],"ids_no_specials":[64,1849,65,447,231,66,5099,222,67],"tokens":["a","Âł","b","âĢ","ī","c","ãĢ","Ģ","d"],"offsets":[[0,1],[1,2],[2,3],[3,4],[3,4],[4,5],[5,6],[5,6],[6,7]],"type_ids":[0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0],"word_ids":[0,1,2,3,3,4,5,5,6],"decoded":"a b c　d","decoded_with_specials":"a b c　d"} +{"kind":"sample","source":"edge:math","text":"∑ᵢ xᵢ² ≤ ∫₀^∞ e⁻ˣ dx ≈ 1","ids":[24861,239,39611,95,2124,39611,95,31185,41305,18872,104,158,224,222,61,24861,252,304,46256,119,135,96,44332,15139,230,352],"ids_no_specials":[24861,239,39611,95,2124,39611,95,31185,41305,18872,104,158,224,222,61,24861,252,304,46256,119,135,96,44332,15139,230,352],"tokens":["âĪ","ij","áµ","¢","Ġx","áµ","¢","²","Ġâī¤","ĠâĪ","«","â","Ĥ","Ģ","^","âĪ","ŀ","Ġe","âģ","»","Ë","£","Ġdx","Ġâī","Ī","Ġ1"],"offsets":[[0,1],[0,1],[1,2],[1,2],[2,4],[4,5],[4,5],[5,6],[6,8],[8,10],[9,10],[10,11],[10,11],[10,11],[11,12],[12,13],[12,13],[13,15],[15,16],[15,16],[16,17],[16,17],[17,20],[20,22],[21,22],[22,24]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,1,1,2,2,2,3,4,5,5,6,6,6,7,7,7,8,9,9,10,10,11,12,12,13],"decoded":"∑ᵢ xᵢ² ≤ ∫₀^∞ e⁻ˣ dx ≈ 1","decoded_with_specials":"∑ᵢ xᵢ² ≤ ∫₀^∞ e⁻ˣ dx ≈ 1"} +{"kind":"sample","source":"fixtures/lang/amh_Ethi.txt[:160]","text":"- ፍርድ ቤቱ በአቶ ዮናታን ተስፋዬ ላይ የ6 አመት ከ6 ወር ጽኑ የእስር ቅጣት አስተላለፈ\n- አንድነት ዶክተር ቴድሮስ ለዓለም ጤና ድርጅት ዋና ዳይሬክተርነት በመመረጣቸው ደስታውን ገለጸ\n- ህንድ መጠነ ሰፊ የድንጋይ ከሰል ሃይል ጣብያዎች የመገንባት እ","ids":[12,28053,235,235,157,230,255,157,233,113,28053,231,97,157,231,109,28053,231,254,157,232,254,157,231,114,28053,233,106,157,232,241,157,231,111,157,232,243,28053,231,108,157,230,113,157,235,233,157,233,105,28053,230,233,157,233,255,28053,233,101,21,28053,232,254,157,230,246,157,231,113,28053,232,101,21,28053,233,230,157,230,255,28053,234,121,157,232,239,28053,233,101,157,232,98,157,230,113,157,230,255,28053,231,227,157,234,96,157,231,113,28053,232,254,157,230,113,157,231,108,157,230,233,157,230,230,157,235,230,198,12,28053,232,254,157,232,243,157,233,113,157,232,238,157,231,113,28053,233,114,157,232,255,157,231,108,157,230,255,28053,231,112,157,233,113,157,230,106,157,230,113,28053,230,230,157,233,241,157,230,230,157,230,251,28053,234,97,157,232,241,28053,233,113,157,230,255,157,234,227,157,231,113,28053,233,233,157,232,241,28053,233,111,157,233,255,157,230,105,157,232,255,157,231,108,157,230,255,157,232,238,157,231,113,28053,231,254,157,230,246,157,230,246,157,230,101,157,234,96,157,231,116,157,233,235,28053,233,108,157,230,113,157,231,111,157,233,235,157,232,243,28053,234,230,157,230,230,157,234,116,198,12,28053,230,227,157,232,243,157,233,113,28053,230,246,157,234,254,157,232,238,28053,230,108,157,235,232,28053,233,101,157,233,113,157,232,243,157,234,233,157,233,255,28053,232,101,157,230,108,157,230,235,28053,230,225,157,233,255,157,230,235,28053,234,96,157,231,98,157,233,104,157,233,236,157,231,121,28053,233,101,157,230,246,157,234,230,157,232,243,157,231,96,157,231,113,28053,232,98],"ids_no_specials":[12,28053,235,235,157,230,255,157,233,113,28053,231,97,157,231,109,28053,231,254,157,232,254,157,231,114,28053,233,106,157,232,241,157,231,111,157,232,243,28053,231,108,157,230,113,157,235,233,157,233,105,28053,230,233,157,233,255,28053,233,101,21,28053,232,254,157,230,246,157,231,113,28053,232,101,21,28053,233,230,157,230,255,28053,234,121,157,232,239,28053,233,101,157,232,98,157,230,113,157,230,255,28053,231,227,157,234,96,157,231,113,28053,232,254,157,230,113,157,231,108,157,230,233,157,230,230,157,235,230,198,12,28053,232,254,157,232,243,157,233,113,157,232,238,157,231,113,28053,233,114,157,232,255,157,231,108,157,230,255,28053,231,112,157,233,113,157,230,106,157,230,113,28053,230,230,157,233,241,157,230,230,157,230,251,28053,234,97,157,232,241,28053,233,113,157,230,255,157,234,227,157,231,113,28053,233,233,157,232,241,28053,233,111,157,233,255,157,230,105,157,232,255,157,231,108,157,230,255,157,232,238,157,231,113,28053,231,254,157,230,246,157,230,246,157,230,101,157,234,96,157,231,116,157,233,235,28053,233,108,157,230,113,157,231,111,157,233,235,157,232,243,28053,234,230,157,230,230,157,234,116,198,12,28053,230,227,157,232,243,157,233,113,28053,230,246,157,234,254,157,232,238,28053,230,108,157,235,232,28053,233,101,157,233,113,157,232,243,157,234,233,157,233,255,28053,232,101,157,230,108,157,230,235,28053,230,225,157,233,255,157,230,235,28053,234,96,157,231,98,157,233,104,157,233,236,157,231,121,28053,233,101,157,230,246,157,234,230,157,232,243,157,231,96,157,231,113,28053,232,98],"tokens":["-","Ġá","į","į","á","Ī","Ń","á","ĭ","µ","Ġá","ī","¤","á","ī","±","Ġá","ī","ł","á","Ĭ","ł","á","ī","¶","Ġá","ĭ","®","á","Ĭ","ĵ","á","ī","³","á","Ĭ","ķ","Ġá","ī","°","á","Ī","µ","á","į","ĭ","á","ĭ","¬","Ġá","Ī","ĭ","á","ĭ","Ń","Ġá","ĭ","¨","6","Ġá","Ĭ","ł","á","Ī","ĺ","á","ī","µ","Ġá","Ĭ","¨","6","Ġá","ĭ","Ī","á","Ī","Ń","Ġá","Į","½","á","Ĭ","ij","Ġá","ĭ","¨","á","Ĭ","¥","á","Ī","µ","á","Ī","Ń","Ġá","ī","ħ","á","Į","£","á","ī","µ","Ġá","Ĭ","ł","á","Ī","µ","á","ī","°","á","Ī","ĭ","á","Ī","Ī","á","į","Ī","Ċ","-","Ġá","Ĭ","ł","á","Ĭ","ķ","á","ĭ","µ","á","Ĭ","IJ","á","ī","µ","Ġá","ĭ","¶","á","Ĭ","Ń","á","ī","°","á","Ī","Ń","Ġá","ī","´","á","ĭ","µ","á","Ī","®","á","Ī","µ","Ġá","Ī","Ī","á","ĭ","ĵ","á","Ī","Ī","á","Ī","Ŀ","Ġá","Į","¤","á","Ĭ","ĵ","Ġá","ĭ","µ","á","Ī","Ń","á","Į","ħ","á","ī","µ","Ġá","ĭ","ĭ","á","Ĭ","ĵ","Ġá","ĭ","³","á","ĭ","Ń","á","Ī","¬","á","Ĭ","Ń","á","ī","°","á","Ī","Ń","á","Ĭ","IJ","á","ī","µ","Ġá","ī","ł","á","Ī","ĺ","á","Ī","ĺ","á","Ī","¨","á","Į","£","á","ī","¸","á","ĭ","į","Ġá","ĭ","°","á","Ī","µ","á","ī","³","á","ĭ","į","á","Ĭ","ķ","Ġá","Į","Ī","á","Ī","Ī","á","Į","¸","Ċ","-","Ġá","Ī","ħ","á","Ĭ","ķ","á","ĭ","µ","Ġá","Ī","ĺ","á","Į","ł","á","Ĭ","IJ","Ġá","Ī","°","á","į","Ĭ","Ġá","ĭ","¨","á","ĭ","µ","á","Ĭ","ķ","á","Į","ĭ","á","ĭ","Ń","Ġá","Ĭ","¨","á","Ī","°","á","Ī","į","Ġá","Ī","ĥ","á","ĭ","Ń","á","Ī","į","Ġá","Į","£","á","ī","¥","á","ĭ","«","á","ĭ","İ","á","ī","½","Ġá","ĭ","¨","á","Ī","ĺ","á","Į","Ī","á","Ĭ","ķ","á","ī","£","á","ī","µ","Ġá","Ĭ","¥"],"offsets":[[0,1],[1,3],[2,3],[2,3],[3,4],[3,4],[3,4],[4,5],[4,5],[4,5],[5,7],[6,7],[6,7],[7,8],[7,8],[7,8],[8,10],[9,10],[9,10],[10,11],[10,11],[10,11],[11,12],[11,12],[11,12],[12,14],[13,14],[13,14],[14,15],[14,15],[14,15],[15,16],[15,16],[15,16],[16,17],[16,17],[16,17],[17,19],[18,19],[18,19],[19,20],[19,20],[19,20],[20,21],[20,21],[20,21],[21,22],[21,22],[21,22],[22,24],[23,24],[23,24],[24,25],[24,25],[24,25],[25,27],[26,27],[26,27],[27,28],[28,30],[29,30],[29,30],[30,31],[30,31],[30,31],[31,32],[31,32],[31,32],[32,34],[33,34],[33,34],[34,35],[35,37],[36,37],[36,37],[37,38],[37,38],[37,38],[38,40],[39,40],[39,40],[40,41],[40,41],[40,41],[41,43],[42,43],[42,43],[43,44],[43,44],[43,44],[44,45],[44,45],[44,45],[45,46],[45,46],[45,46],[46,48],[47,48],[47,48],[48,49],[48,49],[48,49],[49,50],[49,50],[49,50],[50,52],[51,52],[51,52],[52,53],[52,53],[52,53],[53,54],[53,54],[53,54],[54,55],[54,55],[54,55],[55,56],[55,56],[55,56],[56,57],[56,57],[56,57],[57,58],[58,59],[59,61],[60,61],[60,61],[61,62],[61,62],[61,62],[62,63],[62,63],[62,63],[63,64],[63,64],[63,64],[64,65],[64,65],[64,65],[65,67],[66,67],[66,67],[67,68],[67,68],[67,68],[68,69],[68,69],[68,69],[69,70],[69,70],[69,70],[70,72],[71,72],[71,72],[72,73],[72,73],[72,73],[73,74],[73,74],[73,74],[74,75],[74,75],[74,75],[75,77],[76,77],[76,77],[77,78],[77,78],[77,78],[78,79],[78,79],[78,79],[79,80],[79,80],[79,80],[80,82],[81,82],[81,82],[82,83],[82,83],[82,83],[83,85],[84,85],[84,85],[85,86],[85,86],[85,86],[86,87],[86,87],[86,87],[87,88],[87,88],[87,88],[88,90],[89,90],[89,90],[90,91],[90,91],[90,91],[91,93],[92,93],[92,93],[93,94],[93,94],[93,94],[94,95],[94,95],[94,95],[95,96],[95,96],[95,96],[96,97],[96,97],[96,97],[97,98],[97,98],[97,98],[98,99],[98,99],[98,99],[99,100],[99,100],[99,100],[100,102],[101,102],[101,102],[102,103],[102,103],[102,103],[103,104],[103,104],[103,104],[104,105],[104,105],[104,105],[105,106],[105,106],[105,106],[106,107],[106,107],[106,107],[107,108],[107,108],[107,108],[108,110],[109,110],[109,110],[110,111],[110,111],[110,111],[111,112],[111,112],[111,112],[112,113],[112,113],[112,113],[113,114],[113,114],[113,114],[114,116],[115,116],[115,116],[116,117],[116,117],[116,117],[117,118],[117,118],[117,118],[118,119],[119,120],[120,122],[121,122],[121,122],[122,123],[122,123],[122,123],[123,124],[123,124],[123,124],[124,126],[125,126],[125,126],[126,127],[126,127],[126,127],[127,128],[127,128],[127,128],[128,130],[129,130],[129,130],[130,131],[130,131],[130,131],[131,133],[132,133],[132,133],[133,134],[133,134],[133,134],[134,135],[134,135],[134,135],[135,136],[135,136],[135,136],[136,137],[136,137],[136,137],[137,139],[138,139],[138,139],[139,140],[139,140],[139,140],[140,141],[140,141],[140,141],[141,143],[142,143],[142,143],[143,144],[143,144],[143,144],[144,145],[144,145],[144,145],[145,147],[146,147],[146,147],[147,148],[147,148],[147,148],[148,149],[148,149],[148,149],[149,150],[149,150],[149,150],[150,151],[150,151],[150,151],[151,153],[152,153],[152,153],[153,154],[153,154],[153,154],[154,155],[154,155],[154,155],[155,156],[155,156],[155,156],[156,157],[156,157],[156,157],[157,158],[157,158],[157,158],[158,160],[159,160],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,7,7,7,8,9,9,9,9,9,9,9,9,9,10,10,10,11,12,12,12,12,12,12,13,13,13,13,13,13,14,14,14,14,14,14,14,14,14,14,14,14,15,15,15,15,15,15,15,15,15,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,17,18,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,20,20,20,20,20,20,20,20,20,20,20,20,21,21,21,21,21,21,21,21,21,21,21,21,22,22,22,22,22,22,22,22,22,22,22,22,23,23,23,23,23,23,24,24,24,24,24,24,24,24,24,24,24,24,25,25,25,25,25,25,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,29,29,29,29,29,29,29,29,29,30,31,32,32,32,32,32,32,32,32,32,33,33,33,33,33,33,33,33,33,34,34,34,34,34,34,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,36,36,36,36,36,36,36,36,36,37,37,37,37,37,37,37,37,37,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,40,40,40],"decoded":"- ፍርድ ቤቱ በአቶ ዮናታን ተስፋዬ ላይ የ6 አመት ከ6 ወር ጽኑ የእስር ቅጣት አስተላለፈ\n- አንድነት ዶክተር ቴድሮስ ለዓለም ጤና ድርጅት ዋና ዳይሬክተርነት በመመረጣቸው ደስታውን ገለጸ\n- ህንድ መጠነ ሰፊ የድንጋይ ከሰል ሃይል ጣብያዎች የመገንባት እ","decoded_with_specials":"- ፍርድ ቤቱ በአቶ ዮናታን ተስፋዬ ላይ የ6 አመት ከ6 ወር ጽኑ የእስር ቅጣት አስተላለፈ\n- አንድነት ዶክተር ቴድሮስ ለዓለም ጤና ድርጅት ዋና ዳይሬክተርነት በመመረጣቸው ደስታውን ገለጸ\n- ህንድ መጠነ ሰፊ የድንጋይ ከሰል ሃይል ጣብያዎች የመገንባት እ"} +{"kind":"sample","source":"fixtures/lang/arb_Arab.txt[:160]","text":"[بالصور] ترقوميا : الموت يطفىء باقة ورد قيد التفتح !!\nالخليل – دوت كوم – من غسان عبد الحميد - لم يخطر في بال أهالي \"ترقوميا\" التي انخلع قلبها وهي تودع التراب جث","ids":[58,39848,23525,148,113,30335,26897,60,17550,103,26897,149,224,30335,25405,22654,12919,1058,28981,25405,30335,41486,18923,232,148,115,149,223,149,231,148,94,17550,101,12919,149,224,45632,42092,26897,38843,18923,224,22654,38843,28981,41486,149,223,41486,148,255,37867,198,23525,148,106,13862,22654,13862,784,17550,107,30335,41486,18923,225,30335,25405,784,47048,23338,17550,118,45692,12919,23338,17550,117,39848,38843,28981,148,255,25405,22654,38843,532,220,13862,25405,18923,232,148,106,148,115,26897,18923,223,22654,17550,101,23525,17550,96,29519,23525,22654,366,41486,26897,149,224,30335,25405,22654,12919,1,28981,41486,22654,220,12919,23338,148,106,13862,44690,18923,224,13862,39848,29519,12919,42092,29519,22654,17550,103,30335,38843,44690,28981,41486,26897,34247,101,17550,105,148,104],"ids_no_specials":[58,39848,23525,148,113,30335,26897,60,17550,103,26897,149,224,30335,25405,22654,12919,1058,28981,25405,30335,41486,18923,232,148,115,149,223,149,231,148,94,17550,101,12919,149,224,45632,42092,26897,38843,18923,224,22654,38843,28981,41486,149,223,41486,148,255,37867,198,23525,148,106,13862,22654,13862,784,17550,107,30335,41486,18923,225,30335,25405,784,47048,23338,17550,118,45692,12919,23338,17550,117,39848,38843,28981,148,255,25405,22654,38843,532,220,13862,25405,18923,232,148,106,148,115,26897,18923,223,22654,17550,101,23525,17550,96,29519,23525,22654,366,41486,26897,149,224,30335,25405,22654,12919,1,28981,41486,22654,220,12919,23338,148,106,13862,44690,18923,224,13862,39848,29519,12919,42092,29519,22654,17550,103,30335,38843,44690,28981,41486,26897,34247,101,17550,105,148,104],"tokens":["[","ب","اÙĦ","Ø","µ","ÙĪ","ر","]","ĠØ","ª","ر","Ù","Ĥ","ÙĪ","Ùħ","ÙĬ","ا","Ġ:","ĠاÙĦ","Ùħ","ÙĪ","ت","ĠÙ","Ĭ","Ø","·","Ù","ģ","Ù","ī","Ø","¡","ĠØ","¨","ا","Ù","Ĥ","Ø©","ĠÙĪ","ر","د","ĠÙ","Ĥ","ÙĬ","د","ĠاÙĦ","ت","Ù","ģ","ت","Ø","Ń","Ġ!!","Ċ","اÙĦ","Ø","®","ÙĦ","ÙĬ","ÙĦ","ĠâĢĵ","ĠØ","¯","ÙĪ","ت","ĠÙ","ĥ","ÙĪ","Ùħ","ĠâĢĵ","ĠÙħ","ÙĨ","ĠØ","º","س","ا","ÙĨ","ĠØ","¹","ب","د","ĠاÙĦ","Ø","Ń","Ùħ","ÙĬ","د","Ġ-","Ġ","ÙĦ","Ùħ","ĠÙ","Ĭ","Ø","®","Ø","·","ر","ĠÙ","ģ","ÙĬ","ĠØ","¨","اÙĦ","ĠØ","£","Ùĩ","اÙĦ","ÙĬ","Ġ\"","ت","ر","Ù","Ĥ","ÙĪ","Ùħ","ÙĬ","ا","\"","ĠاÙĦ","ت","ÙĬ","Ġ","ا","ÙĨ","Ø","®","ÙĦ","ع","ĠÙ","Ĥ","ÙĦ","ب","Ùĩ","ا","ĠÙĪ","Ùĩ","ÙĬ","ĠØ","ª","ÙĪ","د","ع","ĠاÙĦ","ت","ر","اØ","¨","ĠØ","¬","Ø","«"],"offsets":[[0,1],[1,2],[2,4],[4,5],[4,5],[5,6],[6,7],[7,8],[8,10],[9,10],[10,11],[11,12],[11,12],[12,13],[13,14],[14,15],[15,16],[16,18],[18,21],[21,22],[22,23],[23,24],[24,26],[25,26],[26,27],[26,27],[27,28],[27,28],[28,29],[28,29],[29,30],[29,30],[30,32],[31,32],[32,33],[33,34],[33,34],[34,35],[35,37],[37,38],[38,39],[39,41],[40,41],[41,42],[42,43],[43,46],[46,47],[47,48],[47,48],[48,49],[49,50],[49,50],[50,53],[53,54],[54,56],[56,57],[56,57],[57,58],[58,59],[59,60],[60,62],[62,64],[63,64],[64,65],[65,66],[66,68],[67,68],[68,69],[69,70],[70,72],[72,74],[74,75],[75,77],[76,77],[77,78],[78,79],[79,80],[80,82],[81,82],[82,83],[83,84],[84,87],[87,88],[87,88],[88,89],[89,90],[90,91],[91,93],[93,94],[94,95],[95,96],[96,98],[97,98],[98,99],[98,99],[99,100],[99,100],[100,101],[101,103],[102,103],[103,104],[104,106],[105,106],[106,108],[108,110],[109,110],[110,111],[111,113],[113,114],[114,116],[116,117],[117,118],[118,119],[118,119],[119,120],[120,121],[121,122],[122,123],[123,124],[124,127],[127,128],[128,129],[129,130],[130,131],[131,132],[132,133],[132,133],[133,134],[134,135],[135,137],[136,137],[137,138],[138,139],[139,140],[140,141],[141,143],[143,144],[144,145],[145,147],[146,147],[147,148],[148,149],[149,150],[150,153],[153,154],[154,155],[155,157],[156,157],[157,159],[158,159],[159,160],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,1,1,1,1,1,1,2,3,3,3,3,3,3,3,3,3,4,5,5,5,5,6,6,6,6,6,6,6,6,6,6,7,7,7,7,7,7,8,8,8,9,9,9,9,10,10,10,10,10,10,10,11,12,13,13,13,13,13,13,14,15,15,15,15,16,16,16,16,17,18,18,19,19,19,19,19,20,20,20,20,21,21,21,21,21,21,22,23,23,23,24,24,24,24,24,24,24,25,25,25,26,26,26,27,27,27,27,27,28,29,29,29,29,29,29,29,29,30,31,31,31,32,32,32,32,32,32,32,33,33,33,33,33,33,34,34,34,35,35,35,35,35,36,36,36,36,36,37,37,37,37],"decoded":"[بالصور] ترقوميا : الموت يطفىء باقة ورد قيد التفتح !!\nالخليل – دوت كوم – من غسان عبد الحميد - لم يخطر في بال أهالي \"ترقوميا\" التي انخلع قلبها وهي تودع التراب جث","decoded_with_specials":"[بالصور] ترقوميا : الموت يطفىء باقة ورد قيد التفتح !!\nالخليل – دوت كوم – من غسان عبد الحميد - لم يخطر في بال أهالي \"ترقوميا\" التي انخلع قلبها وهي تودع التراب جث"} +{"kind":"sample","source":"fixtures/lang/ben_Beng.txt[:160]","text":"গ্রহ নীহারিকা\nগ্রহ নীহারিকা (ইংরেজি ভাষায়: Planetary nebula) এক বিশেষ ধরনের গ্যাসীয় নীহারিকা। যেসব তারার ভর কম, নির্দিষ্টভাবে বলতে গেলে যেসব তারার ভর সূর্যের ","ids":[48071,245,156,100,235,48071,108,48071,117,220,48071,101,156,100,222,48071,117,48071,122,48071,108,48071,123,48071,243,48071,122,198,48071,245,156,100,235,48071,108,48071,117,220,48071,101,156,100,222,48071,117,48071,122,48071,108,48071,123,48071,243,48071,122,357,48071,229,48071,224,48071,108,156,100,229,48071,250,48071,123,220,48071,255,48071,122,48071,115,48071,122,48071,107,48071,120,25,43800,45508,4712,8,220,48071,237,48071,243,220,48071,105,48071,123,48071,114,156,100,229,48071,115,220,48071,100,48071,108,48071,101,156,100,229,48071,108,220,48071,245,156,100,235,48071,107,48071,122,48071,116,156,100,222,48071,107,48071,120,220,48071,101,156,100,222,48071,117,48071,122,48071,108,48071,123,48071,243,48071,122,24231,97,220,48071,107,156,100,229,48071,116,48071,105,220,48071,97,48071,122,48071,108,48071,122,48071,108,220,48071,255,48071,108,220,48071,243,48071,106,11,220,48071,101,48071,123,48071,108,156,100,235,48071,99,48071,123,48071,115,156,100,235,48071,253,48071,255,48071,122,48071,105,156,100,229,220,48071,105,48071,110,48071,97,156,100,229,220,48071,245,156,100,229,48071,110,156,100,229,220,48071,107,156,100,229,48071,116,48071,105,220,48071,97,48071,122,48071,108,48071,122,48071,108,220,48071,255,48071,108,220,48071,116,156,100,224,48071,108,156,100,235,48071,107,156,100,229,48071,108,220],"ids_no_specials":[48071,245,156,100,235,48071,108,48071,117,220,48071,101,156,100,222,48071,117,48071,122,48071,108,48071,123,48071,243,48071,122,198,48071,245,156,100,235,48071,108,48071,117,220,48071,101,156,100,222,48071,117,48071,122,48071,108,48071,123,48071,243,48071,122,357,48071,229,48071,224,48071,108,156,100,229,48071,250,48071,123,220,48071,255,48071,122,48071,115,48071,122,48071,107,48071,120,25,43800,45508,4712,8,220,48071,237,48071,243,220,48071,105,48071,123,48071,114,156,100,229,48071,115,220,48071,100,48071,108,48071,101,156,100,229,48071,108,220,48071,245,156,100,235,48071,107,48071,122,48071,116,156,100,222,48071,107,48071,120,220,48071,101,156,100,222,48071,117,48071,122,48071,108,48071,123,48071,243,48071,122,24231,97,220,48071,107,156,100,229,48071,116,48071,105,220,48071,97,48071,122,48071,108,48071,122,48071,108,220,48071,255,48071,108,220,48071,243,48071,106,11,220,48071,101,48071,123,48071,108,156,100,235,48071,99,48071,123,48071,115,156,100,235,48071,253,48071,255,48071,122,48071,105,156,100,229,220,48071,105,48071,110,48071,97,156,100,229,220,48071,245,156,100,229,48071,110,156,100,229,220,48071,107,156,100,229,48071,116,48071,105,220,48071,97,48071,122,48071,108,48071,122,48071,108,220,48071,255,48071,108,220,48071,116,156,100,224,48071,108,156,100,235,48071,107,156,100,229,48071,108,220],"tokens":["à¦","Ĺ","à","§","į","à¦","°","à¦","¹","Ġ","à¦","¨","à","§","Ģ","à¦","¹","à¦","¾","à¦","°","à¦","¿","à¦","ķ","à¦","¾","Ċ","à¦","Ĺ","à","§","į","à¦","°","à¦","¹","Ġ","à¦","¨","à","§","Ģ","à¦","¹","à¦","¾","à¦","°","à¦","¿","à¦","ķ","à¦","¾","Ġ(","à¦","ĩ","à¦","Ĥ","à¦","°","à","§","ĩ","à¦","ľ","à¦","¿","Ġ","à¦","Ń","à¦","¾","à¦","·","à¦","¾","à¦","¯","à¦","¼",":","ĠPlanetary","Ġneb","ula",")","Ġ","à¦","ı","à¦","ķ","Ġ","à¦","¬","à¦","¿","à¦","¶","à","§","ĩ","à¦","·","Ġ","à¦","§","à¦","°","à¦","¨","à","§","ĩ","à¦","°","Ġ","à¦","Ĺ","à","§","į","à¦","¯","à¦","¾","à¦","¸","à","§","Ģ","à¦","¯","à¦","¼","Ġ","à¦","¨","à","§","Ģ","à¦","¹","à¦","¾","à¦","°","à¦","¿","à¦","ķ","à¦","¾","à¥","¤","Ġ","à¦","¯","à","§","ĩ","à¦","¸","à¦","¬","Ġ","à¦","¤","à¦","¾","à¦","°","à¦","¾","à¦","°","Ġ","à¦","Ń","à¦","°","Ġ","à¦","ķ","à¦","®",",","Ġ","à¦","¨","à¦","¿","à¦","°","à","§","į","à¦","¦","à¦","¿","à¦","·","à","§","į","à¦","Ł","à¦","Ń","à¦","¾","à¦","¬","à","§","ĩ","Ġ","à¦","¬","à¦","²","à¦","¤","à","§","ĩ","Ġ","à¦","Ĺ","à","§","ĩ","à¦","²","à","§","ĩ","Ġ","à¦","¯","à","§","ĩ","à¦","¸","à¦","¬","Ġ","à¦","¤","à¦","¾","à¦","°","à¦","¾","à¦","°","Ġ","à¦","Ń","à¦","°","Ġ","à¦","¸","à","§","Ĥ","à¦","°","à","§","į","à¦","¯","à","§","ĩ","à¦","°","Ġ"],"offsets":[[0,1],[0,1],[1,2],[1,2],[1,2],[2,3],[2,3],[3,4],[3,4],[4,5],[5,6],[5,6],[6,7],[6,7],[6,7],[7,8],[7,8],[8,9],[8,9],[9,10],[9,10],[10,11],[10,11],[11,12],[11,12],[12,13],[12,13],[13,14],[14,15],[14,15],[15,16],[15,16],[15,16],[16,17],[16,17],[17,18],[17,18],[18,19],[19,20],[19,20],[20,21],[20,21],[20,21],[21,22],[21,22],[22,23],[22,23],[23,24],[23,24],[24,25],[24,25],[25,26],[25,26],[26,27],[26,27],[27,29],[29,30],[29,30],[30,31],[30,31],[31,32],[31,32],[32,33],[32,33],[32,33],[33,34],[33,34],[34,35],[34,35],[35,36],[36,37],[36,37],[37,38],[37,38],[38,39],[38,39],[39,40],[39,40],[40,41],[40,41],[41,42],[41,42],[42,43],[43,53],[53,57],[57,60],[60,61],[61,62],[62,63],[62,63],[63,64],[63,64],[64,65],[65,66],[65,66],[66,67],[66,67],[67,68],[67,68],[68,69],[68,69],[68,69],[69,70],[69,70],[70,71],[71,72],[71,72],[72,73],[72,73],[73,74],[73,74],[74,75],[74,75],[74,75],[75,76],[75,76],[76,77],[77,78],[77,78],[78,79],[78,79],[78,79],[79,80],[79,80],[80,81],[80,81],[81,82],[81,82],[82,83],[82,83],[82,83],[83,84],[83,84],[84,85],[84,85],[85,86],[86,87],[86,87],[87,88],[87,88],[87,88],[88,89],[88,89],[89,90],[89,90],[90,91],[90,91],[91,92],[91,92],[92,93],[92,93],[93,94],[93,94],[94,95],[94,95],[95,96],[96,97],[96,97],[97,98],[97,98],[97,98],[98,99],[98,99],[99,100],[99,100],[100,101],[101,102],[101,102],[102,103],[102,103],[103,104],[103,104],[104,105],[104,105],[105,106],[105,106],[106,107],[107,108],[107,108],[108,109],[108,109],[109,110],[110,111],[110,111],[111,112],[111,112],[112,113],[113,114],[114,115],[114,115],[115,116],[115,116],[116,117],[116,117],[117,118],[117,118],[117,118],[118,119],[118,119],[119,120],[119,120],[120,121],[120,121],[121,122],[121,122],[121,122],[122,123],[122,123],[123,124],[123,124],[124,125],[124,125],[125,126],[125,126],[126,127],[126,127],[126,127],[127,128],[128,129],[128,129],[129,130],[129,130],[130,131],[130,131],[131,132],[131,132],[131,132],[132,133],[133,134],[133,134],[134,135],[134,135],[134,135],[135,136],[135,136],[136,137],[136,137],[136,137],[137,138],[138,139],[138,139],[139,140],[139,140],[139,140],[140,141],[140,141],[141,142],[141,142],[142,143],[143,144],[143,144],[144,145],[144,145],[145,146],[145,146],[146,147],[146,147],[147,148],[147,148],[148,149],[149,150],[149,150],[150,151],[150,151],[151,152],[152,153],[152,153],[153,154],[153,154],[153,154],[154,155],[154,155],[155,156],[155,156],[155,156],[156,157],[156,157],[157,158],[157,158],[157,158],[158,159],[158,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,1,1,1,2,2,2,2,3,3,3,4,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,12,12,13,13,13,14,14,14,14,15,15,15,16,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,24,24,25,25,26,26,27,27,27,28,28,29,29,30,30,30,31,31,32,32,33,33,34,34,35,35,35,36,37,37,38,39,39,39,39,39,40,40,40,41,41,42,42,43,43,43,44,44,45,45,45,45,45,45,45,46,46,46,47,47,48,48,48,49,49,49,50,50,51,51,52,52,53,53,53,54,54,55,55,56,56,56,57,57,57,58,58,59,59,60,60,61,61,62,62,63,63,63,63,64,64,64,65,65,65,66,66,66,66,67,67,67,68,68,69,69,70,70,71,71,72,72,72,72,72,73,73,73,73,73,74,75,75,75,76,76,77,77,78,78,78,79,79,80,80,81,81,82,82,82,83,83,83,83,84,84,85,85,86,86,86,87,87,87,87,87,87,87,88,88,88,89,89,89,90,90,90,91,91,92,92,92,93,93,93,94,94,94,95,95,95,95,96,96,96,97,97,98,98,99,99,100,100,101,101,101,101,101,102,102,102,103,103,103,104,104,105,105,105,106,106,107,107,107,108,108,109],"decoded":"গ্রহ নীহারিকা\nগ্রহ নীহারিকা (ইংরেজি ভাষায়: Planetary nebula) এক বিশেষ ধরনের গ্যাসীয় নীহারিকা। যেসব তারার ভর কম, নির্দিষ্টভাবে বলতে গেলে যেসব তারার ভর সূর্যের ","decoded_with_specials":"গ্রহ নীহারিকা\nগ্রহ নীহারিকা (ইংরেজি ভাষায়: Planetary nebula) এক বিশেষ ধরনের গ্যাসীয় নীহারিকা। যেসব তারার ভর কম, নির্দিষ্টভাবে বলতে গেলে যেসব তারার ভর সূর্যের "} +{"kind":"sample","source":"fixtures/lang/cmn_Hani.txt[:160]","text":"強力建議大家未來盡量避免英航阿~~~\n班機原訂航程\n6/9 台北→香港\n6/9 香港→倫敦 (原訂23:15起飛，4:50am抵達)\n6/10 倫敦→斯德哥爾摩 (原訂7:40am起飛，11:05am抵達)\n就在第二段，英航no.25班機在香港機場兩度離開閘口又兩度返航\n第一次因有旅客身體不適 (好，不怪他，消耗時間也","ids":[28156,115,27950,249,161,119,118,164,255,108,32014,22522,114,17312,103,160,122,228,33566,94,34932,237,34402,123,17739,235,164,233,109,48958,103,165,246,123,4907,93,198,163,237,255,49960,43889,253,164,101,224,48958,103,163,101,233,198,21,14,24,10263,237,108,44293,245,39310,165,99,247,162,116,107,198,21,14,24,16268,99,247,162,116,107,39310,161,222,104,46763,99,357,43889,253,164,101,224,1954,25,1314,164,113,115,45617,249,171,120,234,19,25,1120,321,162,232,113,34402,242,8,198,21,14,940,10263,222,104,46763,99,39310,23877,107,36181,115,161,241,98,163,230,122,162,239,102,357,43889,253,164,101,224,22,25,1821,321,164,113,115,45617,249,171,120,234,1157,25,2713,321,162,232,113,34402,242,8,198,22887,109,28839,101,163,105,105,12859,234,162,106,113,171,120,234,164,233,109,48958,103,3919,13,1495,163,237,255,49960,28839,101,165,99,247,162,116,107,49960,161,254,112,17739,102,41753,99,37239,95,38461,233,38461,246,20998,96,20998,230,17739,102,41753,99,32573,242,48958,103,198,163,105,105,31660,162,105,94,32368,254,17312,231,33768,227,22522,95,164,118,104,165,104,242,38834,34402,102,357,25001,121,171,120,234,38834,45250,103,20015,244,171,120,234,162,114,230,32003,245,162,25081,38461,241,20046,253],"ids_no_specials":[28156,115,27950,249,161,119,118,164,255,108,32014,22522,114,17312,103,160,122,228,33566,94,34932,237,34402,123,17739,235,164,233,109,48958,103,165,246,123,4907,93,198,163,237,255,49960,43889,253,164,101,224,48958,103,163,101,233,198,21,14,24,10263,237,108,44293,245,39310,165,99,247,162,116,107,198,21,14,24,16268,99,247,162,116,107,39310,161,222,104,46763,99,357,43889,253,164,101,224,1954,25,1314,164,113,115,45617,249,171,120,234,19,25,1120,321,162,232,113,34402,242,8,198,21,14,940,10263,222,104,46763,99,39310,23877,107,36181,115,161,241,98,163,230,122,162,239,102,357,43889,253,164,101,224,22,25,1821,321,164,113,115,45617,249,171,120,234,1157,25,2713,321,162,232,113,34402,242,8,198,22887,109,28839,101,163,105,105,12859,234,162,106,113,171,120,234,164,233,109,48958,103,3919,13,1495,163,237,255,49960,28839,101,165,99,247,162,116,107,49960,161,254,112,17739,102,41753,99,37239,95,38461,233,38461,246,20998,96,20998,230,17739,102,41753,99,32573,242,48958,103,198,163,105,105,31660,162,105,94,32368,254,17312,231,33768,227,22522,95,164,118,104,165,104,242,38834,34402,102,357,25001,121,171,120,234,38834,45250,103,20015,244,171,120,234,162,114,230,32003,245,162,25081,38461,241,20046,253],"tokens":["å¼","·","åĬ","Ľ","å","»","º","è","Ń","°","大","å®","¶","æľ","ª","ä","¾","Ĩ","çĽ","¡","éĩ","ı","éģ","¿","åħ","į","è","ĭ","±","èĪ","ª","é","ĺ","¿","~~","~","Ċ","ç","ı","Ń","æ©Ł","åİ","Ł","è","¨","Ĥ","èĪ","ª","ç","¨","ĭ","Ċ","6","/","9","Ġå","ı","°","åĮ","Ĺ","âĨĴ","é","¦","Ļ","æ","¸","¯","Ċ","6","/","9","Ġé","¦","Ļ","æ","¸","¯","âĨĴ","å","Ģ","«","æķ","¦","Ġ(","åİ","Ł","è","¨","Ĥ","23",":","15","è","µ","·","é£","Ľ","ï","¼","Į","4",":","50","am","æ","Ĭ","µ","éģ","Ķ",")","Ċ","6","/","10","Ġå","Ģ","«","æķ","¦","âĨĴ","æĸ","¯","å¾","·","å","ĵ","¥","ç","Ī","¾","æ","ij","©","Ġ(","åİ","Ł","è","¨","Ĥ","7",":","40","am","è","µ","·","é£","Ľ","ï","¼","Į","11",":","05","am","æ","Ĭ","µ","éģ","Ķ",")","Ċ","å°","±","åľ","¨","ç","¬","¬","äº","Į","æ","®","µ","ï","¼","Į","è","ĭ","±","èĪ","ª","no",".","25","ç","ı","Ń","æ©Ł","åľ","¨","é","¦","Ļ","æ","¸","¯","æ©Ł","å","ł","´","åħ","©","åº","¦","éĽ","¢","éĸ","ĭ","éĸ","ĺ","åı","£","åı","Ī","åħ","©","åº","¦","è¿","Ķ","èĪ","ª","Ċ","ç","¬","¬","ä¸Ģ","æ","¬","¡","åĽ","ł","æľ","ī","æĹ","ħ","å®","¢","è","º","«","é","«","Ķ","ä¸į","éģ","©","Ġ(","å¥","½","ï","¼","Į","ä¸į","æĢ","ª","ä»","ĸ","ï","¼","Į","æ","¶","Ī","èĢ","Ĺ","æ","ĻĤ","éĸ","ĵ","ä¹","Ł"],"offsets":[[0,1],[0,1],[1,2],[1,2],[2,3],[2,3],[2,3],[3,4],[3,4],[3,4],[4,5],[5,6],[5,6],[6,7],[6,7],[7,8],[7,8],[7,8],[8,9],[8,9],[9,10],[9,10],[10,11],[10,11],[11,12],[11,12],[12,13],[12,13],[12,13],[13,14],[13,14],[14,15],[14,15],[14,15],[15,17],[17,18],[18,19],[19,20],[19,20],[19,20],[20,21],[21,22],[21,22],[22,23],[22,23],[22,23],[23,24],[23,24],[24,25],[24,25],[24,25],[25,26],[26,27],[27,28],[28,29],[29,31],[30,31],[30,31],[31,32],[31,32],[32,33],[33,34],[33,34],[33,34],[34,35],[34,35],[34,35],[35,36],[36,37],[37,38],[38,39],[39,41],[40,41],[40,41],[41,42],[41,42],[41,42],[42,43],[43,44],[43,44],[43,44],[44,45],[44,45],[45,47],[47,48],[47,48],[48,49],[48,49],[48,49],[49,51],[51,52],[52,54],[54,55],[54,55],[54,55],[55,56],[55,56],[56,57],[56,57],[56,57],[57,58],[58,59],[59,61],[61,63],[63,64],[63,64],[63,64],[64,65],[64,65],[65,66],[66,67],[67,68],[68,69],[69,71],[71,73],[72,73],[72,73],[73,74],[73,74],[74,75],[75,76],[75,76],[76,77],[76,77],[77,78],[77,78],[77,78],[78,79],[78,79],[78,79],[79,80],[79,80],[79,80],[80,82],[82,83],[82,83],[83,84],[83,84],[83,84],[84,85],[85,86],[86,88],[88,90],[90,91],[90,91],[90,91],[91,92],[91,92],[92,93],[92,93],[92,93],[93,95],[95,96],[96,98],[98,100],[100,101],[100,101],[100,101],[101,102],[101,102],[102,103],[103,104],[104,105],[104,105],[105,106],[105,106],[106,107],[106,107],[106,107],[107,108],[107,108],[108,109],[108,109],[108,109],[109,110],[109,110],[109,110],[110,111],[110,111],[110,111],[111,112],[111,112],[112,114],[114,115],[115,117],[117,118],[117,118],[117,118],[118,119],[119,120],[119,120],[120,121],[120,121],[120,121],[121,122],[121,122],[121,122],[122,123],[123,124],[123,124],[123,124],[124,125],[124,125],[125,126],[125,126],[126,127],[126,127],[127,128],[127,128],[128,129],[128,129],[129,130],[129,130],[130,131],[130,131],[131,132],[131,132],[132,133],[132,133],[133,134],[133,134],[134,135],[134,135],[135,136],[136,137],[136,137],[136,137],[137,138],[138,139],[138,139],[138,139],[139,140],[139,140],[140,141],[140,141],[141,142],[141,142],[142,143],[142,143],[143,144],[143,144],[143,144],[144,145],[144,145],[144,145],[145,146],[146,147],[146,147],[147,149],[149,150],[149,150],[150,151],[150,151],[150,151],[151,152],[152,153],[152,153],[153,154],[153,154],[154,155],[154,155],[154,155],[155,156],[155,156],[155,156],[156,157],[156,157],[157,158],[157,158],[158,159],[158,159],[159,160],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,5,6,7,8,8,8,8,8,9,10,10,10,10,10,10,11,12,13,14,15,15,15,15,15,15,16,17,17,17,17,17,18,19,19,19,19,19,20,21,22,23,23,23,23,23,24,24,24,25,26,27,28,28,28,28,28,28,29,30,31,32,33,34,34,34,34,34,35,36,36,36,36,36,36,36,36,36,36,36,36,36,37,38,38,38,38,38,39,40,41,42,42,42,42,42,42,43,43,43,44,45,46,47,47,47,47,47,47,48,49,50,50,50,50,50,50,50,50,50,50,50,50,51,51,51,52,52,52,52,52,52,53,54,55,55,55,55,55,55,55,55,55,55,55,55,55,55,55,55,55,55,55,55,55,55,55,55,55,55,55,55,55,55,55,55,55,55,55,55,55,55,56,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,58,59,59,60,60,60,61,61,61,61,61,62,62,62,63,63,63,63,63,63,63,63,63,63,63],"decoded":"強力建議大家未來盡量避免英航阿~~~\n班機原訂航程\n6/9 台北→香港\n6/9 香港→倫敦 (原訂23:15起飛，4:50am抵達)\n6/10 倫敦→斯德哥爾摩 (原訂7:40am起飛，11:05am抵達)\n就在第二段，英航no.25班機在香港機場兩度離開閘口又兩度返航\n第一次因有旅客身體不適 (好，不怪他，消耗時間也","decoded_with_specials":"強力建議大家未來盡量避免英航阿~~~\n班機原訂航程\n6/9 台北→香港\n6/9 香港→倫敦 (原訂23:15起飛，4:50am抵達)\n6/10 倫敦→斯德哥爾摩 (原訂7:40am起飛，11:05am抵達)\n就在第二段，英航no.25班機在香港機場兩度離開閘口又兩度返航\n第一次因有旅客身體不適 (好，不怪他，消耗時間也"} +{"kind":"sample","source":"fixtures/lang/ell_Grek.txt[:160]","text":"Θυμήσου με\nΗ πόλη προσφέρει πολλές ευκαιρίες - πήγαινε στο εμπορικό κέντρο, στο σαλόνι ομορφιάς , σε κλάμπ και διασκέδασε με την ψυχή σου.\nΔημιούργησε μαγευτικά","ids":[138,246,139,227,34703,138,106,38392,26517,139,227,18919,30950,198,138,245,18074,222,139,234,39377,138,115,18074,222,33643,26517,38392,139,228,138,255,33643,30950,29945,18074,222,26517,39377,39377,138,255,35558,7377,113,139,227,43000,17394,29945,33643,138,107,30950,35558,532,18074,222,138,106,42063,17394,29945,26180,30950,18074,225,32830,26517,7377,113,34703,46582,26517,33643,29945,43000,139,234,7377,118,138,255,26180,32830,33643,26517,11,18074,225,32830,26517,18074,225,17394,39377,139,234,26180,29945,7377,123,34703,26517,33643,139,228,29945,138,105,35558,837,18074,225,30950,7377,118,39377,138,105,34703,46582,7377,118,17394,29945,7377,112,29945,17394,38392,43000,138,255,138,112,17394,38392,30950,18919,30950,46651,138,115,26180,18074,230,139,227,139,229,138,106,18074,225,26517,139,227,13,198,138,242,138,115,34703,29945,26517,139,235,33643,42063,138,115,38392,30950,18919,17394,42063,30950,139,227,32830,29945,43000,138,105],"ids_no_specials":[138,246,139,227,34703,138,106,38392,26517,139,227,18919,30950,198,138,245,18074,222,139,234,39377,138,115,18074,222,33643,26517,38392,139,228,138,255,33643,30950,29945,18074,222,26517,39377,39377,138,255,35558,7377,113,139,227,43000,17394,29945,33643,138,107,30950,35558,532,18074,222,138,106,42063,17394,29945,26180,30950,18074,225,32830,26517,7377,113,34703,46582,26517,33643,29945,43000,139,234,7377,118,138,255,26180,32830,33643,26517,11,18074,225,32830,26517,18074,225,17394,39377,139,234,26180,29945,7377,123,34703,26517,33643,139,228,29945,138,105,35558,837,18074,225,30950,7377,118,39377,138,105,34703,46582,7377,118,17394,29945,7377,112,29945,17394,38392,43000,138,255,138,112,17394,38392,30950,18919,30950,46651,138,115,26180,18074,230,139,227,139,229,138,106,18074,225,26517,139,227,13,198,138,242,138,115,34703,29945,26517,139,235,33643,42063,138,115,38392,30950,18919,17394,42063,30950,139,227,32830,29945,43000,138,105],"tokens":["Î","ĺ","Ï","ħ","μ","Î","®","Ïĥ","ο","Ï","ħ","Ġμ","ε","Ċ","Î","Ĺ","ĠÏ","Ģ","Ï","Į","λ","Î","·","ĠÏ","Ģ","Ïģ","ο","Ïĥ","Ï","Ĩ","Î","Ń","Ïģ","ε","ι","ĠÏ","Ģ","ο","λ","λ","Î","Ń","ÏĤ","ĠÎ","µ","Ï","ħ","κ","α","ι","Ïģ","Î","¯","ε","ÏĤ","Ġ-","ĠÏ","Ģ","Î","®","γ","α","ι","ν","ε","ĠÏ","ĥ","ÏĦ","ο","ĠÎ","µ","μ","ÏĢ","ο","Ïģ","ι","κ","Ï","Į","ĠÎ","º","Î","Ń","ν","ÏĦ","Ïģ","ο",",","ĠÏ","ĥ","ÏĦ","ο","ĠÏ","ĥ","α","λ","Ï","Į","ν","ι","ĠÎ","¿","μ","ο","Ïģ","Ï","Ĩ","ι","Î","¬","ÏĤ","Ġ,","ĠÏ","ĥ","ε","ĠÎ","º","λ","Î","¬","μ","ÏĢ","ĠÎ","º","α","ι","ĠÎ","´","ι","α","Ïĥ","κ","Î","Ń","Î","´","α","Ïĥ","ε","Ġμ","ε","ĠÏĦ","Î","·","ν","ĠÏ","Ī","Ï","ħ","Ï","ĩ","Î","®","ĠÏ","ĥ","ο","Ï","ħ",".","Ċ","Î","Ķ","Î","·","μ","ι","ο","Ï","į","Ïģ","γ","Î","·","Ïĥ","ε","Ġμ","α","γ","ε","Ï","ħ","ÏĦ","ι","κ","Î","¬"],"offsets":[[0,1],[0,1],[1,2],[1,2],[2,3],[3,4],[3,4],[4,5],[5,6],[6,7],[6,7],[7,9],[9,10],[10,11],[11,12],[11,12],[12,14],[13,14],[14,15],[14,15],[15,16],[16,17],[16,17],[17,19],[18,19],[19,20],[20,21],[21,22],[22,23],[22,23],[23,24],[23,24],[24,25],[25,26],[26,27],[27,29],[28,29],[29,30],[30,31],[31,32],[32,33],[32,33],[33,34],[34,36],[35,36],[36,37],[36,37],[37,38],[38,39],[39,40],[40,41],[41,42],[41,42],[42,43],[43,44],[44,46],[46,48],[47,48],[48,49],[48,49],[49,50],[50,51],[51,52],[52,53],[53,54],[54,56],[55,56],[56,57],[57,58],[58,60],[59,60],[60,61],[61,62],[62,63],[63,64],[64,65],[65,66],[66,67],[66,67],[67,69],[68,69],[69,70],[69,70],[70,71],[71,72],[72,73],[73,74],[74,75],[75,77],[76,77],[77,78],[78,79],[79,81],[80,81],[81,82],[82,83],[83,84],[83,84],[84,85],[85,86],[86,88],[87,88],[88,89],[89,90],[90,91],[91,92],[91,92],[92,93],[93,94],[93,94],[94,95],[95,97],[97,99],[98,99],[99,100],[100,102],[101,102],[102,103],[103,104],[103,104],[104,105],[105,106],[106,108],[107,108],[108,109],[109,110],[110,112],[111,112],[112,113],[113,114],[114,115],[115,116],[116,117],[116,117],[117,118],[117,118],[118,119],[119,120],[120,121],[121,123],[123,124],[124,126],[126,127],[126,127],[127,128],[128,130],[129,130],[130,131],[130,131],[131,132],[131,132],[132,133],[132,133],[133,135],[134,135],[135,136],[136,137],[136,137],[137,138],[138,139],[139,140],[139,140],[140,141],[140,141],[141,142],[142,143],[143,144],[144,145],[144,145],[145,146],[146,147],[147,148],[147,148],[148,149],[149,150],[150,152],[152,153],[153,154],[154,155],[155,156],[155,156],[156,157],[157,158],[158,159],[159,160],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,0,0,0,0,1,1,2,3,3,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,7,7,7,7,7,8,9,9,9,9,9,9,9,9,9,10,10,10,10,11,11,11,11,11,11,11,11,11,11,12,12,12,12,12,12,12,12,13,14,14,14,14,15,15,15,15,15,15,15,15,16,16,16,16,16,16,16,16,16,16,16,17,18,18,18,19,19,19,19,19,19,19,20,20,20,20,21,21,21,21,21,21,21,21,21,21,21,21,21,22,22,23,23,23,23,24,24,24,24,24,24,24,24,25,25,25,25,25,26,27,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,29,29,29,29,29,29,29,29,29,29,29],"decoded":"Θυμήσου με\nΗ πόλη προσφέρει πολλές ευκαιρίες - πήγαινε στο εμπορικό κέντρο, στο σαλόνι ομορφιάς , σε κλάμπ και διασκέδασε με την ψυχή σου.\nΔημιούργησε μαγευτικά","decoded_with_specials":"Θυμήσου με\nΗ πόλη προσφέρει πολλές ευκαιρίες - πήγαινε στο εμπορικό κέντρο, στο σαλόνι ομορφιάς , σε κλάμπ και διασκέδασε με την ψυχή σου.\nΔημιούργησε μαγευτικά"} +{"kind":"sample","source":"fixtures/lang/eng_Latn.txt[:160]","text":"|Viewing Single Post From: Spoilers for the Week of February 11th|\n|Lil||Feb 1 2013, 09:58 AM|\nDon't care about Chloe/Taniel/Jen-Jen. Don't care about Sami, rea","ids":[91,7680,278,14206,2947,3574,25,1338,9437,364,329,262,6119,286,3945,1367,400,91,198,91,43,346,15886,15146,352,2211,11,7769,25,3365,3001,91,198,3987,470,1337,546,29476,14,51,6321,14,44875,12,44875,13,2094,470,1337,546,3409,72,11,302,64],"ids_no_specials":[91,7680,278,14206,2947,3574,25,1338,9437,364,329,262,6119,286,3945,1367,400,91,198,91,43,346,15886,15146,352,2211,11,7769,25,3365,3001,91,198,3987,470,1337,546,29476,14,51,6321,14,44875,12,44875,13,2094,470,1337,546,3409,72,11,302,64],"tokens":["|","View","ing","ĠSingle","ĠPost","ĠFrom",":","ĠSp","oil","ers","Ġfor","Ġthe","ĠWeek","Ġof","ĠFebruary","Ġ11","th","|","Ċ","|","L","il","||","Feb","Ġ1","Ġ2013",",","Ġ09",":","58","ĠAM","|","Ċ","Don","'t","Ġcare","Ġabout","ĠChloe","/","T","aniel","/","Jen","-","Jen",".","ĠDon","'t","Ġcare","Ġabout","ĠSam","i",",","Ġre","a"],"offsets":[[0,1],[1,5],[5,8],[8,15],[15,20],[20,25],[25,26],[26,29],[29,32],[32,35],[35,39],[39,43],[43,48],[48,51],[51,60],[60,63],[63,65],[65,66],[66,67],[67,68],[68,69],[69,71],[71,73],[73,76],[76,78],[78,83],[83,84],[84,87],[87,88],[88,90],[90,93],[93,94],[94,95],[95,98],[98,100],[100,105],[105,111],[111,117],[117,118],[118,119],[119,124],[124,125],[125,128],[128,129],[129,132],[132,133],[133,137],[137,139],[139,144],[144,150],[150,154],[154,155],[155,156],[156,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,1,1,2,3,4,5,6,6,6,7,8,9,10,11,12,13,14,15,16,17,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,35,36,37,38,39,40,41,42,43,44,45,45,46,47,47],"decoded":"|Viewing Single Post From: Spoilers for the Week of February 11th|\n|Lil||Feb 1 2013, 09:58 AM|\nDon't care about Chloe/Taniel/Jen-Jen. Don't care about Sami, rea","decoded_with_specials":"|Viewing Single Post From: Spoilers for the Week of February 11th|\n|Lil||Feb 1 2013, 09:58 AM|\nDon't care about Chloe/Taniel/Jen-Jen. Don't care about Sami, rea"} +{"kind":"sample","source":"fixtures/lang/heb_Hebr.txt[:160]","text":"סיכונים – אלמנט מרכזי, חיוני לפעילות מסחרית. להבין מה הסיכון הוא מאוד חשוב. החוויה האנושית עולה כי מי יודע איך לקחת סיכונים בזמן, היא לנצח גדולים. לזכור פוליטיק","ids":[147,94,33951,249,27072,147,254,33951,251,784,14360,238,40010,49168,147,254,147,246,14360,252,37778,147,249,147,244,25529,11,14360,245,25529,27072,147,254,25529,14360,250,147,97,147,95,33951,250,27072,42064,14360,252,147,94,147,245,37778,33951,103,13,14360,250,38269,49603,33951,253,14360,252,38269,14360,242,147,94,33951,249,27072,147,253,14360,242,27072,42973,14360,252,42973,27072,147,241,14360,245,50227,27072,49603,13,14360,242,147,245,27072,27072,33951,242,14360,242,42973,147,254,27072,50227,33951,103,14360,95,27072,40010,38269,14360,249,25529,14360,252,25529,14360,247,27072,147,241,147,95,14360,238,33951,248,14360,250,147,100,147,245,42064,14360,94,33951,249,27072,147,254,33951,251,14360,239,147,244,49168,147,253,11,14360,242,33951,238,14360,250,147,254,147,99,147,245,14360,240,147,241,27072,40010,33951,251,13,14360,250,147,244,147,249,27072,37778,14360,97,27072,40010,33951,246,33951,100],"ids_no_specials":[147,94,33951,249,27072,147,254,33951,251,784,14360,238,40010,49168,147,254,147,246,14360,252,37778,147,249,147,244,25529,11,14360,245,25529,27072,147,254,25529,14360,250,147,97,147,95,33951,250,27072,42064,14360,252,147,94,147,245,37778,33951,103,13,14360,250,38269,49603,33951,253,14360,252,38269,14360,242,147,94,33951,249,27072,147,253,14360,242,27072,42973,14360,252,42973,27072,147,241,14360,245,50227,27072,49603,13,14360,242,147,245,27072,27072,33951,242,14360,242,42973,147,254,27072,50227,33951,103,14360,95,27072,40010,38269,14360,249,25529,14360,252,25529,14360,247,27072,147,241,147,95,14360,238,33951,248,14360,250,147,100,147,245,42064,14360,94,33951,249,27072,147,254,33951,251,14360,239,147,244,49168,147,253,11,14360,242,33951,238,14360,250,147,254,147,99,147,245,14360,240,147,241,27072,40010,33951,251,13,14360,250,147,244,147,249,27072,37778,14360,97,27072,40010,33951,246,33951,100],"tokens":["×","¡","×Ļ×","Ľ","×ķ","×","ł","×Ļ×","Ŀ","ĠâĢĵ","Ġ×","IJ","׾","×ŀ","×","ł","×","ĺ","Ġ×","ŀ","ר","×","Ľ","×","ĸ","×Ļ",",","Ġ×","Ĺ","×Ļ","×ķ","×","ł","×Ļ","Ġ×","ľ","×","¤","×","¢","×Ļ×","ľ","×ķ","ת","Ġ×","ŀ","×","¡","×","Ĺ","ר","×Ļ×","ª",".","Ġ×","ľ","×Ķ","×ij","×Ļ×","Ł","Ġ×","ŀ","×Ķ","Ġ×","Ķ","×","¡","×Ļ×","Ľ","×ķ","×","Ł","Ġ×","Ķ","×ķ","×IJ","Ġ×","ŀ","×IJ","×ķ","×","ĵ","Ġ×","Ĺ","ש","×ķ","×ij",".","Ġ×","Ķ","×","Ĺ","×ķ","×ķ","×Ļ×","Ķ","Ġ×","Ķ","×IJ","×","ł","×ķ","ש","×Ļ×","ª","Ġ×","¢","×ķ","׾","×Ķ","Ġ×","Ľ","×Ļ","Ġ×","ŀ","×Ļ","Ġ×","Ļ","×ķ","×","ĵ","×","¢","Ġ×","IJ","×Ļ×","ļ","Ġ×","ľ","×","§","×","Ĺ","ת","Ġ×","¡","×Ļ×","Ľ","×ķ","×","ł","×Ļ×","Ŀ","Ġ×","ij","×","ĸ","×ŀ","×","Ł",",","Ġ×","Ķ","×Ļ×","IJ","Ġ×","ľ","×","ł","×","¦","×","Ĺ","Ġ×","Ĵ","×","ĵ","×ķ","׾","×Ļ×","Ŀ",".","Ġ×","ľ","×","ĸ","×","Ľ","×ķ","ר","Ġ×","¤","×ķ","׾","×Ļ×","ĺ","×Ļ×","§"],"offsets":[[0,1],[0,1],[1,3],[2,3],[3,4],[4,5],[4,5],[5,7],[6,7],[7,9],[9,11],[10,11],[11,12],[12,13],[13,14],[13,14],[14,15],[14,15],[15,17],[16,17],[17,18],[18,19],[18,19],[19,20],[19,20],[20,21],[21,22],[22,24],[23,24],[24,25],[25,26],[26,27],[26,27],[27,28],[28,30],[29,30],[30,31],[30,31],[31,32],[31,32],[32,34],[33,34],[34,35],[35,36],[36,38],[37,38],[38,39],[38,39],[39,40],[39,40],[40,41],[41,43],[42,43],[43,44],[44,46],[45,46],[46,47],[47,48],[48,50],[49,50],[50,52],[51,52],[52,53],[53,55],[54,55],[55,56],[55,56],[56,58],[57,58],[58,59],[59,60],[59,60],[60,62],[61,62],[62,63],[63,64],[64,66],[65,66],[66,67],[67,68],[68,69],[68,69],[69,71],[70,71],[71,72],[72,73],[73,74],[74,75],[75,77],[76,77],[77,78],[77,78],[78,79],[79,80],[80,82],[81,82],[82,84],[83,84],[84,85],[85,86],[85,86],[86,87],[87,88],[88,90],[89,90],[90,92],[91,92],[92,93],[93,94],[94,95],[95,97],[96,97],[97,98],[98,100],[99,100],[100,101],[101,103],[102,103],[103,104],[104,105],[104,105],[105,106],[105,106],[106,108],[107,108],[108,110],[109,110],[110,112],[111,112],[112,113],[112,113],[113,114],[113,114],[114,115],[115,117],[116,117],[117,119],[118,119],[119,120],[120,121],[120,121],[121,123],[122,123],[123,125],[124,125],[125,126],[125,126],[126,127],[127,128],[127,128],[128,129],[129,131],[130,131],[131,133],[132,133],[133,135],[134,135],[135,136],[135,136],[136,137],[136,137],[137,138],[137,138],[138,140],[139,140],[140,141],[140,141],[141,142],[142,143],[143,145],[144,145],[145,146],[146,148],[147,148],[148,149],[148,149],[149,150],[149,150],[150,151],[151,152],[152,154],[153,154],[154,155],[155,156],[156,158],[157,158],[158,160],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,0,0,1,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,4,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,7,7,8,9,9,9,9,9,9,10,10,10,11,11,11,11,11,11,11,11,11,12,12,12,12,13,13,13,13,13,13,14,14,14,14,14,15,16,16,16,16,16,16,16,16,17,17,17,17,17,17,17,17,17,18,18,18,18,18,19,19,19,20,20,20,21,21,21,21,21,21,21,22,22,22,22,23,23,23,23,23,23,23,24,24,24,24,24,24,24,24,24,25,25,25,25,25,25,25,26,27,27,27,27,28,28,28,28,28,28,28,28,29,29,29,29,29,29,29,29,30,31,31,31,31,31,31,31,31,32,32,32,32,32,32,32,32],"decoded":"סיכונים – אלמנט מרכזי, חיוני לפעילות מסחרית. להבין מה הסיכון הוא מאוד חשוב. החוויה האנושית עולה כי מי יודע איך לקחת סיכונים בזמן, היא לנצח גדולים. לזכור פוליטיק","decoded_with_specials":"סיכונים – אלמנט מרכזי, חיוני לפעילות מסחרית. להבין מה הסיכון הוא מאוד חשוב. החוויה האנושית עולה כי מי יודע איך לקחת סיכונים בזמן, היא לנצח גדולים. לזכור פוליטיק"} +{"kind":"sample","source":"fixtures/lang/hin_Deva.txt[:160]","text":"PHOTOS: न्यूज पढ़ते-पढ़ते अचानक ये क्या करने लगी एंकर!\nकुछ समय पहले एक टीवी चैनल पर एंकर खबर पढ़ रही थी और पीछे की स्क्रीन पर 10 मिनिट का पोर्न वीडियो चलता रहा,","ids":[42709,25,28225,101,24231,235,11976,107,24231,224,11976,250,28225,103,11976,95,11976,120,11976,97,24231,229,12,11976,103,11976,95,11976,120,11976,97,24231,229,28225,227,11976,248,48077,11976,101,11976,243,28225,107,24231,229,28225,243,24231,235,11976,107,48077,28225,243,11976,108,11976,101,24231,229,28225,110,11976,245,24231,222,28225,237,11976,224,11976,243,11976,108,0,198,11976,243,24231,223,11976,249,28225,116,11976,106,11976,107,28225,103,11976,117,11976,110,24231,229,28225,237,11976,243,28225,253,24231,222,11976,113,24231,222,28225,248,24231,230,11976,101,11976,110,28225,103,11976,108,28225,237,11976,224,11976,243,11976,108,28225,244,11976,105,11976,108,28225,103,11976,95,11976,120,28225,108,11976,117,24231,222,28225,98,24231,222,28225,242,11976,108,28225,103,24231,222,11976,249,24231,229,28225,243,24231,222,28225,116,24231,235,11976,243,24231,235,11976,108,24231,222,11976,101,28225,103,11976,108,838,28225,106,11976,123,11976,101,11976,123,11976,253,28225,243,48077,28225,103,24231,233,11976,108,24231,235,11976,101,28225,113,24231,222,11976,94,11976,123,11976,107,24231,233,28225,248,11976,110,11976,97,48077,28225,108,11976,117,48077,11],"ids_no_specials":[42709,25,28225,101,24231,235,11976,107,24231,224,11976,250,28225,103,11976,95,11976,120,11976,97,24231,229,12,11976,103,11976,95,11976,120,11976,97,24231,229,28225,227,11976,248,48077,11976,101,11976,243,28225,107,24231,229,28225,243,24231,235,11976,107,48077,28225,243,11976,108,11976,101,24231,229,28225,110,11976,245,24231,222,28225,237,11976,224,11976,243,11976,108,0,198,11976,243,24231,223,11976,249,28225,116,11976,106,11976,107,28225,103,11976,117,11976,110,24231,229,28225,237,11976,243,28225,253,24231,222,11976,113,24231,222,28225,248,24231,230,11976,101,11976,110,28225,103,11976,108,28225,237,11976,224,11976,243,11976,108,28225,244,11976,105,11976,108,28225,103,11976,95,11976,120,28225,108,11976,117,24231,222,28225,98,24231,222,28225,242,11976,108,28225,103,24231,222,11976,249,24231,229,28225,243,24231,222,28225,116,24231,235,11976,243,24231,235,11976,108,24231,222,11976,101,28225,103,11976,108,838,28225,106,11976,123,11976,101,11976,123,11976,253,28225,243,48077,28225,103,24231,233,11976,108,24231,235,11976,101,28225,113,24231,222,11976,94,11976,123,11976,107,24231,233,28225,248,11976,110,11976,97,48077,28225,108,11976,117,48077,11],"tokens":["PHOTOS",":","Ġà¤","¨","à¥","į","à¤","¯","à¥","Ĥ","à¤","ľ","Ġà¤","ª","à¤","¢","à¤","¼","à¤","¤","à¥","ĩ","-","à¤","ª","à¤","¢","à¤","¼","à¤","¤","à¥","ĩ","Ġà¤","ħ","à¤","ļ","ा","à¤","¨","à¤","ķ","Ġà¤","¯","à¥","ĩ","Ġà¤","ķ","à¥","į","à¤","¯","ा","Ġà¤","ķ","à¤","°","à¤","¨","à¥","ĩ","Ġà¤","²","à¤","Ĺ","à¥","Ģ","Ġà¤","ı","à¤","Ĥ","à¤","ķ","à¤","°","!","Ċ","à¤","ķ","à¥","ģ","à¤","Ľ","Ġà¤","¸","à¤","®","à¤","¯","Ġà¤","ª","à¤","¹","à¤","²","à¥","ĩ","Ġà¤","ı","à¤","ķ","Ġà¤","Ł","à¥","Ģ","à¤","µ","à¥","Ģ","Ġà¤","ļ","à¥","Ī","à¤","¨","à¤","²","Ġà¤","ª","à¤","°","Ġà¤","ı","à¤","Ĥ","à¤","ķ","à¤","°","Ġà¤","ĸ","à¤","¬","à¤","°","Ġà¤","ª","à¤","¢","à¤","¼","Ġà¤","°","à¤","¹","à¥","Ģ","Ġà¤","¥","à¥","Ģ","Ġà¤","Ķ","à¤","°","Ġà¤","ª","à¥","Ģ","à¤","Ľ","à¥","ĩ","Ġà¤","ķ","à¥","Ģ","Ġà¤","¸","à¥","į","à¤","ķ","à¥","į","à¤","°","à¥","Ģ","à¤","¨","Ġà¤","ª","à¤","°","Ġ10","Ġà¤","®","à¤","¿","à¤","¨","à¤","¿","à¤","Ł","Ġà¤","ķ","ा","Ġà¤","ª","à¥","ĭ","à¤","°","à¥","į","à¤","¨","Ġà¤","µ","à¥","Ģ","à¤","¡","à¤","¿","à¤","¯","à¥","ĭ","Ġà¤","ļ","à¤","²","à¤","¤","ा","Ġà¤","°","à¤","¹","ा",","],"offsets":[[0,6],[6,7],[7,9],[8,9],[9,10],[9,10],[10,11],[10,11],[11,12],[11,12],[12,13],[12,13],[13,15],[14,15],[15,16],[15,16],[16,17],[16,17],[17,18],[17,18],[18,19],[18,19],[19,20],[20,21],[20,21],[21,22],[21,22],[22,23],[22,23],[23,24],[23,24],[24,25],[24,25],[25,27],[26,27],[27,28],[27,28],[28,29],[29,30],[29,30],[30,31],[30,31],[31,33],[32,33],[33,34],[33,34],[34,36],[35,36],[36,37],[36,37],[37,38],[37,38],[38,39],[39,41],[40,41],[41,42],[41,42],[42,43],[42,43],[43,44],[43,44],[44,46],[45,46],[46,47],[46,47],[47,48],[47,48],[48,50],[49,50],[50,51],[50,51],[51,52],[51,52],[52,53],[52,53],[53,54],[54,55],[55,56],[55,56],[56,57],[56,57],[57,58],[57,58],[58,60],[59,60],[60,61],[60,61],[61,62],[61,62],[62,64],[63,64],[64,65],[64,65],[65,66],[65,66],[66,67],[66,67],[67,69],[68,69],[69,70],[69,70],[70,72],[71,72],[72,73],[72,73],[73,74],[73,74],[74,75],[74,75],[75,77],[76,77],[77,78],[77,78],[78,79],[78,79],[79,80],[79,80],[80,82],[81,82],[82,83],[82,83],[83,85],[84,85],[85,86],[85,86],[86,87],[86,87],[87,88],[87,88],[88,90],[89,90],[90,91],[90,91],[91,92],[91,92],[92,94],[93,94],[94,95],[94,95],[95,96],[95,96],[96,98],[97,98],[98,99],[98,99],[99,100],[99,100],[100,102],[101,102],[102,103],[102,103],[103,105],[104,105],[105,106],[105,106],[106,108],[107,108],[108,109],[108,109],[109,110],[109,110],[110,111],[110,111],[111,113],[112,113],[113,114],[113,114],[114,116],[115,116],[116,117],[116,117],[117,118],[117,118],[118,119],[118,119],[119,120],[119,120],[120,121],[120,121],[121,122],[121,122],[122,124],[123,124],[124,125],[124,125],[125,128],[128,130],[129,130],[130,131],[130,131],[131,132],[131,132],[132,133],[132,133],[133,134],[133,134],[134,136],[135,136],[136,137],[137,139],[138,139],[139,140],[139,140],[140,141],[140,141],[141,142],[141,142],[142,143],[142,143],[143,145],[144,145],[145,146],[145,146],[146,147],[146,147],[147,148],[147,148],[148,149],[148,149],[149,150],[149,150],[150,152],[151,152],[152,153],[152,153],[153,154],[153,154],[154,155],[155,157],[156,157],[157,158],[157,158],[158,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,1,2,2,3,3,4,4,5,5,6,6,7,7,7,7,8,8,9,9,10,10,10,11,11,11,11,12,12,13,13,14,14,15,15,15,15,16,17,17,17,17,18,18,19,19,20,20,21,21,22,22,23,24,24,24,24,24,24,25,25,26,26,26,26,27,27,28,28,29,29,30,30,30,30,31,32,33,33,34,34,35,35,36,36,36,36,36,36,37,37,37,37,37,37,38,38,39,39,39,39,40,40,41,41,42,42,43,43,44,44,45,45,46,46,46,46,47,47,47,47,48,48,49,49,50,50,50,50,51,51,51,51,51,51,52,52,52,52,53,53,54,54,54,54,55,55,56,56,57,57,58,58,58,58,59,59,60,60,61,61,62,62,63,63,64,64,65,65,66,66,67,67,68,68,69,69,70,70,71,71,72,72,72,72,73,74,74,75,75,76,76,77,77,78,78,79,79,80,81,81,82,82,83,83,84,84,85,85,86,86,87,87,88,88,89,89,90,90,91,91,92,92,92,92,92,92,93,94,94,94,94,95,95],"decoded":"PHOTOS: न्यूज पढ़ते-पढ़ते अचानक ये क्या करने लगी एंकर!\nकुछ समय पहले एक टीवी चैनल पर एंकर खबर पढ़ रही थी और पीछे की स्क्रीन पर 10 मिनिट का पोर्न वीडियो चलता रहा,","decoded_with_specials":"PHOTOS: न्यूज पढ़ते-पढ़ते अचानक ये क्या करने लगी एंकर!\nकुछ समय पहले एक टीवी चैनल पर एंकर खबर पढ़ रही थी और पीछे की स्क्रीन पर 10 मिनिट का पोर्न वीडियो चलता रहा,"} +{"kind":"sample","source":"fixtures/lang/jpn_Jpan.txt[:160]","text":"Omni Dallas Parkwest Hotelでは4ツ星ホテルで、アイアンホース・ゴルフコース、Love Field Airport とテキサス・スタジアムから7.8kmかかるのところにあります。 優れたホテルにオープンし、ダラスにある古代の建築の象徴です。\n部屋\n快適なゲストルームには、モダンな設備を備えたプレ","ids":[46,76,8461,8533,3250,7038,12696,30640,31676,19,41115,23626,253,1209,249,24336,9202,30640,23513,11839,11482,11839,6527,1209,249,6312,8943,4707,17933,9202,17681,24679,6312,8943,23513,18565,7663,12690,23294,101,24336,25084,26503,8943,4707,8943,23376,21091,11839,25795,27370,36853,22,13,23,13276,27370,27370,25748,5641,30201,46036,1792,235,28618,40948,28255,30159,33623,16764,10263,226,103,39258,25224,1209,249,24336,9202,28618,20513,12045,245,6527,22180,23513,27852,9263,8943,28618,40948,25748,20998,97,47987,15474,119,118,163,107,231,5641,164,109,94,36181,112,30640,33623,16764,198,32849,101,161,109,233,198,33232,104,34402,102,26945,1792,110,43302,9202,12045,254,28618,31676,23513,40361,27852,6527,26945,164,101,255,43636,247,31758,43636,247,2515,230,25224,30965,24186],"ids_no_specials":[46,76,8461,8533,3250,7038,12696,30640,31676,19,41115,23626,253,1209,249,24336,9202,30640,23513,11839,11482,11839,6527,1209,249,6312,8943,4707,17933,9202,17681,24679,6312,8943,23513,18565,7663,12690,23294,101,24336,25084,26503,8943,4707,8943,23376,21091,11839,25795,27370,36853,22,13,23,13276,27370,27370,25748,5641,30201,46036,1792,235,28618,40948,28255,30159,33623,16764,10263,226,103,39258,25224,1209,249,24336,9202,28618,20513,12045,245,6527,22180,23513,27852,9263,8943,28618,40948,25748,20998,97,47987,15474,119,118,163,107,231,5641,164,109,94,36181,112,30640,33623,16764,198,32849,101,161,109,233,198,33232,104,34402,102,26945,1792,110,43302,9202,12045,254,28618,31676,23513,40361,27852,6527,26945,164,101,255,43636,247,31758,43636,247,2515,230,25224,30965,24186],"tokens":["O","m","ni","ĠDallas","ĠPark","west","ĠHotel","ãģ§","ãģ¯","4","ãĥĦ","æĺ","Ł","ãĥ","Ľ","ãĥĨ","ãĥ«","ãģ§","ãĢģ","ãĤ¢","ãĤ¤","ãĤ¢","ãĥ³","ãĥ","Ľ","ãĥ¼","ãĤ¹","ãĥ»","ãĤ´","ãĥ«","ãĥķ","ãĤ³","ãĥ¼","ãĤ¹","ãĢģ","Love","ĠField","ĠAirport","Ġãģ","¨","ãĥĨ","ãĤŃ","ãĤµ","ãĤ¹","ãĥ»","ãĤ¹","ãĤ¿","ãĤ¸","ãĤ¢","ãĥł","ãģĭ","ãĤī","7",".","8","km","ãģĭ","ãģĭ","ãĤĭ","ãģ®","ãģ¨","ãģĵ","ãĤ","į","ãģ«","ãģĤ","ãĤĬ","ãģ¾","ãģĻ","ãĢĤ","Ġå","Ħ","ª","ãĤĮ","ãģŁ","ãĥ","Ľ","ãĥĨ","ãĥ«","ãģ«","ãĤª","ãĥ¼ãĥ","Ĺ","ãĥ³","ãģĹ","ãĢģ","ãĥĢ","ãĥ©","ãĤ¹","ãģ«","ãģĤ","ãĤĭ","åı","¤","代","ãģ®å","»","º","ç","¯","ī","ãģ®","è","±","¡","å¾","´","ãģ§","ãģĻ","ãĢĤ","Ċ","éĥ","¨","å","±","ĭ","Ċ","å¿","«","éģ","©","ãģª","ãĤ","²","ãĤ¹ãĥĪ","ãĥ«","ãĥ¼ãĥ","ł","ãģ«","ãģ¯","ãĢģ","ãĥ¢","ãĥĢ","ãĥ³","ãģª","è","¨","Ń","åĤ","Ļ","ãĤĴ","åĤ","Ļ","ãģ","Ī","ãģŁ","ãĥĹ","ãĥ¬"],"offsets":[[0,1],[1,2],[2,4],[4,11],[11,16],[16,20],[20,26],[26,27],[27,28],[28,29],[29,30],[30,31],[30,31],[31,32],[31,32],[32,33],[33,34],[34,35],[35,36],[36,37],[37,38],[38,39],[39,40],[40,41],[40,41],[41,42],[42,43],[43,44],[44,45],[45,46],[46,47],[47,48],[48,49],[49,50],[50,51],[51,55],[55,61],[61,69],[69,71],[70,71],[71,72],[72,73],[73,74],[74,75],[75,76],[76,77],[77,78],[78,79],[79,80],[80,81],[81,82],[82,83],[83,84],[84,85],[85,86],[86,88],[88,89],[89,90],[90,91],[91,92],[92,93],[93,94],[94,95],[94,95],[95,96],[96,97],[97,98],[98,99],[99,100],[100,101],[101,103],[102,103],[102,103],[103,104],[104,105],[105,106],[105,106],[106,107],[107,108],[108,109],[109,110],[110,112],[111,112],[112,113],[113,114],[114,115],[115,116],[116,117],[117,118],[118,119],[119,120],[120,121],[121,122],[121,122],[122,123],[123,125],[124,125],[124,125],[125,126],[125,126],[125,126],[126,127],[127,128],[127,128],[127,128],[128,129],[128,129],[129,130],[130,131],[131,132],[132,133],[133,134],[133,134],[134,135],[134,135],[134,135],[135,136],[136,137],[136,137],[137,138],[137,138],[138,139],[139,140],[139,140],[140,142],[142,143],[143,145],[144,145],[145,146],[146,147],[147,148],[148,149],[149,150],[150,151],[151,152],[152,153],[152,153],[152,153],[153,154],[153,154],[154,155],[155,156],[155,156],[156,157],[156,157],[157,158],[158,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,1,2,2,3,3,3,4,5,5,5,5,5,5,5,5,6,7,7,7,7,7,7,7,7,8,9,9,9,9,9,9,10,11,12,13,14,14,14,14,14,14,15,16,16,16,16,16,16,16,17,18,19,20,20,20,20,20,20,20,20,20,20,20,20,20,20,21,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,23,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,25,26,27,27,27,27,27,28,29,29,29,29,29,29,29,29,29,29,29,29,29,30,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31],"decoded":"Omni Dallas Parkwest Hotelでは4ツ星ホテルで、アイアンホース・ゴルフコース、Love Field Airport とテキサス・スタジアムから7.8kmかかるのところにあります。 優れたホテルにオープンし、ダラスにある古代の建築の象徴です。\n部屋\n快適なゲストルームには、モダンな設備を備えたプレ","decoded_with_specials":"Omni Dallas Parkwest Hotelでは4ツ星ホテルで、アイアンホース・ゴルフコース、Love Field Airport とテキサス・スタジアムから7.8kmかかるのところにあります。 優れたホテルにオープンし、ダラスにある古代の建築の象徴です。\n部屋\n快適なゲストルームには、モダンな設備を備えたプレ"} +{"kind":"sample","source":"fixtures/lang/kat_Geor.txt[:160]","text":"ჩვენ მსოფლიოს სხვადასხვა ქვეყანაში ვცხოვრობთ და სხვადასხვა ენაზე ვსაუბრობთ, მაგრამ საერთო მიზანი გვაერთიანებს. ჩვენი მთავარი მიზანი სამყაროს შემოქმედისა და ბიბლ","ids":[157,225,102,157,225,243,157,225,242,157,225,250,28053,225,249,157,225,94,157,225,251,157,225,97,157,225,248,157,225,246,157,225,251,157,225,94,28053,225,94,157,225,106,157,225,243,157,225,238,157,225,241,157,225,238,157,225,94,157,225,106,157,225,243,157,225,238,28053,225,98,157,225,243,157,225,242,157,225,100,157,225,238,157,225,250,157,225,238,157,225,101,157,225,246,28053,225,243,157,225,103,157,225,106,157,225,251,157,225,243,157,225,254,157,225,251,157,225,239,157,225,245,28053,225,241,157,225,238,28053,225,94,157,225,106,157,225,243,157,225,238,157,225,241,157,225,238,157,225,94,157,225,106,157,225,243,157,225,238,28053,225,242,157,225,250,157,225,238,157,225,244,157,225,242,28053,225,243,157,225,94,157,225,238,157,225,96,157,225,239,157,225,254,157,225,251,157,225,239,157,225,245,11,28053,225,249,157,225,238,157,225,240,157,225,254,157,225,238,157,225,249,28053,225,94,157,225,238,157,225,242,157,225,254,157,225,245,157,225,251,28053,225,249,157,225,246,157,225,244,157,225,238,157,225,250,157,225,246,28053,225,240,157,225,243,157,225,238,157,225,242,157,225,254,157,225,245,157,225,246,157,225,238,157,225,250,157,225,242,157,225,239,157,225,94,13,28053,225,102,157,225,243,157,225,242,157,225,250,157,225,246,28053,225,249,157,225,245,157,225,238,157,225,243,157,225,238,157,225,254,157,225,246,28053,225,249,157,225,246,157,225,244,157,225,238,157,225,250,157,225,246,28053,225,94,157,225,238,157,225,249,157,225,100,157,225,238,157,225,254,157,225,251,157,225,94,28053,225,101,157,225,242,157,225,249,157,225,251,157,225,98,157,225,249,157,225,242,157,225,241,157,225,246,157,225,94,157,225,238,28053,225,241,157,225,238,28053,225,239,157,225,246,157,225,239,157,225,248],"ids_no_specials":[157,225,102,157,225,243,157,225,242,157,225,250,28053,225,249,157,225,94,157,225,251,157,225,97,157,225,248,157,225,246,157,225,251,157,225,94,28053,225,94,157,225,106,157,225,243,157,225,238,157,225,241,157,225,238,157,225,94,157,225,106,157,225,243,157,225,238,28053,225,98,157,225,243,157,225,242,157,225,100,157,225,238,157,225,250,157,225,238,157,225,101,157,225,246,28053,225,243,157,225,103,157,225,106,157,225,251,157,225,243,157,225,254,157,225,251,157,225,239,157,225,245,28053,225,241,157,225,238,28053,225,94,157,225,106,157,225,243,157,225,238,157,225,241,157,225,238,157,225,94,157,225,106,157,225,243,157,225,238,28053,225,242,157,225,250,157,225,238,157,225,244,157,225,242,28053,225,243,157,225,94,157,225,238,157,225,96,157,225,239,157,225,254,157,225,251,157,225,239,157,225,245,11,28053,225,249,157,225,238,157,225,240,157,225,254,157,225,238,157,225,249,28053,225,94,157,225,238,157,225,242,157,225,254,157,225,245,157,225,251,28053,225,249,157,225,246,157,225,244,157,225,238,157,225,250,157,225,246,28053,225,240,157,225,243,157,225,238,157,225,242,157,225,254,157,225,245,157,225,246,157,225,238,157,225,250,157,225,242,157,225,239,157,225,94,13,28053,225,102,157,225,243,157,225,242,157,225,250,157,225,246,28053,225,249,157,225,245,157,225,238,157,225,243,157,225,238,157,225,254,157,225,246,28053,225,249,157,225,246,157,225,244,157,225,238,157,225,250,157,225,246,28053,225,94,157,225,238,157,225,249,157,225,100,157,225,238,157,225,254,157,225,251,157,225,94,28053,225,101,157,225,242,157,225,249,157,225,251,157,225,98,157,225,249,157,225,242,157,225,241,157,225,246,157,225,94,157,225,238,28053,225,241,157,225,238,28053,225,239,157,225,246,157,225,239,157,225,248],"tokens":["á","ĥ","©","á","ĥ","ķ","á","ĥ","Ķ","á","ĥ","ľ","Ġá","ĥ","Ľ","á","ĥ","¡","á","ĥ","Ŀ","á","ĥ","¤","á","ĥ","ļ","á","ĥ","ĺ","á","ĥ","Ŀ","á","ĥ","¡","Ġá","ĥ","¡","á","ĥ","®","á","ĥ","ķ","á","ĥ","IJ","á","ĥ","ĵ","á","ĥ","IJ","á","ĥ","¡","á","ĥ","®","á","ĥ","ķ","á","ĥ","IJ","Ġá","ĥ","¥","á","ĥ","ķ","á","ĥ","Ķ","á","ĥ","§","á","ĥ","IJ","á","ĥ","ľ","á","ĥ","IJ","á","ĥ","¨","á","ĥ","ĺ","Ġá","ĥ","ķ","á","ĥ","ª","á","ĥ","®","á","ĥ","Ŀ","á","ĥ","ķ","á","ĥ","ł","á","ĥ","Ŀ","á","ĥ","ij","á","ĥ","Ĺ","Ġá","ĥ","ĵ","á","ĥ","IJ","Ġá","ĥ","¡","á","ĥ","®","á","ĥ","ķ","á","ĥ","IJ","á","ĥ","ĵ","á","ĥ","IJ","á","ĥ","¡","á","ĥ","®","á","ĥ","ķ","á","ĥ","IJ","Ġá","ĥ","Ķ","á","ĥ","ľ","á","ĥ","IJ","á","ĥ","ĸ","á","ĥ","Ķ","Ġá","ĥ","ķ","á","ĥ","¡","á","ĥ","IJ","á","ĥ","£","á","ĥ","ij","á","ĥ","ł","á","ĥ","Ŀ","á","ĥ","ij","á","ĥ","Ĺ",",","Ġá","ĥ","Ľ","á","ĥ","IJ","á","ĥ","Ĵ","á","ĥ","ł","á","ĥ","IJ","á","ĥ","Ľ","Ġá","ĥ","¡","á","ĥ","IJ","á","ĥ","Ķ","á","ĥ","ł","á","ĥ","Ĺ","á","ĥ","Ŀ","Ġá","ĥ","Ľ","á","ĥ","ĺ","á","ĥ","ĸ","á","ĥ","IJ","á","ĥ","ľ","á","ĥ","ĺ","Ġá","ĥ","Ĵ","á","ĥ","ķ","á","ĥ","IJ","á","ĥ","Ķ","á","ĥ","ł","á","ĥ","Ĺ","á","ĥ","ĺ","á","ĥ","IJ","á","ĥ","ľ","á","ĥ","Ķ","á","ĥ","ij","á","ĥ","¡",".","Ġá","ĥ","©","á","ĥ","ķ","á","ĥ","Ķ","á","ĥ","ľ","á","ĥ","ĺ","Ġá","ĥ","Ľ","á","ĥ","Ĺ","á","ĥ","IJ","á","ĥ","ķ","á","ĥ","IJ","á","ĥ","ł","á","ĥ","ĺ","Ġá","ĥ","Ľ","á","ĥ","ĺ","á","ĥ","ĸ","á","ĥ","IJ","á","ĥ","ľ","á","ĥ","ĺ","Ġá","ĥ","¡","á","ĥ","IJ","á","ĥ","Ľ","á","ĥ","§","á","ĥ","IJ","á","ĥ","ł","á","ĥ","Ŀ","á","ĥ","¡","Ġá","ĥ","¨","á","ĥ","Ķ","á","ĥ","Ľ","á","ĥ","Ŀ","á","ĥ","¥","á","ĥ","Ľ","á","ĥ","Ķ","á","ĥ","ĵ","á","ĥ","ĺ","á","ĥ","¡","á","ĥ","IJ","Ġá","ĥ","ĵ","á","ĥ","IJ","Ġá","ĥ","ij","á","ĥ","ĺ","á","ĥ","ij","á","ĥ","ļ"],"offsets":[[0,1],[0,1],[0,1],[1,2],[1,2],[1,2],[2,3],[2,3],[2,3],[3,4],[3,4],[3,4],[4,6],[5,6],[5,6],[6,7],[6,7],[6,7],[7,8],[7,8],[7,8],[8,9],[8,9],[8,9],[9,10],[9,10],[9,10],[10,11],[10,11],[10,11],[11,12],[11,12],[11,12],[12,13],[12,13],[12,13],[13,15],[14,15],[14,15],[15,16],[15,16],[15,16],[16,17],[16,17],[16,17],[17,18],[17,18],[17,18],[18,19],[18,19],[18,19],[19,20],[19,20],[19,20],[20,21],[20,21],[20,21],[21,22],[21,22],[21,22],[22,23],[22,23],[22,23],[23,24],[23,24],[23,24],[24,26],[25,26],[25,26],[26,27],[26,27],[26,27],[27,28],[27,28],[27,28],[28,29],[28,29],[28,29],[29,30],[29,30],[29,30],[30,31],[30,31],[30,31],[31,32],[31,32],[31,32],[32,33],[32,33],[32,33],[33,34],[33,34],[33,34],[34,36],[35,36],[35,36],[36,37],[36,37],[36,37],[37,38],[37,38],[37,38],[38,39],[38,39],[38,39],[39,40],[39,40],[39,40],[40,41],[40,41],[40,41],[41,42],[41,42],[41,42],[42,43],[42,43],[42,43],[43,44],[43,44],[43,44],[44,46],[45,46],[45,46],[46,47],[46,47],[46,47],[47,49],[48,49],[48,49],[49,50],[49,50],[49,50],[50,51],[50,51],[50,51],[51,52],[51,52],[51,52],[52,53],[52,53],[52,53],[53,54],[53,54],[53,54],[54,55],[54,55],[54,55],[55,56],[55,56],[55,56],[56,57],[56,57],[56,57],[57,58],[57,58],[57,58],[58,60],[59,60],[59,60],[60,61],[60,61],[60,61],[61,62],[61,62],[61,62],[62,63],[62,63],[62,63],[63,64],[63,64],[63,64],[64,66],[65,66],[65,66],[66,67],[66,67],[66,67],[67,68],[67,68],[67,68],[68,69],[68,69],[68,69],[69,70],[69,70],[69,70],[70,71],[70,71],[70,71],[71,72],[71,72],[71,72],[72,73],[72,73],[72,73],[73,74],[73,74],[73,74],[74,75],[75,77],[76,77],[76,77],[77,78],[77,78],[77,78],[78,79],[78,79],[78,79],[79,80],[79,80],[79,80],[80,81],[80,81],[80,81],[81,82],[81,82],[81,82],[82,84],[83,84],[83,84],[84,85],[84,85],[84,85],[85,86],[85,86],[85,86],[86,87],[86,87],[86,87],[87,88],[87,88],[87,88],[88,89],[88,89],[88,89],[89,91],[90,91],[90,91],[91,92],[91,92],[91,92],[92,93],[92,93],[92,93],[93,94],[93,94],[93,94],[94,95],[94,95],[94,95],[95,96],[95,96],[95,96],[96,98],[97,98],[97,98],[98,99],[98,99],[98,99],[99,100],[99,100],[99,100],[100,101],[100,101],[100,101],[101,102],[101,102],[101,102],[102,103],[102,103],[102,103],[103,104],[103,104],[103,104],[104,105],[104,105],[104,105],[105,106],[105,106],[105,106],[106,107],[106,107],[106,107],[107,108],[107,108],[107,108],[108,109],[108,109],[108,109],[109,110],[110,112],[111,112],[111,112],[112,113],[112,113],[112,113],[113,114],[113,114],[113,114],[114,115],[114,115],[114,115],[115,116],[115,116],[115,116],[116,118],[117,118],[117,118],[118,119],[118,119],[118,119],[119,120],[119,120],[119,120],[120,121],[120,121],[120,121],[121,122],[121,122],[121,122],[122,123],[122,123],[122,123],[123,124],[123,124],[123,124],[124,126],[125,126],[125,126],[126,127],[126,127],[126,127],[127,128],[127,128],[127,128],[128,129],[128,129],[128,129],[129,130],[129,130],[129,130],[130,131],[130,131],[130,131],[131,133],[132,133],[132,133],[133,134],[133,134],[133,134],[134,135],[134,135],[134,135],[135,136],[135,136],[135,136],[136,137],[136,137],[136,137],[137,138],[137,138],[137,138],[138,139],[138,139],[138,139],[139,140],[139,140],[139,140],[140,142],[141,142],[141,142],[142,143],[142,143],[142,143],[143,144],[143,144],[143,144],[144,145],[144,145],[144,145],[145,146],[145,146],[145,146],[146,147],[146,147],[146,147],[147,148],[147,148],[147,148],[148,149],[148,149],[148,149],[149,150],[149,150],[149,150],[150,151],[150,151],[150,151],[151,152],[151,152],[151,152],[152,154],[153,154],[153,154],[154,155],[154,155],[154,155],[155,157],[156,157],[156,157],[157,158],[157,158],[157,158],[158,159],[158,159],[158,159],[159,160],[159,160],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,9,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,14,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,20,20,20,20,20,20,21,21,21,21,21,21,21,21,21,21,21,21],"decoded":"ჩვენ მსოფლიოს სხვადასხვა ქვეყანაში ვცხოვრობთ და სხვადასხვა ენაზე ვსაუბრობთ, მაგრამ საერთო მიზანი გვაერთიანებს. ჩვენი მთავარი მიზანი სამყაროს შემოქმედისა და ბიბლ","decoded_with_specials":"ჩვენ მსოფლიოს სხვადასხვა ქვეყანაში ვცხოვრობთ და სხვადასხვა ენაზე ვსაუბრობთ, მაგრამ საერთო მიზანი გვაერთიანებს. ჩვენი მთავარი მიზანი სამყაროს შემოქმედისა და ბიბლ"} +{"kind":"sample","source":"fixtures/lang/kor_Hang.txt[:160]","text":"전화번호:\n위치: 뉴질랜드 > 남섬 > 말버러 > 블레넘\n가격대 (1박): 117,886 원 - 140,244 원\n4.5성급 — Lugano Motor Lodge 4.5*\n- 예약 옵션:\n- 트립어드바이저는 호텔스닷컴, Expedia, Agoda, Asia Web Direct 및 Boo","ids":[168,254,226,169,247,242,167,110,230,169,246,116,25,198,168,250,226,168,117,246,25,31619,231,112,168,100,230,167,252,250,167,241,250,1875,31619,224,101,168,226,105,1875,31619,100,238,167,110,226,167,253,105,1875,31619,116,242,167,254,230,167,226,246,198,166,108,222,166,110,102,167,234,222,357,16,167,108,243,2599,19048,11,44980,23821,249,238,532,12713,11,25707,23821,249,238,198,19,13,20,168,226,109,166,116,231,851,31541,5733,12533,27046,604,13,20,9,198,12,23821,246,230,168,243,121,23821,246,113,168,227,246,25,198,12,220,169,232,116,167,99,121,168,244,112,167,241,250,167,108,242,35975,112,168,254,222,167,232,242,220,169,246,116,169,227,242,168,232,97,46695,115,168,119,112,11,5518,5507,11,2449,11329,11,7229,5313,4128,31619,108,237,21458],"ids_no_specials":[168,254,226,169,247,242,167,110,230,169,246,116,25,198,168,250,226,168,117,246,25,31619,231,112,168,100,230,167,252,250,167,241,250,1875,31619,224,101,168,226,105,1875,31619,100,238,167,110,226,167,253,105,1875,31619,116,242,167,254,230,167,226,246,198,166,108,222,166,110,102,167,234,222,357,16,167,108,243,2599,19048,11,44980,23821,249,238,532,12713,11,25707,23821,249,238,198,19,13,20,168,226,109,166,116,231,851,31541,5733,12533,27046,604,13,20,9,198,12,23821,246,230,168,243,121,23821,246,113,168,227,246,25,198,12,220,169,232,116,167,99,121,168,244,112,167,241,250,167,108,242,35975,112,168,254,222,167,232,242,220,169,246,116,169,227,242,168,232,97,46695,115,168,119,112,11,5518,5507,11,2449,11329,11,7229,5313,4128,31619,108,237,21458],"tokens":["ì","ł","Ħ","í","Ļ","Ķ","ë","²","Ī","í","ĺ","¸",":","Ċ","ì","ľ","Ħ","ì","¹","ĺ",":","Ġë","ī","´","ì","§","Ī","ë","ŀ","ľ","ë","ĵ","ľ","Ġ>","Ġë","Ĥ","¨","ì","Ħ","¬","Ġ>","Ġë","§","IJ","ë","²","Ħ","ë","Ł","¬","Ġ>","Ġë","¸","Ķ","ë","ł","Ī","ë","Ħ","ĺ","Ċ","ê","°","Ģ","ê","²","©","ë","Į","Ģ","Ġ(","1","ë","°","ķ","):","Ġ117",",","886","Ġì","Ľ","IJ","Ġ-","Ġ140",",","244","Ġì","Ľ","IJ","Ċ","4",".","5","ì","Ħ","±","ê","¸","ī","ĠâĢĶ","ĠLug","ano","ĠMotor","ĠLodge","Ġ4",".","5","*","Ċ","-","Ġì","ĺ","Ī","ì","ķ","½","Ġì","ĺ","µ","ì","ħ","ĺ",":","Ċ","-","Ġ","í","Ĭ","¸","ë","¦","½","ì","ĸ","´","ë","ĵ","ľ","ë","°","Ķ","ìĿ","´","ì","ł","Ģ","ë","Ĭ","Ķ","Ġ","í","ĺ","¸","í","ħ","Ķ","ì","Ĭ","¤","ëĭ","·","ì","»","´",",","ĠExp","edia",",","ĠAg","oda",",","ĠAsia","ĠWeb","ĠDirect","Ġë","°","ı","ĠBoo"],"offsets":[[0,1],[0,1],[0,1],[1,2],[1,2],[1,2],[2,3],[2,3],[2,3],[3,4],[3,4],[3,4],[4,5],[5,6],[6,7],[6,7],[6,7],[7,8],[7,8],[7,8],[8,9],[9,11],[10,11],[10,11],[11,12],[11,12],[11,12],[12,13],[12,13],[12,13],[13,14],[13,14],[13,14],[14,16],[16,18],[17,18],[17,18],[18,19],[18,19],[18,19],[19,21],[21,23],[22,23],[22,23],[23,24],[23,24],[23,24],[24,25],[24,25],[24,25],[25,27],[27,29],[28,29],[28,29],[29,30],[29,30],[29,30],[30,31],[30,31],[30,31],[31,32],[32,33],[32,33],[32,33],[33,34],[33,34],[33,34],[34,35],[34,35],[34,35],[35,37],[37,38],[38,39],[38,39],[38,39],[39,41],[41,45],[45,46],[46,49],[49,51],[50,51],[50,51],[51,53],[53,57],[57,58],[58,61],[61,63],[62,63],[62,63],[63,64],[64,65],[65,66],[66,67],[67,68],[67,68],[67,68],[68,69],[68,69],[68,69],[69,71],[71,75],[75,78],[78,84],[84,90],[90,92],[92,93],[93,94],[94,95],[95,96],[96,97],[97,99],[98,99],[98,99],[99,100],[99,100],[99,100],[100,102],[101,102],[101,102],[102,103],[102,103],[102,103],[103,104],[104,105],[105,106],[106,107],[107,108],[107,108],[107,108],[108,109],[108,109],[108,109],[109,110],[109,110],[109,110],[110,111],[110,111],[110,111],[111,112],[111,112],[111,112],[112,113],[112,113],[113,114],[113,114],[113,114],[114,115],[114,115],[114,115],[115,116],[116,117],[116,117],[116,117],[117,118],[117,118],[117,118],[118,119],[118,119],[118,119],[119,120],[119,120],[120,121],[120,121],[120,121],[121,122],[122,126],[126,130],[130,131],[131,134],[134,137],[137,138],[138,143],[143,147],[147,154],[154,156],[155,156],[155,156],[156,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,0,0,0,0,0,1,2,3,3,3,3,3,3,4,5,5,5,5,5,5,5,5,5,5,5,5,6,7,7,7,7,7,7,8,9,9,9,9,9,9,9,9,9,10,11,11,11,11,11,11,11,11,11,12,13,13,13,13,13,13,13,13,13,14,15,16,16,16,17,18,19,20,21,21,21,22,23,24,25,26,26,26,27,28,29,30,31,31,31,31,31,31,32,33,33,34,35,36,37,38,39,40,41,42,42,42,42,42,42,43,43,43,43,43,43,44,45,46,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,49,50,50,51,52,52,53,54,55,56,57,57,57,58],"decoded":"전화번호:\n위치: 뉴질랜드 > 남섬 > 말버러 > 블레넘\n가격대 (1박): 117,886 원 - 140,244 원\n4.5성급 — Lugano Motor Lodge 4.5*\n- 예약 옵션:\n- 트립어드바이저는 호텔스닷컴, Expedia, Agoda, Asia Web Direct 및 Boo","decoded_with_specials":"전화번호:\n위치: 뉴질랜드 > 남섬 > 말버러 > 블레넘\n가격대 (1박): 117,886 원 - 140,244 원\n4.5성급 — Lugano Motor Lodge 4.5*\n- 예약 옵션:\n- 트립어드바이저는 호텔스닷컴, Expedia, Agoda, Asia Web Direct 및 Boo"} +{"kind":"sample","source":"fixtures/lang/rus_Cyrl.txt[:160]","text":"Покупая продукты в супермаркетах, сегодня уже мало кто верит в их качество. Как не сделать свое меню экстремальным и на что следует обращать внимание – читайте!","ids":[140,253,25443,118,35072,140,123,16142,40623,12466,123,21169,25443,112,35072,31583,20375,45035,12466,110,220,21727,35072,140,123,16843,21169,43108,16142,21169,31583,16843,20375,16142,141,227,11,220,21727,16843,140,111,25443,112,22177,40623,220,35072,140,114,16843,12466,120,16142,30143,15166,12466,118,20375,15166,12466,110,16843,21169,18849,20375,12466,110,12466,116,141,227,12466,118,16142,141,229,16843,21727,20375,38857,15166,13,12466,248,16142,31583,12466,121,16843,220,21727,43666,16843,30143,16142,20375,45367,220,21727,38857,15166,16843,12466,120,16843,22177,141,236,220,141,235,31583,21727,20375,21169,16843,43108,16142,30143,45367,22177,45035,43108,12466,116,12466,121,16142,220,141,229,20375,15166,220,21727,30143,16843,43666,35072,16843,20375,12466,122,140,109,21169,16142,141,231,16142,20375,45367,12466,110,22177,18849,43108,16142,22177,18849,16843,784,220,141,229,18849,20375,16142,140,117,20375,16843,0],"ids_no_specials":[140,253,25443,118,35072,140,123,16142,40623,12466,123,21169,25443,112,35072,31583,20375,45035,12466,110,220,21727,35072,140,123,16843,21169,43108,16142,21169,31583,16843,20375,16142,141,227,11,220,21727,16843,140,111,25443,112,22177,40623,220,35072,140,114,16843,12466,120,16142,30143,15166,12466,118,20375,15166,12466,110,16843,21169,18849,20375,12466,110,12466,116,141,227,12466,118,16142,141,229,16843,21727,20375,38857,15166,13,12466,248,16142,31583,12466,121,16843,220,21727,43666,16843,30143,16142,20375,45367,220,21727,38857,15166,16843,12466,120,16843,22177,141,236,220,141,235,31583,21727,20375,21169,16843,43108,16142,30143,45367,22177,45035,43108,12466,116,12466,121,16142,220,141,229,20375,15166,220,21727,30143,16843,43666,35072,16843,20375,12466,122,140,109,21169,16142,141,231,16142,20375,45367,12466,110,22177,18849,43108,16142,22177,18849,16843,784,220,141,229,18849,20375,16142,140,117,20375,16843,0],"tokens":["Ð","Ł","оÐ","º","Ñĥ","Ð","¿","а","Ñı","ĠÐ","¿","ÑĢ","оÐ","´","Ñĥ","к","ÑĤ","Ñĭ","ĠÐ","²","Ġ","Ñģ","Ñĥ","Ð","¿","е","ÑĢ","м","а","ÑĢ","к","е","ÑĤ","а","Ñ","ħ",",","Ġ","Ñģ","е","Ð","³","оÐ","´","н","Ñı","Ġ","Ñĥ","Ð","¶","е","ĠÐ","¼","а","л","о","ĠÐ","º","ÑĤ","о","ĠÐ","²","е","ÑĢ","и","ÑĤ","ĠÐ","²","ĠÐ","¸","Ñ","ħ","ĠÐ","º","а","Ñ","ĩ","е","Ñģ","ÑĤ","в","о",".","ĠÐ","ļ","а","к","ĠÐ","½","е","Ġ","Ñģ","д","е","л","а","ÑĤ","ÑĮ","Ġ","Ñģ","в","о","е","ĠÐ","¼","е","н","Ñ","İ","Ġ","Ñ","į","к","Ñģ","ÑĤ","ÑĢ","е","м","а","л","ÑĮ","н","Ñĭ","м","ĠÐ","¸","ĠÐ","½","а","Ġ","Ñ","ĩ","ÑĤ","о","Ġ","Ñģ","л","е","д","Ñĥ","е","ÑĤ","ĠÐ","¾","Ð","±","ÑĢ","а","Ñ","ī","а","ÑĤ","ÑĮ","ĠÐ","²","н","и","м","а","н","и","е","ĠâĢĵ","Ġ","Ñ","ĩ","и","ÑĤ","а","Ð","¹","ÑĤ","е","!"],"offsets":[[0,1],[0,1],[1,3],[2,3],[3,4],[4,5],[4,5],[5,6],[6,7],[7,9],[8,9],[9,10],[10,12],[11,12],[12,13],[13,14],[14,15],[15,16],[16,18],[17,18],[18,19],[19,20],[20,21],[21,22],[21,22],[22,23],[23,24],[24,25],[25,26],[26,27],[27,28],[28,29],[29,30],[30,31],[31,32],[31,32],[32,33],[33,34],[34,35],[35,36],[36,37],[36,37],[37,39],[38,39],[39,40],[40,41],[41,42],[42,43],[43,44],[43,44],[44,45],[45,47],[46,47],[47,48],[48,49],[49,50],[50,52],[51,52],[52,53],[53,54],[54,56],[55,56],[56,57],[57,58],[58,59],[59,60],[60,62],[61,62],[62,64],[63,64],[64,65],[64,65],[65,67],[66,67],[67,68],[68,69],[68,69],[69,70],[70,71],[71,72],[72,73],[73,74],[74,75],[75,77],[76,77],[77,78],[78,79],[79,81],[80,81],[81,82],[82,83],[83,84],[84,85],[85,86],[86,87],[87,88],[88,89],[89,90],[90,91],[91,92],[92,93],[93,94],[94,95],[95,97],[96,97],[97,98],[98,99],[99,100],[99,100],[100,101],[101,102],[101,102],[102,103],[103,104],[104,105],[105,106],[106,107],[107,108],[108,109],[109,110],[110,111],[111,112],[112,113],[113,114],[114,116],[115,116],[116,118],[117,118],[118,119],[119,120],[120,121],[120,121],[121,122],[122,123],[123,124],[124,125],[125,126],[126,127],[127,128],[128,129],[129,130],[130,131],[131,133],[132,133],[133,134],[133,134],[134,135],[135,136],[136,137],[136,137],[137,138],[138,139],[139,140],[140,142],[141,142],[142,143],[143,144],[144,145],[145,146],[146,147],[147,148],[148,149],[149,151],[151,152],[152,153],[152,153],[153,154],[154,155],[155,156],[156,157],[156,157],[157,158],[158,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,2,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,5,5,5,5,5,5,5,5,5,6,6,6,6,6,7,7,7,7,7,8,8,8,8,9,9,9,9,9,9,10,10,11,11,11,11,12,12,12,12,12,12,12,12,12,12,13,14,14,14,14,15,15,15,16,16,16,16,16,16,16,16,17,17,17,17,17,18,18,18,18,18,18,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,20,20,21,21,21,22,22,22,22,22,23,23,23,23,23,23,23,23,24,24,24,24,24,24,24,24,24,24,24,25,25,25,25,25,25,25,25,25,26,27,27,27,27,27,27,27,27,27,27,28],"decoded":"Покупая продукты в супермаркетах, сегодня уже мало кто верит в их качество. Как не сделать свое меню экстремальным и на что следует обращать внимание – читайте!","decoded_with_specials":"Покупая продукты в супермаркетах, сегодня уже мало кто верит в их качество. Как не сделать свое меню экстремальным и на что следует обращать внимание – читайте!"} +{"kind":"sample","source":"fixtures/lang/tam_Taml.txt[:160]","text":"‘ஹலோ கலெக்டர் சாரா… சரக்கு வேணும்… கடை எப்ப திறப்பீங்க?’\nஒரு படத்தில் ஒயின்ஷாப்புக்குள் திருடப் போன வடிவேலு நன்றாக சரக்கடித்து விட்டு, ஒயின்ஷாப் ஓனருக்கு போன் ப","ids":[447,246,156,106,117,156,106,110,156,107,233,220,156,106,243,156,106,110,156,107,228,156,106,243,156,107,235,156,106,253,156,106,108,156,107,235,220,156,106,248,156,106,122,156,106,108,156,106,122,1399,220,156,106,248,156,106,108,156,106,243,156,107,235,156,106,243,156,107,223,220,156,106,113,156,107,229,156,106,96,156,107,223,156,106,106,156,107,235,1399,220,156,106,243,156,106,253,156,107,230,220,156,106,236,156,106,103,156,107,235,156,106,103,220,156,106,97,156,106,123,156,106,109,156,106,103,156,107,235,156,106,103,156,107,222,156,106,247,156,107,235,156,106,243,30,447,247,198,156,106,240,156,106,108,156,107,223,220,156,106,103,156,106,253,156,106,97,156,107,235,156,106,97,156,106,123,156,106,110,156,107,235,220,156,106,240,156,106,107,156,106,123,156,106,102,156,107,235,156,106,115,156,106,122,156,106,103,156,107,235,156,106,103,156,107,223,156,106,243,156,107,235,156,106,243,156,107,223,156,106,111,156,107,235,220,156,106,97,156,106,123,156,106,108,156,107,223,156,106,253,156,106,103,156,107,235,220,156,106,103,156,107,233,156,106,102,220,156,106,113,156,106,253,156,106,123,156,106,113,156,107,229,156,106,110,156,107,223,220,156,106,101,156,106,102,156,107,235,156,106,109,156,106,122,156,106,243,220,156,106,248,156,106,108,156,106,243,156,107,235,156,106,243,156,106,253,156,106,123,156,106,97,156,107,235,156,106,97,156,107,223,220,156,106,113,156,106,123,156,106,253,156,107,235,156,106,253,156,107,223,11,220,156,106,240,156,106,107,156,106,123,156,106,102,156,107,235,156,106,115,156,106,122,156,106,103,156,107,235,220,156,106,241,156,106,102,156,106,108,156,107,223,156,106,243,156,107,235,156,106,243,156,107,223,220,156,106,103,156,107,233,156,106,102,156,107,235,220,156,106,103],"ids_no_specials":[447,246,156,106,117,156,106,110,156,107,233,220,156,106,243,156,106,110,156,107,228,156,106,243,156,107,235,156,106,253,156,106,108,156,107,235,220,156,106,248,156,106,122,156,106,108,156,106,122,1399,220,156,106,248,156,106,108,156,106,243,156,107,235,156,106,243,156,107,223,220,156,106,113,156,107,229,156,106,96,156,107,223,156,106,106,156,107,235,1399,220,156,106,243,156,106,253,156,107,230,220,156,106,236,156,106,103,156,107,235,156,106,103,220,156,106,97,156,106,123,156,106,109,156,106,103,156,107,235,156,106,103,156,107,222,156,106,247,156,107,235,156,106,243,30,447,247,198,156,106,240,156,106,108,156,107,223,220,156,106,103,156,106,253,156,106,97,156,107,235,156,106,97,156,106,123,156,106,110,156,107,235,220,156,106,240,156,106,107,156,106,123,156,106,102,156,107,235,156,106,115,156,106,122,156,106,103,156,107,235,156,106,103,156,107,223,156,106,243,156,107,235,156,106,243,156,107,223,156,106,111,156,107,235,220,156,106,97,156,106,123,156,106,108,156,107,223,156,106,253,156,106,103,156,107,235,220,156,106,103,156,107,233,156,106,102,220,156,106,113,156,106,253,156,106,123,156,106,113,156,107,229,156,106,110,156,107,223,220,156,106,101,156,106,102,156,107,235,156,106,109,156,106,122,156,106,243,220,156,106,248,156,106,108,156,106,243,156,107,235,156,106,243,156,106,253,156,106,123,156,106,97,156,107,235,156,106,97,156,107,223,220,156,106,113,156,106,123,156,106,253,156,107,235,156,106,253,156,107,223,11,220,156,106,240,156,106,107,156,106,123,156,106,102,156,107,235,156,106,115,156,106,122,156,106,103,156,107,235,220,156,106,241,156,106,102,156,106,108,156,107,223,156,106,243,156,107,235,156,106,243,156,107,223,220,156,106,103,156,107,233,156,106,102,156,107,235,220,156,106,103],"tokens":["âĢ","ĺ","à","®","¹","à","®","²","à","¯","ĭ","Ġ","à","®","ķ","à","®","²","à","¯","Ĩ","à","®","ķ","à","¯","į","à","®","Ł","à","®","°","à","¯","į","Ġ","à","®","ļ","à","®","¾","à","®","°","à","®","¾","âĢ¦","Ġ","à","®","ļ","à","®","°","à","®","ķ","à","¯","į","à","®","ķ","à","¯","ģ","Ġ","à","®","µ","à","¯","ĩ","à","®","£","à","¯","ģ","à","®","®","à","¯","į","âĢ¦","Ġ","à","®","ķ","à","®","Ł","à","¯","Ī","Ġ","à","®","İ","à","®","ª","à","¯","į","à","®","ª","Ġ","à","®","¤","à","®","¿","à","®","±","à","®","ª","à","¯","į","à","®","ª","à","¯","Ģ","à","®","Ļ","à","¯","į","à","®","ķ","?","âĢ","Ļ","Ċ","à","®","Ĵ","à","®","°","à","¯","ģ","Ġ","à","®","ª","à","®","Ł","à","®","¤","à","¯","į","à","®","¤","à","®","¿","à","®","²","à","¯","į","Ġ","à","®","Ĵ","à","®","¯","à","®","¿","à","®","©","à","¯","į","à","®","·","à","®","¾","à","®","ª","à","¯","į","à","®","ª","à","¯","ģ","à","®","ķ","à","¯","į","à","®","ķ","à","¯","ģ","à","®","³","à","¯","į","Ġ","à","®","¤","à","®","¿","à","®","°","à","¯","ģ","à","®","Ł","à","®","ª","à","¯","į","Ġ","à","®","ª","à","¯","ĭ","à","®","©","Ġ","à","®","µ","à","®","Ł","à","®","¿","à","®","µ","à","¯","ĩ","à","®","²","à","¯","ģ","Ġ","à","®","¨","à","®","©","à","¯","į","à","®","±","à","®","¾","à","®","ķ","Ġ","à","®","ļ","à","®","°","à","®","ķ","à","¯","į","à","®","ķ","à","®","Ł","à","®","¿","à","®","¤","à","¯","į","à","®","¤","à","¯","ģ","Ġ","à","®","µ","à","®","¿","à","®","Ł","à","¯","į","à","®","Ł","à","¯","ģ",",","Ġ","à","®","Ĵ","à","®","¯","à","®","¿","à","®","©","à","¯","į","à","®","·","à","®","¾","à","®","ª","à","¯","į","Ġ","à","®","ĵ","à","®","©","à","®","°","à","¯","ģ","à","®","ķ","à","¯","į","à","®","ķ","à","¯","ģ","Ġ","à","®","ª","à","¯","ĭ","à","®","©","à","¯","į","Ġ","à","®","ª"],"offsets":[[0,1],[0,1],[1,2],[1,2],[1,2],[2,3],[2,3],[2,3],[3,4],[3,4],[3,4],[4,5],[5,6],[5,6],[5,6],[6,7],[6,7],[6,7],[7,8],[7,8],[7,8],[8,9],[8,9],[8,9],[9,10],[9,10],[9,10],[10,11],[10,11],[10,11],[11,12],[11,12],[11,12],[12,13],[12,13],[12,13],[13,14],[14,15],[14,15],[14,15],[15,16],[15,16],[15,16],[16,17],[16,17],[16,17],[17,18],[17,18],[17,18],[18,19],[19,20],[20,21],[20,21],[20,21],[21,22],[21,22],[21,22],[22,23],[22,23],[22,23],[23,24],[23,24],[23,24],[24,25],[24,25],[24,25],[25,26],[25,26],[25,26],[26,27],[27,28],[27,28],[27,28],[28,29],[28,29],[28,29],[29,30],[29,30],[29,30],[30,31],[30,31],[30,31],[31,32],[31,32],[31,32],[32,33],[32,33],[32,33],[33,34],[34,35],[35,36],[35,36],[35,36],[36,37],[36,37],[36,37],[37,38],[37,38],[37,38],[38,39],[39,40],[39,40],[39,40],[40,41],[40,41],[40,41],[41,42],[41,42],[41,42],[42,43],[42,43],[42,43],[43,44],[44,45],[44,45],[44,45],[45,46],[45,46],[45,46],[46,47],[46,47],[46,47],[47,48],[47,48],[47,48],[48,49],[48,49],[48,49],[49,50],[49,50],[49,50],[50,51],[50,51],[50,51],[51,52],[51,52],[51,52],[52,53],[52,53],[52,53],[53,54],[53,54],[53,54],[54,55],[55,56],[55,56],[56,57],[57,58],[57,58],[57,58],[58,59],[58,59],[58,59],[59,60],[59,60],[59,60],[60,61],[61,62],[61,62],[61,62],[62,63],[62,63],[62,63],[63,64],[63,64],[63,64],[64,65],[64,65],[64,65],[65,66],[65,66],[65,66],[66,67],[66,67],[66,67],[67,68],[67,68],[67,68],[68,69],[68,69],[68,69],[69,70],[70,71],[70,71],[70,71],[71,72],[71,72],[71,72],[72,73],[72,73],[72,73],[73,74],[73,74],[73,74],[74,75],[74,75],[74,75],[75,76],[75,76],[75,76],[76,77],[76,77],[76,77],[77,78],[77,78],[77,78],[78,79],[78,79],[78,79],[79,80],[79,80],[79,80],[80,81],[80,81],[80,81],[81,82],[81,82],[81,82],[82,83],[82,83],[82,83],[83,84],[83,84],[83,84],[84,85],[84,85],[84,85],[85,86],[85,86],[85,86],[86,87],[86,87],[86,87],[87,88],[88,89],[88,89],[88,89],[89,90],[89,90],[89,90],[90,91],[90,91],[90,91],[91,92],[91,92],[91,92],[92,93],[92,93],[92,93],[93,94],[93,94],[93,94],[94,95],[94,95],[94,95],[95,96],[96,97],[96,97],[96,97],[97,98],[97,98],[97,98],[98,99],[98,99],[98,99],[99,100],[100,101],[100,101],[100,101],[101,102],[101,102],[101,102],[102,103],[102,103],[102,103],[103,104],[103,104],[103,104],[104,105],[104,105],[104,105],[105,106],[105,106],[105,106],[106,107],[106,107],[106,107],[107,108],[108,109],[108,109],[108,109],[109,110],[109,110],[109,110],[110,111],[110,111],[110,111],[111,112],[111,112],[111,112],[112,113],[112,113],[112,113],[113,114],[113,114],[113,114],[114,115],[115,116],[115,116],[115,116],[116,117],[116,117],[116,117],[117,118],[117,118],[117,118],[118,119],[118,119],[118,119],[119,120],[119,120],[119,120],[120,121],[120,121],[120,121],[121,122],[121,122],[121,122],[122,123],[122,123],[122,123],[123,124],[123,124],[123,124],[124,125],[124,125],[124,125],[125,126],[125,126],[125,126],[126,127],[127,128],[127,128],[127,128],[128,129],[128,129],[128,129],[129,130],[129,130],[129,130],[130,131],[130,131],[130,131],[131,132],[131,132],[131,132],[132,133],[132,133],[132,133],[133,134],[134,135],[135,136],[135,136],[135,136],[136,137],[136,137],[136,137],[137,138],[137,138],[137,138],[138,139],[138,139],[138,139],[139,140],[139,140],[139,140],[140,141],[140,141],[140,141],[141,142],[141,142],[141,142],[142,143],[142,143],[142,143],[143,144],[143,144],[143,144],[144,145],[145,146],[145,146],[145,146],[146,147],[146,147],[146,147],[147,148],[147,148],[147,148],[148,149],[148,149],[148,149],[149,150],[149,150],[149,150],[150,151],[150,151],[150,151],[151,152],[151,152],[151,152],[152,153],[152,153],[152,153],[153,154],[154,155],[154,155],[154,155],[155,156],[155,156],[155,156],[156,157],[156,157],[156,157],[157,158],[157,158],[157,158],[158,159],[159,160],[159,160],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,1,1,1,1,1,1,2,2,2,3,3,3,3,3,3,3,4,4,4,5,5,5,6,6,6,7,7,7,7,7,7,8,8,8,9,9,9,9,10,10,10,11,11,11,12,12,12,12,13,13,13,13,13,13,13,13,13,13,14,14,14,15,15,15,16,16,16,17,17,17,17,18,18,18,19,19,19,20,20,20,21,21,21,22,22,22,22,23,23,23,23,23,23,23,24,24,24,25,25,25,25,25,25,25,26,26,26,27,27,27,28,28,28,28,29,29,29,30,30,30,30,30,30,31,31,31,32,32,32,33,33,33,34,34,34,35,35,35,36,36,36,37,37,37,38,39,39,39,39,39,39,40,40,40,41,41,41,41,41,41,41,41,41,41,42,42,42,43,43,43,44,44,44,45,45,45,46,46,46,47,47,47,47,47,47,47,48,48,48,49,49,49,50,50,50,51,51,51,52,52,52,53,53,53,54,54,54,55,55,55,56,56,56,57,57,57,58,58,58,59,59,59,60,60,60,61,61,61,62,62,62,63,63,63,63,64,64,64,65,65,65,66,66,66,67,67,67,67,67,67,68,68,68,69,69,69,69,70,70,70,71,71,71,72,72,72,72,72,72,72,73,73,73,74,74,74,75,75,75,76,76,76,77,77,77,78,78,78,78,78,78,78,79,79,79,80,80,80,81,81,81,82,82,82,83,83,83,83,83,83,83,83,83,83,84,84,84,85,85,85,85,85,85,86,86,86,87,87,87,88,88,88,89,89,89,90,90,90,91,91,91,91,92,92,92,93,93,93,94,94,94,95,95,95,96,96,96,96,97,97,97,97,97,97,97,98,98,98,99,99,99,100,100,100,101,101,101,102,102,102,103,103,103,104,104,104,105,105,105,105,105,105,105,105,105,105,106,106,106,107,107,107,108,108,108,109,109,109,110,110,110,111,111,111,111,112,112,112,113,113,113,114,114,114,115,115,115,115],"decoded":"‘ஹலோ கலெக்டர் சாரா… சரக்கு வேணும்… கடை எப்ப திறப்பீங்க?’\nஒரு படத்தில் ஒயின்ஷாப்புக்குள் திருடப் போன வடிவேலு நன்றாக சரக்கடித்து விட்டு, ஒயின்ஷாப் ஓனருக்கு போன் ப","decoded_with_specials":"‘ஹலோ கலெக்டர் சாரா… சரக்கு வேணும்… கடை எப்ப திறப்பீங்க?’\nஒரு படத்தில் ஒயின்ஷாப்புக்குள் திருடப் போன வடிவேலு நன்றாக சரக்கடித்து விட்டு, ஒயின்ஷாப் ஓனருக்கு போன் ப"} +{"kind":"sample","source":"fixtures/lang/tha_Thai.txt[:160]","text":"นับเจดีย์ดูนะครับว่าครบยี่สิบองค์รึเปล่า คำว่าซาว ทางเหนือแปลว่่ายี่สิบครับ ผมนับดูแล้วก็ครบนะ ลองดู เสร็จแล้วก็เข้าเยี่ยมชมพิพธภัณฑ์ เดินเข้าไปด้านในหน่อยก็จะม","ids":[19567,247,19567,109,19567,248,31479,222,19567,230,19567,242,19567,113,19567,95,31479,234,19567,242,19567,117,19567,247,19567,108,19567,226,19567,96,19567,109,19567,248,19567,100,31479,230,19567,110,19567,226,19567,96,19567,248,19567,95,19567,113,31479,230,19567,103,19567,112,19567,248,19567,255,19567,229,19567,226,31479,234,19567,96,19567,114,31479,222,19567,249,19567,98,31479,230,19567,110,220,19567,226,19567,111,19567,100,31479,230,19567,110,19567,233,19567,110,19567,100,220,19567,245,19567,110,19567,229,31479,222,19567,104,19567,247,19567,115,19567,255,31479,223,19567,249,19567,98,19567,100,31479,230,31479,230,19567,110,19567,95,19567,113,31479,230,19567,103,19567,112,19567,248,19567,226,19567,96,19567,109,19567,248,220,19567,250,19567,94,19567,247,19567,109,19567,248,19567,242,19567,117,31479,223,19567,98,31479,231,19567,100,19567,223,31479,229,19567,226,19567,96,19567,248,19567,247,19567,108,220,19567,98,19567,255,19567,229,19567,242,19567,117,220,31479,222,19567,103,19567,96,31479,229,19567,230,31479,223,19567,98,31479,231,19567,100,19567,223,31479,229,31479,222,19567,224,31479,231,19567,110,31479,222,19567,95,19567,113,31479,230,19567,95,19567,94,19567,232,19567,94,19567,252,19567,112,19567,252,19567,246,19567,254,19567,109,19567,241,19567,239,31479,234,220,31479,222,19567,242,19567,112,19567,247,31479,222,19567,224,31479,231,19567,110,31479,226,19567,249,19567,242,31479,231,19567,110,19567,247,31479,225,19567,247,19567,104,19567,247,31479,230,19567,255,19567,95,19567,223,31479,229,19567,230,19567,108,19567,94],"ids_no_specials":[19567,247,19567,109,19567,248,31479,222,19567,230,19567,242,19567,113,19567,95,31479,234,19567,242,19567,117,19567,247,19567,108,19567,226,19567,96,19567,109,19567,248,19567,100,31479,230,19567,110,19567,226,19567,96,19567,248,19567,95,19567,113,31479,230,19567,103,19567,112,19567,248,19567,255,19567,229,19567,226,31479,234,19567,96,19567,114,31479,222,19567,249,19567,98,31479,230,19567,110,220,19567,226,19567,111,19567,100,31479,230,19567,110,19567,233,19567,110,19567,100,220,19567,245,19567,110,19567,229,31479,222,19567,104,19567,247,19567,115,19567,255,31479,223,19567,249,19567,98,19567,100,31479,230,31479,230,19567,110,19567,95,19567,113,31479,230,19567,103,19567,112,19567,248,19567,226,19567,96,19567,109,19567,248,220,19567,250,19567,94,19567,247,19567,109,19567,248,19567,242,19567,117,31479,223,19567,98,31479,231,19567,100,19567,223,31479,229,19567,226,19567,96,19567,248,19567,247,19567,108,220,19567,98,19567,255,19567,229,19567,242,19567,117,220,31479,222,19567,103,19567,96,31479,229,19567,230,31479,223,19567,98,31479,231,19567,100,19567,223,31479,229,31479,222,19567,224,31479,231,19567,110,31479,222,19567,95,19567,113,31479,230,19567,95,19567,94,19567,232,19567,94,19567,252,19567,112,19567,252,19567,246,19567,254,19567,109,19567,241,19567,239,31479,234,220,31479,222,19567,242,19567,112,19567,247,31479,222,19567,224,31479,231,19567,110,31479,226,19567,249,19567,242,31479,231,19567,110,19567,247,31479,225,19567,247,19567,104,19567,247,31479,230,19567,255,19567,95,19567,223,31479,229,19567,230,19567,108,19567,94],"tokens":["à¸","Ļ","à¸","±","à¸","ļ","à¹","Ģ","à¸","Ī","à¸","Ķ","à¸","µ","à¸","¢","à¹","Į","à¸","Ķ","à¸","¹","à¸","Ļ","à¸","°","à¸","Ħ","à¸","£","à¸","±","à¸","ļ","à¸","§","à¹","Ī","à¸","²","à¸","Ħ","à¸","£","à¸","ļ","à¸","¢","à¸","µ","à¹","Ī","à¸","ª","à¸","´","à¸","ļ","à¸","Ń","à¸","ĩ","à¸","Ħ","à¹","Į","à¸","£","à¸","¶","à¹","Ģ","à¸","Ľ","à¸","¥","à¹","Ī","à¸","²","Ġ","à¸","Ħ","à¸","³","à¸","§","à¹","Ī","à¸","²","à¸","ĭ","à¸","²","à¸","§","Ġ","à¸","Ĺ","à¸","²","à¸","ĩ","à¹","Ģ","à¸","«","à¸","Ļ","à¸","·","à¸","Ń","à¹","ģ","à¸","Ľ","à¸","¥","à¸","§","à¹","Ī","à¹","Ī","à¸","²","à¸","¢","à¸","µ","à¹","Ī","à¸","ª","à¸","´","à¸","ļ","à¸","Ħ","à¸","£","à¸","±","à¸","ļ","Ġ","à¸","ľ","à¸","¡","à¸","Ļ","à¸","±","à¸","ļ","à¸","Ķ","à¸","¹","à¹","ģ","à¸","¥","à¹","ī","à¸","§","à¸","ģ","à¹","ĩ","à¸","Ħ","à¸","£","à¸","ļ","à¸","Ļ","à¸","°","Ġ","à¸","¥","à¸","Ń","à¸","ĩ","à¸","Ķ","à¸","¹","Ġ","à¹","Ģ","à¸","ª","à¸","£","à¹","ĩ","à¸","Ī","à¹","ģ","à¸","¥","à¹","ī","à¸","§","à¸","ģ","à¹","ĩ","à¹","Ģ","à¸","Ĥ","à¹","ī","à¸","²","à¹","Ģ","à¸","¢","à¸","µ","à¹","Ī","à¸","¢","à¸","¡","à¸","Ĭ","à¸","¡","à¸","ŀ","à¸","´","à¸","ŀ","à¸","ĺ","à¸","ł","à¸","±","à¸","ĵ","à¸","ij","à¹","Į","Ġ","à¹","Ģ","à¸","Ķ","à¸","´","à¸","Ļ","à¹","Ģ","à¸","Ĥ","à¹","ī","à¸","²","à¹","Ħ","à¸","Ľ","à¸","Ķ","à¹","ī","à¸","²","à¸","Ļ","à¹","ĥ","à¸","Ļ","à¸","«","à¸","Ļ","à¹","Ī","à¸","Ń","à¸","¢","à¸","ģ","à¹","ĩ","à¸","Ī","à¸","°","à¸","¡"],"offsets":[[0,1],[0,1],[1,2],[1,2],[2,3],[2,3],[3,4],[3,4],[4,5],[4,5],[5,6],[5,6],[6,7],[6,7],[7,8],[7,8],[8,9],[8,9],[9,10],[9,10],[10,11],[10,11],[11,12],[11,12],[12,13],[12,13],[13,14],[13,14],[14,15],[14,15],[15,16],[15,16],[16,17],[16,17],[17,18],[17,18],[18,19],[18,19],[19,20],[19,20],[20,21],[20,21],[21,22],[21,22],[22,23],[22,23],[23,24],[23,24],[24,25],[24,25],[25,26],[25,26],[26,27],[26,27],[27,28],[27,28],[28,29],[28,29],[29,30],[29,30],[30,31],[30,31],[31,32],[31,32],[32,33],[32,33],[33,34],[33,34],[34,35],[34,35],[35,36],[35,36],[36,37],[36,37],[37,38],[37,38],[38,39],[38,39],[39,40],[39,40],[40,41],[41,42],[41,42],[42,43],[42,43],[43,44],[43,44],[44,45],[44,45],[45,46],[45,46],[46,47],[46,47],[47,48],[47,48],[48,49],[48,49],[49,50],[50,51],[50,51],[51,52],[51,52],[52,53],[52,53],[53,54],[53,54],[54,55],[54,55],[55,56],[55,56],[56,57],[56,57],[57,58],[57,58],[58,59],[58,59],[59,60],[59,60],[60,61],[60,61],[61,62],[61,62],[62,63],[62,63],[63,64],[63,64],[64,65],[64,65],[65,66],[65,66],[66,67],[66,67],[67,68],[67,68],[68,69],[68,69],[69,70],[69,70],[70,71],[70,71],[71,72],[71,72],[72,73],[72,73],[73,74],[73,74],[74,75],[74,75],[75,76],[76,77],[76,77],[77,78],[77,78],[78,79],[78,79],[79,80],[79,80],[80,81],[80,81],[81,82],[81,82],[82,83],[82,83],[83,84],[83,84],[84,85],[84,85],[85,86],[85,86],[86,87],[86,87],[87,88],[87,88],[88,89],[88,89],[89,90],[89,90],[90,91],[90,91],[91,92],[91,92],[92,93],[92,93],[93,94],[93,94],[94,95],[95,96],[95,96],[96,97],[96,97],[97,98],[97,98],[98,99],[98,99],[99,100],[99,100],[100,101],[101,102],[101,102],[102,103],[102,103],[103,104],[103,104],[104,105],[104,105],[105,106],[105,106],[106,107],[106,107],[107,108],[107,108],[108,109],[108,109],[109,110],[109,110],[110,111],[110,111],[111,112],[111,112],[112,113],[112,113],[113,114],[113,114],[114,115],[114,115],[115,116],[115,116],[116,117],[116,117],[117,118],[117,118],[118,119],[118,119],[119,120],[119,120],[120,121],[120,121],[121,122],[121,122],[122,123],[122,123],[123,124],[123,124],[124,125],[124,125],[125,126],[125,126],[126,127],[126,127],[127,128],[127,128],[128,129],[128,129],[129,130],[129,130],[130,131],[130,131],[131,132],[131,132],[132,133],[132,133],[133,134],[134,135],[134,135],[135,136],[135,136],[136,137],[136,137],[137,138],[137,138],[138,139],[138,139],[139,140],[139,140],[140,141],[140,141],[141,142],[141,142],[142,143],[142,143],[143,144],[143,144],[144,145],[144,145],[145,146],[145,146],[146,147],[146,147],[147,148],[147,148],[148,149],[148,149],[149,150],[149,150],[150,151],[150,151],[151,152],[151,152],[152,153],[152,153],[153,154],[153,154],[154,155],[154,155],[155,156],[155,156],[156,157],[156,157],[157,158],[157,158],[158,159],[158,159],[159,160],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,1,1,2,2,2,2,2,2,2,2,3,3,4,4,5,5,6,6,7,7,8,8,8,8,8,8,8,8,9,9,10,10,10,10,11,11,12,12,12,12,12,12,12,12,12,12,13,13,13,13,14,14,15,15,16,16,16,16,16,16,16,16,17,17,18,18,19,19,20,20,20,20,20,20,21,21,22,22,23,23,23,23,23,23,23,24,24,25,25,25,25,25,25,25,25,26,26,26,26,26,26,26,26,26,26,26,26,26,27,27,28,28,28,28,28,28,28,28,28,28,29,29,29,29,30,30,30,30,31,31,31,31,32,32,33,33,34,34,34,34,34,34,35,35,36,36,37,37,37,37,37,37,37,38,38,39,39,39,39,40,40,41,41,41,41,42,42,43,43,43,43,44,44,45,45,45,45,45,45,45,45,45,45,46,46,46,46,46,46,46,46,46,47,47,48,48,48,48,48,48,48,49,49,50,50,50,50,50,50,51,51,52,52,52,52,53,53,54,54,54,54,55,55,56,56,56,56,56,56,57,57,57,57,58,58,58,58,58,58,58,58,58,58,59,59,60,60,60,60,60,60,61,61,62,62,62,62,63,63,64,64,64,64,64,65,65,66,66,66,66,66,66,67,67,68,68,68,68,68,68,68,68,69,69,70,70,70,70,70,70,70,70,70,70,70,70,71,71,72,72,72,72,72,72,73,73,74,74,74,74,74,74],"decoded":"นับเจดีย์ดูนะครับว่าครบยี่สิบองค์รึเปล่า คำว่าซาว ทางเหนือแปลว่่ายี่สิบครับ ผมนับดูแล้วก็ครบนะ ลองดู เสร็จแล้วก็เข้าเยี่ยมชมพิพธภัณฑ์ เดินเข้าไปด้านในหน่อยก็จะม","decoded_with_specials":"นับเจดีย์ดูนะครับว่าครบยี่สิบองค์รึเปล่า คำว่าซาว ทางเหนือแปลว่่ายี่สิบครับ ผมนับดูแล้วก็ครบนะ ลองดู เสร็จแล้วก็เข้าเยี่ยมชมพิพธภัณฑ์ เดินเข้าไปด้านในหน่อยก็จะม"} +{"kind":"sample","source":"fixtures/modalities/added_normalized_dense.txt[:160]","text":"FLIBBERJAST CRUNGLEDORF FLIBBERJAST SNORLAXIAN fast BLORPTRONIC ZORPTASTIC split WIDGETRON split stage SNORLAXIAN BLORPTRONIC BLORPTRONIC \n QUIBBLENAUT CRUNGLED","ids":[3697,9865,13246,41,11262,8740,4944,8763,1961,1581,37,9977,9865,13246,41,11262,11346,1581,13534,55,16868,3049,9878,1581,47,5446,1340,2149,1168,1581,11571,11262,2149,6626,370,2389,18851,45806,6626,3800,11346,1581,13534,55,16868,9878,1581,47,5446,1340,2149,9878,1581,47,5446,1340,2149,220,198,19604,9865,9148,1677,39371,8740,4944,8763,1961],"ids_no_specials":[3697,9865,13246,41,11262,8740,4944,8763,1961,1581,37,9977,9865,13246,41,11262,11346,1581,13534,55,16868,3049,9878,1581,47,5446,1340,2149,1168,1581,11571,11262,2149,6626,370,2389,18851,45806,6626,3800,11346,1581,13534,55,16868,9878,1581,47,5446,1340,2149,9878,1581,47,5446,1340,2149,220,198,19604,9865,9148,1677,39371,8740,4944,8763,1961],"tokens":["FL","IB","BER","J","AST","ĠCR","UN","GL","ED","OR","F","ĠFL","IB","BER","J","AST","ĠSN","OR","LA","X","IAN","Ġfast","ĠBL","OR","P","TR","ON","IC","ĠZ","OR","PT","AST","IC","Ġsplit","ĠW","ID","GET","RON","Ġsplit","Ġstage","ĠSN","OR","LA","X","IAN","ĠBL","OR","P","TR","ON","IC","ĠBL","OR","P","TR","ON","IC","Ġ","Ċ","ĠQU","IB","BL","EN","AUT","ĠCR","UN","GL","ED"],"offsets":[[0,2],[2,4],[4,7],[7,8],[8,11],[11,14],[14,16],[16,18],[18,20],[20,22],[22,23],[23,26],[26,28],[28,31],[31,32],[32,35],[35,38],[38,40],[40,42],[42,43],[43,46],[46,51],[51,54],[54,56],[56,57],[57,59],[59,61],[61,63],[63,65],[65,67],[67,69],[69,72],[72,74],[74,80],[80,82],[82,84],[84,87],[87,90],[90,96],[96,102],[102,105],[105,107],[107,109],[109,110],[110,113],[113,116],[116,118],[118,119],[119,121],[121,123],[123,125],[125,128],[128,130],[130,131],[131,133],[133,135],[135,137],[137,138],[138,139],[139,142],[142,144],[144,146],[146,148],[148,151],[151,154],[154,156],[156,158],[158,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,1,1,1,1,1,1,2,2,2,2,2,3,3,3,3,3,4,5,5,5,5,5,5,6,6,6,6,6,7,8,8,8,8,9,10,11,11,11,11,11,12,12,12,12,12,12,13,13,13,13,13,13,14,14,15,15,15,15,15,16,16,16,16],"decoded":"FLIBBERJAST CRUNGLEDORF FLIBBERJAST SNORLAXIAN fast BLORPTRONIC ZORPTASTIC split WIDGETRON split stage SNORLAXIAN BLORPTRONIC BLORPTRONIC \n QUIBBLENAUT CRUNGLED","decoded_with_specials":"FLIBBERJAST CRUNGLEDORF FLIBBERJAST SNORLAXIAN fast BLORPTRONIC ZORPTASTIC split WIDGETRON split stage SNORLAXIAN BLORPTRONIC BLORPTRONIC \n QUIBBLENAUT CRUNGLED"} +{"kind":"sample","source":"fixtures/modalities/added_normalized_sparse.txt[:160]","text":"ZORPTASTIC for we for decode CRUNGLEDORF here through ZORPTASTIC and the we WIDGETRON model \n normalize bytes and back flows text CRUNGLEDORF WUZZLEFANG chunk b","ids":[57,1581,11571,11262,2149,329,356,329,36899,8740,4944,8763,1961,1581,37,994,832,1168,1581,11571,11262,2149,290,262,356,370,2389,18851,45806,2746,220,198,3487,1096,9881,290,736,15623,2420,8740,4944,8763,1961,1581,37,370,52,30148,2538,37,15567,16058,275],"ids_no_specials":[57,1581,11571,11262,2149,329,356,329,36899,8740,4944,8763,1961,1581,37,994,832,1168,1581,11571,11262,2149,290,262,356,370,2389,18851,45806,2746,220,198,3487,1096,9881,290,736,15623,2420,8740,4944,8763,1961,1581,37,370,52,30148,2538,37,15567,16058,275],"tokens":["Z","OR","PT","AST","IC","Ġfor","Ġwe","Ġfor","Ġdecode","ĠCR","UN","GL","ED","OR","F","Ġhere","Ġthrough","ĠZ","OR","PT","AST","IC","Ġand","Ġthe","Ġwe","ĠW","ID","GET","RON","Ġmodel","Ġ","Ċ","Ġnormal","ize","Ġbytes","Ġand","Ġback","Ġflows","Ġtext","ĠCR","UN","GL","ED","OR","F","ĠW","U","ZZ","LE","F","ANG","Ġchunk","Ġb"],"offsets":[[0,1],[1,3],[3,5],[5,8],[8,10],[10,14],[14,17],[17,21],[21,28],[28,31],[31,33],[33,35],[35,37],[37,39],[39,40],[40,45],[45,53],[53,55],[55,57],[57,59],[59,62],[62,64],[64,68],[68,72],[72,75],[75,77],[77,79],[79,82],[82,85],[85,91],[91,92],[92,93],[93,100],[100,103],[103,109],[109,113],[113,118],[118,124],[124,129],[129,132],[132,134],[134,136],[136,138],[138,140],[140,141],[141,143],[143,144],[144,146],[146,148],[148,149],[149,152],[152,158],[158,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,0,0,0,1,2,3,4,5,5,5,5,5,5,6,7,8,8,8,8,8,9,10,11,12,12,12,12,13,14,14,15,15,16,17,18,19,20,21,21,21,21,21,21,22,22,22,22,22,22,23,24],"decoded":"ZORPTASTIC for we for decode CRUNGLEDORF here through ZORPTASTIC and the we WIDGETRON model \n normalize bytes and back flows text CRUNGLEDORF WUZZLEFANG chunk b","decoded_with_specials":"ZORPTASTIC for we for decode CRUNGLEDORF here through ZORPTASTIC and the we WIDGETRON model \n normalize bytes and back flows text CRUNGLEDORF WUZZLEFANG chunk b"} +{"kind":"sample","source":"fixtures/modalities/added_special_dense.txt[:160]","text":"fast <|xs2|> chunk <|xs1|> <|xs4|> <|xs3|> <|xs4|> modality language <|xs0|> reads <|xs2|> and and \n <|xs1|> <|xs0|> <|xs1|> <|xs4|> <|xs4|> <|xs1|> back and re","ids":[7217,1279,91,34223,17,91,29,16058,1279,91,34223,16,91,29,1279,91,34223,19,91,29,1279,91,34223,18,91,29,1279,91,34223,19,91,29,953,1483,3303,1279,91,34223,15,91,29,9743,1279,91,34223,17,91,29,290,290,220,198,1279,91,34223,16,91,29,1279,91,34223,15,91,29,1279,91,34223,16,91,29,1279,91,34223,19,91,29,1279,91,34223,19,91,29,1279,91,34223,16,91,29,736,290,302],"ids_no_specials":[7217,1279,91,34223,17,91,29,16058,1279,91,34223,16,91,29,1279,91,34223,19,91,29,1279,91,34223,18,91,29,1279,91,34223,19,91,29,953,1483,3303,1279,91,34223,15,91,29,9743,1279,91,34223,17,91,29,290,290,220,198,1279,91,34223,16,91,29,1279,91,34223,15,91,29,1279,91,34223,16,91,29,1279,91,34223,19,91,29,1279,91,34223,19,91,29,1279,91,34223,16,91,29,736,290,302],"tokens":["fast","Ġ<","|","xs","2","|",">","Ġchunk","Ġ<","|","xs","1","|",">","Ġ<","|","xs","4","|",">","Ġ<","|","xs","3","|",">","Ġ<","|","xs","4","|",">","Ġmod","ality","Ġlanguage","Ġ<","|","xs","0","|",">","Ġreads","Ġ<","|","xs","2","|",">","Ġand","Ġand","Ġ","Ċ","Ġ<","|","xs","1","|",">","Ġ<","|","xs","0","|",">","Ġ<","|","xs","1","|",">","Ġ<","|","xs","4","|",">","Ġ<","|","xs","4","|",">","Ġ<","|","xs","1","|",">","Ġback","Ġand","Ġre"],"offsets":[[0,4],[4,6],[6,7],[7,9],[9,10],[10,11],[11,12],[12,18],[18,20],[20,21],[21,23],[23,24],[24,25],[25,26],[26,28],[28,29],[29,31],[31,32],[32,33],[33,34],[34,36],[36,37],[37,39],[39,40],[40,41],[41,42],[42,44],[44,45],[45,47],[47,48],[48,49],[49,50],[50,54],[54,59],[59,68],[68,70],[70,71],[71,73],[73,74],[74,75],[75,76],[76,82],[82,84],[84,85],[85,87],[87,88],[88,89],[89,90],[90,94],[94,98],[98,99],[99,100],[100,102],[102,103],[103,105],[105,106],[106,107],[107,108],[108,110],[110,111],[111,113],[113,114],[114,115],[115,116],[116,118],[118,119],[119,121],[121,122],[122,123],[123,124],[124,126],[126,127],[127,129],[129,130],[130,131],[131,132],[132,134],[134,135],[135,137],[137,138],[138,139],[139,140],[140,142],[142,143],[143,145],[145,146],[146,147],[147,148],[148,153],[153,157],[157,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,1,1,2,3,4,4,5,6,6,7,8,9,9,10,10,11,12,13,13,14,14,15,16,17,17,18,18,19,20,21,21,22,22,23,24,24,25,26,27,27,28,29,29,30,31,32,32,33,34,35,35,36,36,37,38,39,39,40,40,41,42,43,43,44,44,45,46,47,47,48,48,49,50,51,51,52,52,53,54,55,55,56,56,57,58,59,59,60,61,62],"decoded":"fast <|xs2|> chunk <|xs1|> <|xs4|> <|xs3|> <|xs4|> modality language <|xs0|> reads <|xs2|> and and \n <|xs1|> <|xs0|> <|xs1|> <|xs4|> <|xs4|> <|xs1|> back and re","decoded_with_specials":"fast <|xs2|> chunk <|xs1|> <|xs4|> <|xs3|> <|xs4|> modality language <|xs0|> reads <|xs2|> and and \n <|xs1|> <|xs0|> <|xs1|> <|xs4|> <|xs4|> <|xs1|> back and re"} +{"kind":"sample","source":"fixtures/modalities/added_special_sparse.txt[:160]","text":"<|xs0|> every for encode <|xs0|> bytes the normalize decode text model <|xs4|> <|xs3|> and \n and <|xs3|> here <|xs1|> again model here text <|xs2|> <|xs2|> lang","ids":[27,91,34223,15,91,29,790,329,37773,1279,91,34223,15,91,29,9881,262,3487,1096,36899,2420,2746,1279,91,34223,19,91,29,1279,91,34223,18,91,29,290,220,198,290,1279,91,34223,18,91,29,994,1279,91,34223,16,91,29,757,2746,994,2420,1279,91,34223,17,91,29,1279,91,34223,17,91,29,42392],"ids_no_specials":[27,91,34223,15,91,29,790,329,37773,1279,91,34223,15,91,29,9881,262,3487,1096,36899,2420,2746,1279,91,34223,19,91,29,1279,91,34223,18,91,29,290,220,198,290,1279,91,34223,18,91,29,994,1279,91,34223,16,91,29,757,2746,994,2420,1279,91,34223,17,91,29,1279,91,34223,17,91,29,42392],"tokens":["<","|","xs","0","|",">","Ġevery","Ġfor","Ġencode","Ġ<","|","xs","0","|",">","Ġbytes","Ġthe","Ġnormal","ize","Ġdecode","Ġtext","Ġmodel","Ġ<","|","xs","4","|",">","Ġ<","|","xs","3","|",">","Ġand","Ġ","Ċ","Ġand","Ġ<","|","xs","3","|",">","Ġhere","Ġ<","|","xs","1","|",">","Ġagain","Ġmodel","Ġhere","Ġtext","Ġ<","|","xs","2","|",">","Ġ<","|","xs","2","|",">","Ġlang"],"offsets":[[0,1],[1,2],[2,4],[4,5],[5,6],[6,7],[7,13],[13,17],[17,24],[24,26],[26,27],[27,29],[29,30],[30,31],[31,32],[32,38],[38,42],[42,49],[49,52],[52,59],[59,64],[64,70],[70,72],[72,73],[73,75],[75,76],[76,77],[77,78],[78,80],[80,81],[81,83],[83,84],[84,85],[85,86],[86,90],[90,91],[91,92],[92,96],[96,98],[98,99],[99,101],[101,102],[102,103],[103,104],[104,109],[109,111],[111,112],[112,114],[114,115],[115,116],[116,117],[117,123],[123,129],[129,134],[134,139],[139,141],[141,142],[142,144],[144,145],[145,146],[146,147],[147,149],[149,150],[150,152],[152,153],[153,154],[154,155],[155,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,1,2,3,3,4,5,6,7,7,8,9,10,10,11,12,13,13,14,15,16,17,17,18,19,20,20,21,21,22,23,24,24,25,26,26,27,28,28,29,30,31,31,32,33,33,34,35,36,36,37,38,39,40,41,41,42,43,44,44,45,45,46,47,48,48,49],"decoded":"<|xs0|> every for encode <|xs0|> bytes the normalize decode text model <|xs4|> <|xs3|> and \n and <|xs3|> here <|xs1|> again model here text <|xs2|> <|xs2|> lang","decoded_with_specials":"<|xs0|> every for encode <|xs0|> bytes the normalize decode text model <|xs4|> <|xs3|> and \n and <|xs3|> here <|xs1|> again model here text <|xs2|> <|xs2|> lang"} +{"kind":"sample","source":"fixtures/modalities/agentic-traces.txt[:160]","text":"Analyze this codebase and summarize what it is and how it works.\nI need to explore the codebase structure by reading the main entry points and configuration fil","ids":[37702,2736,428,2438,8692,290,35743,644,340,318,290,703,340,2499,13,198,40,761,284,7301,262,2438,8692,4645,416,3555,262,1388,5726,2173,290,8398,1226],"ids_no_specials":[37702,2736,428,2438,8692,290,35743,644,340,318,290,703,340,2499,13,198,40,761,284,7301,262,2438,8692,4645,416,3555,262,1388,5726,2173,290,8398,1226],"tokens":["Analy","ze","Ġthis","Ġcode","base","Ġand","Ġsummarize","Ġwhat","Ġit","Ġis","Ġand","Ġhow","Ġit","Ġworks",".","Ċ","I","Ġneed","Ġto","Ġexplore","Ġthe","Ġcode","base","Ġstructure","Ġby","Ġreading","Ġthe","Ġmain","Ġentry","Ġpoints","Ġand","Ġconfiguration","Ġfil"],"offsets":[[0,5],[5,7],[7,12],[12,17],[17,21],[21,25],[25,35],[35,40],[40,43],[43,46],[46,50],[50,54],[54,57],[57,63],[63,64],[64,65],[65,66],[66,71],[71,74],[74,82],[82,86],[86,91],[91,95],[95,105],[105,108],[108,116],[116,120],[120,125],[125,131],[131,138],[138,142],[142,156],[156,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,1,2,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,19,20,21,22,23,24,25,26,27,28,29],"decoded":"Analyze this codebase and summarize what it is and how it works.\nI need to explore the codebase structure by reading the main entry points and configuration fil","decoded_with_specials":"Analyze this codebase and summarize what it is and how it works.\nI need to explore the codebase structure by reading the main entry points and configuration fil"} +{"kind":"sample","source":"fixtures/modalities/agentic_swe.txt[:160]","text":"[system]\nYou are a helpful assistant that can interact with a computer to solve tasks.\n\n[user]\n\n/testbed\n\nI've uploaded a pytho","ids":[58,10057,60,198,1639,389,257,7613,8796,326,460,9427,351,257,3644,284,8494,8861,13,198,198,58,7220,60,198,27,25850,276,62,16624,29,198,14,9288,3077,198,3556,25850,276,62,16624,29,198,40,1053,19144,257,279,5272,78],"ids_no_specials":[58,10057,60,198,1639,389,257,7613,8796,326,460,9427,351,257,3644,284,8494,8861,13,198,198,58,7220,60,198,27,25850,276,62,16624,29,198,14,9288,3077,198,3556,25850,276,62,16624,29,198,40,1053,19144,257,279,5272,78],"tokens":["[","system","]","Ċ","You","Ġare","Ġa","Ġhelpful","Ġassistant","Ġthat","Ġcan","Ġinteract","Ġwith","Ġa","Ġcomputer","Ġto","Ġsolve","Ġtasks",".","Ċ","Ċ","[","user","]","Ċ","<","upload","ed","_","files",">","Ċ","/","test","bed","Ċ","","Ċ","I","'ve","Ġuploaded","Ġa","Ġp","yth","o"],"offsets":[[0,1],[1,7],[7,8],[8,9],[9,12],[12,16],[16,18],[18,26],[26,36],[36,41],[41,45],[45,54],[54,59],[59,61],[61,70],[70,73],[73,79],[79,85],[85,86],[86,87],[87,88],[88,89],[89,93],[93,94],[94,95],[95,96],[96,102],[102,104],[104,105],[105,110],[110,111],[111,112],[112,113],[113,117],[117,120],[120,121],[121,123],[123,129],[129,131],[131,132],[132,137],[137,138],[138,139],[139,140],[140,143],[143,152],[152,154],[154,156],[156,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,26,27,28,29,30,31,32,32,33,34,35,35,36,37,38,39,40,41,42,43,44,44,44],"decoded":"[system]\nYou are a helpful assistant that can interact with a computer to solve tasks.\n\n[user]\n\n/testbed\n\nI've uploaded a pytho","decoded_with_specials":"[system]\nYou are a helpful assistant that can interact with a computer to solve tasks.\n\n[user]\n\n/testbed\n\nI've uploaded a pytho"} +{"kind":"sample","source":"fixtures/modalities/code_mixed.txt[:160]","text":"// django-f3f9601cff03b38942e70ce8042bdfdec6002449/django/__init__.py\nfrom django.utils.version import get_version\n\nVERSION = (6, 2, 0, \"alpha\", 0)\n\n__version__","ids":[1003,42625,14208,12,69,18,69,4846,486,66,487,3070,65,29769,3682,68,2154,344,1795,3682,65,7568,12501,8054,1731,2920,14,28241,14208,14,834,15003,834,13,9078,198,6738,42625,14208,13,26791,13,9641,1330,651,62,9641,198,198,43717,796,357,21,11,362,11,657,11,366,26591,1600,657,8,198,198,834,9641,834],"ids_no_specials":[1003,42625,14208,12,69,18,69,4846,486,66,487,3070,65,29769,3682,68,2154,344,1795,3682,65,7568,12501,8054,1731,2920,14,28241,14208,14,834,15003,834,13,9078,198,6738,42625,14208,13,26791,13,9641,1330,651,62,9641,198,198,43717,796,357,21,11,362,11,657,11,366,26591,1600,657,8,198,198,834,9641,834],"tokens":["//","Ġdj","ango","-","f","3","f","96","01","c","ff","03","b","389","42","e","70","ce","80","42","b","df","dec","600","24","49","/","dj","ango","/","__","init","__",".","py","Ċ","from","Ġdj","ango",".","utils",".","version","Ġimport","Ġget","_","version","Ċ","Ċ","VERSION","Ġ=","Ġ(","6",",","Ġ2",",","Ġ0",",","Ġ\"","alpha","\",","Ġ0",")","Ċ","Ċ","__","version","__"],"offsets":[[0,2],[2,5],[5,9],[9,10],[10,11],[11,12],[12,13],[13,15],[15,17],[17,18],[18,20],[20,22],[22,23],[23,26],[26,28],[28,29],[29,31],[31,33],[33,35],[35,37],[37,38],[38,40],[40,43],[43,46],[46,48],[48,50],[50,51],[51,53],[53,57],[57,58],[58,60],[60,64],[64,66],[66,67],[67,69],[69,70],[70,74],[74,77],[77,81],[81,82],[82,87],[87,88],[88,95],[95,102],[102,106],[106,107],[107,114],[114,115],[115,116],[116,123],[123,125],[125,127],[127,128],[128,129],[129,131],[131,132],[132,134],[134,135],[135,137],[137,142],[142,144],[144,146],[146,147],[147,148],[148,149],[149,151],[151,158],[158,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,1,1,2,3,4,5,6,6,7,7,8,9,10,10,11,12,13,14,14,15,15,15,16,16,16,17,18,18,19,19,20,21,21,22,23,24,25,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54],"decoded":"// django-f3f9601cff03b38942e70ce8042bdfdec6002449/django/__init__.py\nfrom django.utils.version import get_version\n\nVERSION = (6, 2, 0, \"alpha\", 0)\n\n__version__","decoded_with_specials":"// django-f3f9601cff03b38942e70ce8042bdfdec6002449/django/__init__.py\nfrom django.utils.version import get_version\n\nVERSION = (6, 2, 0, \"alpha\", 0)\n\n__version__"} +{"kind":"sample","source":"fixtures/modalities/math_latex.txt[:160]","text":"Bayes and his Theorem\n\nMy earlier post on Bayesian probability seems to have generated quite a lot of readers, so this lunchtime I thought I’d add a little bit ","ids":[15262,274,290,465,1849,464,29625,198,198,3666,2961,1281,319,4696,35610,12867,2331,284,423,7560,2407,257,1256,286,7183,11,523,428,9965,2435,314,1807,314,447,247,67,751,257,1310,1643,220],"ids_no_specials":[15262,274,290,465,1849,464,29625,198,198,3666,2961,1281,319,4696,35610,12867,2331,284,423,7560,2407,257,1256,286,7183,11,523,428,9965,2435,314,1807,314,447,247,67,751,257,1310,1643,220],"tokens":["Bay","es","Ġand","Ġhis","Âł","The","orem","Ċ","Ċ","My","Ġearlier","Ġpost","Ġon","ĠBay","esian","Ġprobability","Ġseems","Ġto","Ġhave","Ġgenerated","Ġquite","Ġa","Ġlot","Ġof","Ġreaders",",","Ġso","Ġthis","Ġlunch","time","ĠI","Ġthought","ĠI","âĢ","Ļ","d","Ġadd","Ġa","Ġlittle","Ġbit","Ġ"],"offsets":[[0,3],[3,5],[5,9],[9,13],[13,14],[14,17],[17,21],[21,22],[22,23],[23,25],[25,33],[33,38],[38,41],[41,45],[45,50],[50,62],[62,68],[68,71],[71,76],[76,86],[86,92],[92,94],[94,98],[98,101],[101,109],[109,110],[110,113],[113,118],[118,124],[124,128],[128,130],[130,138],[138,140],[140,141],[140,141],[141,142],[142,146],[146,148],[148,155],[155,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[0,0,1,2,3,4,4,5,6,7,8,9,10,11,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,25,26,27,28,29,29,30,31,32,33,34,35],"decoded":"Bayes and his Theorem\n\nMy earlier post on Bayesian probability seems to have generated quite a lot of readers, so this lunchtime I thought I’d add a little bit ","decoded_with_specials":"Bayes and his Theorem\n\nMy earlier post on Bayesian probability seems to have generated quite a lot of readers, so this lunchtime I thought I’d add a little bit "} +{"kind":"pair","text":"What is the capital of France?","pair":"Paris is the capital of France.","ids":[2061,318,262,3139,286,4881,30,40313,318,262,3139,286,4881,13],"tokens":["What","Ġis","Ġthe","Ġcapital","Ġof","ĠFrance","?","Paris","Ġis","Ġthe","Ġcapital","Ġof","ĠFrance","."],"type_ids":[0,0,0,0,0,0,0,1,1,1,1,1,1,1],"sequence_ids":[0,0,0,0,0,0,0,1,1,1,1,1,1,1],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0],"offsets":[[0,4],[4,7],[7,11],[11,19],[19,22],[22,29],[29,30],[0,5],[5,8],[8,12],[12,20],[20,23],[23,30],[30,31]]} +{"kind":"pair","text":"Question in English?","pair":"Ответ на русском языке, с цифрами 123.","ids":[24361,287,3594,30,140,252,20375,38857,16843,20375,12466,121,16142,220,21169,35072,21727,21727,31583,25443,120,220,40623,140,115,45035,31583,16843,11,220,21727,220,141,228,18849,141,226,21169,16142,43108,18849,17031,13],"tokens":["Question","Ġin","ĠEnglish","?","Ð","ŀ","ÑĤ","в","е","ÑĤ","ĠÐ","½","а","Ġ","ÑĢ","Ñĥ","Ñģ","Ñģ","к","оÐ","¼","Ġ","Ñı","Ð","·","Ñĭ","к","е",",","Ġ","Ñģ","Ġ","Ñ","Ĩ","и","Ñ","Ħ","ÑĢ","а","м","и","Ġ123","."],"type_ids":[0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],"sequence_ids":[0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"offsets":[[0,8],[8,11],[11,19],[19,20],[0,1],[0,1],[1,2],[2,3],[3,4],[4,5],[5,7],[6,7],[7,8],[8,9],[9,10],[10,11],[11,12],[12,13],[13,14],[14,16],[15,16],[16,17],[17,18],[18,19],[18,19],[19,20],[20,21],[21,22],[22,23],[23,24],[24,25],[25,26],[26,27],[26,27],[27,28],[28,29],[28,29],[29,30],[30,31],[31,32],[32,33],[33,37],[37,38]]} +{"kind":"digest","file":"fixtures/lang/amh_Ethi.txt","cap_chars":200000,"text_sha256":"e353d32f45b449b76fb65ee48f1522086c02a6bffa607e73ed8c875c03c84908","n_tokens":372868,"ids_sha256":"6ebe0023f59ed96836fa706a9dcbc2a11036704b8992a21fbf11f2f2f9994315"} +{"kind":"digest","file":"fixtures/lang/arb_Arab.txt","cap_chars":200000,"text_sha256":"529587117db0a7abaa2c8f31ab05d7b2f77e84a11068b46e26b2b37774d836c7","n_tokens":197781,"ids_sha256":"08bb757a7eca6c2d806fcebf8173b7cae68bbade30fa8b4cdd37abc8f1db594a"} +{"kind":"digest","file":"fixtures/lang/ben_Beng.txt","cap_chars":200000,"text_sha256":"915afd7dbbea8256bdb6112b16424c83aa47dae63c5cca0ebadfc6c227d2b89f","n_tokens":396512,"ids_sha256":"c7e62422dc8982bd8d6293dc0a00f0d0de605707c72442b4b58ba8555b6f210d"} +{"kind":"digest","file":"fixtures/lang/cmn_Hani.txt","cap_chars":200000,"text_sha256":"1273c0c093a00739be9efea7804b10a295f902359d11663306d097211934c6df","n_tokens":388336,"ids_sha256":"29ad139b0ad01a443b9cafe76650c408005541920e7e0782ac9167a366a2b82c"} +{"kind":"digest","file":"fixtures/lang/ell_Grek.txt","cap_chars":200000,"text_sha256":"158c8d3e4933091099bf12a1e21f57c7b7ff3f5381d4b25901eb1fa7ca0926f4","n_tokens":224991,"ids_sha256":"f88f1d2d77a206b57fe468ff35fe8043b0b14964f02108d75493029dbd3f5c08"} +{"kind":"digest","file":"fixtures/lang/eng_Latn.txt","cap_chars":200000,"text_sha256":"d998cb3153e3a56c2ecdf49293062e8bc6ca43365b6c91cfed147d73b84c19fb","n_tokens":46419,"ids_sha256":"04b113c9420da5b8f6324e7f40a5ebed852581404959cf7a5879e925aa509860"} +{"kind":"digest","file":"fixtures/lang/heb_Hebr.txt","cap_chars":200000,"text_sha256":"386da8c0e0d416bffbe84b51baf895c67dbce6e0fd153be597933261794300fd","n_tokens":227167,"ids_sha256":"8e53d51441cab2675190f127ddf12b79cf31276268655b0e56f663fc80f78a24"} +{"kind":"digest","file":"fixtures/lang/hin_Deva.txt","cap_chars":200000,"text_sha256":"2d114148c81035b1b63c94c7e9e805933c215d8aa9500d916923aec0a124ddde","n_tokens":299136,"ids_sha256":"851422859c23e62f1c8c34bca66da21dc89a044c7ac98d03e5cdac1061517923"} +{"kind":"digest","file":"fixtures/lang/jpn_Jpan.txt","cap_chars":200000,"text_sha256":"2e55c036f500d7720d020be2db45c87a49c17d3b7e63b30bada9cef4cb6aa158","n_tokens":270962,"ids_sha256":"760ce68f564bd76b60f1efd5c5da9b6eb9a4e9819347b074e8644aacc0c2a960"} +{"kind":"digest","file":"fixtures/lang/kat_Geor.txt","cap_chars":200000,"text_sha256":"c69a1aa8fb80b41459c98c15c756eb8f39a9151468b1d5b3f9bbffe4d82eb87a","n_tokens":504654,"ids_sha256":"e6ca7ae28480153d69ccf758862cef7b6ffd7940666d6394d38443ada4767892"} +{"kind":"digest","file":"fixtures/lang/kor_Hang.txt","cap_chars":200000,"text_sha256":"68659781c288136c1caffcb10eec9130406ae8a7b003c1e327b7e3601eba041d","n_tokens":393938,"ids_sha256":"7296b5c8b07723266022086cc241c5a99da4875bbbe1a96b8fac668b50136f0f"} +{"kind":"digest","file":"fixtures/lang/rus_Cyrl.txt","cap_chars":200000,"text_sha256":"5f967a82627afbd72c2f21c6e56183dfa5efc986ba34350c1cf9eea6c27a6132","n_tokens":212656,"ids_sha256":"4ba0465d89d797303c5291d21133858c04d79e5e03b798cc4c822172dfdd7540"} +{"kind":"digest","file":"fixtures/lang/tam_Taml.txt","cap_chars":200000,"text_sha256":"bf09e1bca55aeab34162d5dc61afec7ac976b4e66eb5833ac5a037305911d165","n_tokens":536957,"ids_sha256":"289b01888dc66d36fbc33329816efd82d4c083b4d172d040603e6d5a7c37f4a5"} +{"kind":"digest","file":"fixtures/lang/tha_Thai.txt","cap_chars":200000,"text_sha256":"25209f4750b3d3f04440f98aeff50316a118cb982462a4763106e8747172b79c","n_tokens":372373,"ids_sha256":"b3b5fb80c00a4662bfc8edbfe169f00fce121ddccbb80cb99436eea201474dd6"} +{"kind":"digest","file":"fixtures/modalities/added_normalized_dense.txt","cap_chars":200000,"text_sha256":"9d49b3cacf40962c051a09f1350a0f5dc2fc210556889ed0f341cfe0cf0af4f9","n_tokens":35103,"ids_sha256":"574791d2d3e3240fb8b16ee3889a5e510dacc61f9ff16e0fede4d3f5fbd12b24"} +{"kind":"digest","file":"fixtures/modalities/added_normalized_sparse.txt","cap_chars":200000,"text_sha256":"55383f52b02a5d9686f9e4347729e5c08e61b9cff77913bd94ceb65c2fdf7fe4","n_tokens":25584,"ids_sha256":"19149a3e8b10be35558dd54720e319d3309fef909426dd069e1773ecdf0393e9"} +{"kind":"digest","file":"fixtures/modalities/added_special_dense.txt","cap_chars":200000,"text_sha256":"472f90f6f2f5803626df5d7088f8ef12984de6e88fedf67723887511d35b64b2","n_tokens":52822,"ids_sha256":"82de623d318402090dc470534252fc59abe5890c5bff786798df6bd6eb827539"} +{"kind":"digest","file":"fixtures/modalities/added_special_sparse.txt","cap_chars":200000,"text_sha256":"9c52602f7c8d009b11377743f95e5bfe9053694fc6b48c3dd6c1c6dc0681ab5a","n_tokens":31362,"ids_sha256":"256570f5946e77f4feef57fb8ff20ff0d2c42ac3544503d33ba9b5591a5cb969"} +{"kind":"digest","file":"fixtures/modalities/agentic-traces.txt","cap_chars":200000,"text_sha256":"ae9d63c1adc49f572d9af635ac1b0421461e4d1b92c5f35ea572980ff1e2480c","n_tokens":63685,"ids_sha256":"b49f41d134b25851ccb6d040af5a1d2add55c3af73f39fed8a99bce5aea12535"} +{"kind":"digest","file":"fixtures/modalities/agentic_swe.txt","cap_chars":200000,"text_sha256":"b2d31e91ff91bb6c9a3aec3832d25acbecf9b58d5c0674e364d3b287182e7253","n_tokens":85188,"ids_sha256":"4e2c4dfa09a8676f6178a1b1e4bda2068985532eedd7ed95a03be2a3e1bd43ad"} +{"kind":"digest","file":"fixtures/modalities/code_mixed.txt","cap_chars":200000,"text_sha256":"0fa7314727726db056433d36bc1bda021803be7f1d150cae83e362a3ff41821d","n_tokens":95826,"ids_sha256":"20663c304a309143797f60fbc682705755afc13b8c01cbef86735dc78752eb9d"} +{"kind":"digest","file":"fixtures/modalities/math_latex.txt","cap_chars":200000,"text_sha256":"2d17be8704f1f7b4f2174a054c0b85d6d22039e00be8e4a487927d27f3852e90","n_tokens":55566,"ids_sha256":"78c46b9c86fa023d85015f08ff7370ce0e1adff3434ec803865c245874c7aac8"} diff --git a/bindings/python/tests/golden/goldens/llama-2.jsonl b/bindings/python/tests/golden/goldens/llama-2.jsonl new file mode 100644 index 000000000..73dc70316 --- /dev/null +++ b/bindings/python/tests/golden/goldens/llama-2.jsonl @@ -0,0 +1,64 @@ +{"kind":"meta","model":"llama-2","tokenizer_file":"llama-2.json","tokenizers_version":"0.23.1"} +{"kind":"sample","source":"edge:empty","text":"","ids":[1],"ids_no_specials":[],"tokens":[""],"offsets":[[0,0]],"type_ids":[0],"special_tokens_mask":[1],"word_ids":[null],"decoded":"","decoded_with_specials":""} +{"kind":"sample","source":"edge:spaces","text":" ","ids":[1,268],"ids_no_specials":[268],"tokens":["","▁▁▁▁"],"offsets":[[0,0],[0,3]],"type_ids":[0,0],"special_tokens_mask":[1,0],"word_ids":[null,0],"decoded":" ","decoded_with_specials":" "} +{"kind":"sample","source":"edge:hello","text":"Hello world","ids":[1,15043,3186],"ids_no_specials":[15043,3186],"tokens":["","▁Hello","▁world"],"offsets":[[0,0],[0,5],[5,11]],"type_ids":[0,0,0],"special_tokens_mask":[1,0,0],"word_ids":[null,0,0],"decoded":" Hello world","decoded_with_specials":" Hello world"} +{"kind":"sample","source":"edge:punctuation","text":"Hello, world!! How's it going? (fine; thanks...)","ids":[1,15043,29892,3186,6824,1128,29915,29879,372,2675,29973,313,29888,457,29936,3969,11410],"ids_no_specials":[15043,29892,3186,6824,1128,29915,29879,372,2675,29973,313,29888,457,29936,3969,11410],"tokens":["","▁Hello",",","▁world","!!","▁How","'","s","▁it","▁going","?","▁(","f","ine",";","▁thanks","...)"],"offsets":[[0,0],[0,5],[5,6],[6,12],[12,14],[14,18],[18,19],[19,20],[20,23],[23,29],[29,30],[30,32],[32,33],[33,36],[36,37],[37,44],[44,48]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":" Hello, world!! How's it going? (fine; thanks...)","decoded_with_specials":" Hello, world!! How's it going? (fine; thanks...)"} +{"kind":"sample","source":"edge:whitespace-mix","text":"line one\nline two\r\n\tindented\n trailing ","ids":[1,1196,697,13,1220,1023,30004,13,12,12860,287,13,29871,25053,259],"ids_no_specials":[1196,697,13,1220,1023,30004,13,12,12860,287,13,29871,25053,259],"tokens":["","▁line","▁one","<0x0A>","line","▁two","\r","<0x0A>","<0x09>","indent","ed","<0x0A>","▁","▁trailing","▁▁"],"offsets":[[0,0],[0,4],[4,8],[8,9],[9,13],[13,17],[17,18],[18,19],[19,20],[20,26],[26,28],[28,29],[29,30],[30,39],[39,41]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":" line one\nline two\r\n\tindented\n trailing ","decoded_with_specials":" line one\nline two\r\n\tindented\n trailing "} +{"kind":"sample","source":"edge:accents","text":"café naïve résumé — déjà vu","ids":[1,274,28059,1055,30085,345,6896,398,29948,813,20737,18679],"ids_no_specials":[274,28059,1055,30085,345,6896,398,29948,813,20737,18679],"tokens":["","▁c","afé","▁na","ï","ve","▁rés","um","é","▁—","▁déjà","▁vu"],"offsets":[[0,0],[0,1],[1,4],[4,7],[7,8],[8,10],[10,14],[14,16],[16,17],[17,19],[19,24],[24,27]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0],"decoded":" café naïve résumé — déjà vu","decoded_with_specials":" café naïve résumé — déjà vu"} +{"kind":"sample","source":"edge:accents-decomposed","text":"café naïve résumé","ids":[1,5777,1725,30103,1055,29875,31719,345,337,30103,29879,2017,30103],"ids_no_specials":[5777,1725,30103,1055,29875,31719,345,337,30103,29879,2017,30103],"tokens":["","▁ca","fe","́","▁na","i","̈","ve","▁re","́","s","ume","́"],"offsets":[[0,0],[0,2],[2,4],[4,5],[5,8],[8,9],[9,10],[10,12],[12,15],[15,16],[16,17],[17,20],[20,21]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":" café naïve résumé","decoded_with_specials":" café naïve résumé"} +{"kind":"sample","source":"edge:emoji","text":"🤗 emoji, families 👨‍👩‍👧‍👦, flags 🇫🇷 and skin tones 👍🏽","ids":[1,29871,243,162,167,154,953,29877,2397,29892,13175,29871,243,162,148,171,30722,243,162,148,172,30722,243,162,148,170,30722,243,162,148,169,29892,13449,29871,243,162,138,174,243,162,138,186,322,19309,260,2873,29871,243,162,148,144,243,162,146,192],"ids_no_specials":[29871,243,162,167,154,953,29877,2397,29892,13175,29871,243,162,148,171,30722,243,162,148,172,30722,243,162,148,170,30722,243,162,148,169,29892,13449,29871,243,162,138,174,243,162,138,186,322,19309,260,2873,29871,243,162,148,144,243,162,146,192],"tokens":["","▁","<0xF0>","<0x9F>","<0xA4>","<0x97>","▁em","o","ji",",","▁families","▁","<0xF0>","<0x9F>","<0x91>","<0xA8>","‍","<0xF0>","<0x9F>","<0x91>","<0xA9>","‍","<0xF0>","<0x9F>","<0x91>","<0xA7>","‍","<0xF0>","<0x9F>","<0x91>","<0xA6>",",","▁flags","▁","<0xF0>","<0x9F>","<0x87>","<0xAB>","<0xF0>","<0x9F>","<0x87>","<0xB7>","▁and","▁skin","▁t","ones","▁","<0xF0>","<0x9F>","<0x91>","<0x8D>","<0xF0>","<0x9F>","<0x8F>","<0xBD>"],"offsets":[[0,0],[0,1],[0,1],[0,1],[0,1],[0,1],[1,4],[4,5],[5,7],[7,8],[8,17],[17,18],[18,19],[18,19],[18,19],[18,19],[19,20],[20,21],[20,21],[20,21],[20,21],[21,22],[22,23],[22,23],[22,23],[22,23],[23,24],[24,25],[24,25],[24,25],[24,25],[25,26],[26,32],[32,33],[33,34],[33,34],[33,34],[33,34],[34,35],[34,35],[34,35],[34,35],[35,39],[39,44],[44,46],[46,50],[50,51],[51,52],[51,52],[51,52],[51,52],[52,53],[52,53],[52,53],[52,53]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":" 🤗 emoji, families 👨‍👩‍👧‍👦, flags 🇫🇷 and skin tones 👍🏽","decoded_with_specials":" 🤗 emoji, families 👨‍👩‍👧‍👦, flags 🇫🇷 and skin tones 👍🏽"} +{"kind":"sample","source":"edge:cjk","text":"漢字とひらがなとカタカナが混ざった文章です。","ids":[1,29871,31652,30578,30364,31287,30513,30458,30371,30364,30439,30369,30439,30576,30458,233,186,186,230,132,153,30665,30366,30333,31374,30499,30427,30267],"ids_no_specials":[29871,31652,30578,30364,31287,30513,30458,30371,30364,30439,30369,30439,30576,30458,233,186,186,230,132,153,30665,30366,30333,31374,30499,30427,30267],"tokens":["","▁","漢","字","と","ひ","ら","が","な","と","カ","タ","カ","ナ","が","<0xE6>","<0xB7>","<0xB7>","<0xE3>","<0x81>","<0x96>","っ","た","文","章","で","す","。"],"offsets":[[0,0],[0,1],[0,1],[1,2],[2,3],[3,4],[4,5],[5,6],[6,7],[7,8],[8,9],[9,10],[10,11],[11,12],[12,13],[13,14],[13,14],[13,14],[14,15],[14,15],[14,15],[15,16],[16,17],[17,18],[18,19],[19,20],[20,21],[21,22]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":" 漢字とひらがなとカタカナが混ざった文章です。","decoded_with_specials":" 漢字とひらがなとカタカナが混ざった文章です。"} +{"kind":"sample","source":"edge:korean","text":"한국어 텍스트 조각","ids":[1,29871,30877,31293,31129,29871,240,136,144,30784,31177,29871,31408,237,179,132],"ids_no_specials":[29871,30877,31293,31129,29871,240,136,144,30784,31177,29871,31408,237,179,132],"tokens":["","▁","한","국","어","▁","<0xED>","<0x85>","<0x8D>","스","트","▁","조","<0xEA>","<0xB0>","<0x81>"],"offsets":[[0,0],[0,1],[0,1],[1,2],[2,3],[3,4],[4,5],[4,5],[4,5],[5,6],[6,7],[7,8],[8,9],[9,10],[9,10],[9,10]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":" 한국어 텍스트 조각","decoded_with_specials":" 한국어 텍스트 조각"} +{"kind":"sample","source":"edge:rtl","text":"مرحبا بالعالم — שלום עולם","ids":[1,29871,30159,30156,30240,30177,30112,29871,30177,19233,30218,19233,30159,813,29871,30294,30249,30205,30404,29871,30324,30205,30249,30404],"ids_no_specials":[29871,30159,30156,30240,30177,30112,29871,30177,19233,30218,19233,30159,813,29871,30294,30249,30205,30404,29871,30324,30205,30249,30404],"tokens":["","▁","م","ر","ح","ب","ا","▁","ب","ال","ع","ال","م","▁—","▁","ש","ל","ו","ם","▁","ע","ו","ל","ם"],"offsets":[[0,0],[0,1],[0,1],[1,2],[2,3],[3,4],[4,5],[5,6],[6,7],[7,9],[9,10],[10,12],[12,13],[13,15],[15,16],[16,17],[17,18],[18,19],[19,20],[20,21],[21,22],[22,23],[23,24],[24,25]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":" مرحبا بالعالم — שלום עולם","decoded_with_specials":" مرحبا بالعالم — שלום עולם"} +{"kind":"sample","source":"edge:numbers","text":"1234567890, 3.14159, 1,000,000th","ids":[1,29871,29896,29906,29941,29946,29945,29953,29955,29947,29929,29900,29892,29871,29941,29889,29896,29946,29896,29945,29929,29892,29871,29896,29892,29900,29900,29900,29892,29900,29900,29900,386],"ids_no_specials":[29871,29896,29906,29941,29946,29945,29953,29955,29947,29929,29900,29892,29871,29941,29889,29896,29946,29896,29945,29929,29892,29871,29896,29892,29900,29900,29900,29892,29900,29900,29900,386],"tokens":["","▁","1","2","3","4","5","6","7","8","9","0",",","▁","3",".","1","4","1","5","9",",","▁","1",",","0","0","0",",","0","0","0","th"],"offsets":[[0,0],[0,1],[0,1],[1,2],[2,3],[3,4],[4,5],[5,6],[6,7],[7,8],[8,9],[9,10],[10,11],[11,12],[12,13],[13,14],[14,15],[15,16],[16,17],[17,18],[18,19],[19,20],[20,21],[21,22],[22,23],[23,24],[24,25],[25,26],[26,27],[27,28],[28,29],[29,30],[30,32]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":" 1234567890, 3.14159, 1,000,000th","decoded_with_specials":" 1234567890, 3.14159, 1,000,000th"} +{"kind":"sample","source":"edge:code","text":"def f(x):\n return x**2 # squared\nprint(f'{f(3)=}')","ids":[1,822,285,29898,29916,1125,13,1678,736,921,1068,29906,29871,396,10674,1965,13,2158,29898,29888,29915,29912,29888,29898,29941,3892,29913,1495],"ids_no_specials":[822,285,29898,29916,1125,13,1678,736,921,1068,29906,29871,396,10674,1965,13,2158,29898,29888,29915,29912,29888,29898,29941,3892,29913,1495],"tokens":["","▁def","▁f","(","x","):","<0x0A>","▁▁▁","▁return","▁x","**","2","▁","▁#","▁squ","ared","<0x0A>","print","(","f","'","{","f","(","3",")=","}","')"],"offsets":[[0,0],[0,3],[3,5],[5,6],[6,7],[7,9],[9,10],[10,13],[13,20],[20,22],[22,24],[24,25],[25,26],[26,28],[28,32],[32,36],[36,37],[37,42],[42,43],[43,44],[44,45],[45,46],[46,47],[47,48],[48,49],[49,51],[51,52],[52,54]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":" def f(x):\n return x**2 # squared\nprint(f'{f(3)=}')","decoded_with_specials":" def f(x):\n return x**2 # squared\nprint(f'{f(3)=}')"} +{"kind":"sample","source":"edge:long-word","text":"Donaudampfschifffahrtsgesellschaftskapitänsmützenabzeichen","ids":[1,360,2681,566,16817,816,361,600,5610,1372,26175,24892,17975,1983,29885,6621,2256,370,5185,264],"ids_no_specials":[360,2681,566,16817,816,361,600,5610,1372,26175,24892,17975,1983,29885,6621,2256,370,5185,264],"tokens":["","▁D","ona","ud","ampf","sch","if","ff","ahr","ts","gesellschaft","skap","itä","ns","m","üt","zen","ab","zeich","en"],"offsets":[[0,0],[0,1],[1,4],[4,6],[6,10],[10,13],[13,15],[15,17],[17,20],[20,22],[22,34],[34,38],[38,41],[41,43],[43,44],[44,46],[46,49],[49,51],[51,56],[56,58]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":" Donaudampfschifffahrtsgesellschaftskapitänsmützenabzeichen","decoded_with_specials":" Donaudampfschifffahrtsgesellschaftskapitänsmützenabzeichen"} +{"kind":"sample","source":"edge:url-email","text":"https://example.com/a/b?q=1&r=2#frag user.name+tag@example.co.uk","ids":[1,2045,597,4773,29889,510,29914,29874,29914,29890,29973,29939,29922,29896,29987,29878,29922,29906,29937,29888,1431,1404,29889,978,29974,4039,29992,4773,29889,1111,29889,2679],"ids_no_specials":[2045,597,4773,29889,510,29914,29874,29914,29890,29973,29939,29922,29896,29987,29878,29922,29906,29937,29888,1431,1404,29889,978,29974,4039,29992,4773,29889,1111,29889,2679],"tokens":["","▁https","://","example",".","com","/","a","/","b","?","q","=","1","&","r","=","2","#","f","rag","▁user",".","name","+","tag","@","example",".","co",".","uk"],"offsets":[[0,0],[0,5],[5,8],[8,15],[15,16],[16,19],[19,20],[20,21],[21,22],[22,23],[23,24],[24,25],[25,26],[26,27],[27,28],[28,29],[29,30],[30,31],[31,32],[32,33],[33,36],[36,41],[41,42],[42,46],[46,47],[47,50],[50,51],[51,58],[58,59],[59,61],[61,62],[62,64]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":" https://example.com/a/b?q=1&r=2#frag user.name+tag@example.co.uk","decoded_with_specials":" https://example.com/a/b?q=1&r=2#frag user.name+tag@example.co.uk"} +{"kind":"sample","source":"edge:unicode-spaces","text":"a b c　d","ids":[1,263,30081,29890,30080,29883,30358,29881],"ids_no_specials":[263,30081,29890,30080,29883,30358,29881],"tokens":["","▁a"," ","b"," ","c","　","d"],"offsets":[[0,0],[0,1],[1,2],[2,3],[3,4],[4,5],[5,6],[6,7]],"type_ids":[0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0],"decoded":" a b c　d","decoded_with_specials":" a b c　d"} +{"kind":"sample","source":"edge:math","text":"∑ᵢ xᵢ² ≤ ∫₀^∞ e⁻ˣ dx ≈ 1","ids":[1,29871,31060,228,184,165,921,228,184,165,30088,29871,30248,29871,31230,30220,29985,30306,321,30760,206,166,15414,29871,30583,29871,29896],"ids_no_specials":[29871,31060,228,184,165,921,228,184,165,30088,29871,30248,29871,31230,30220,29985,30306,321,30760,206,166,15414,29871,30583,29871,29896],"tokens":["","▁","∑","<0xE1>","<0xB5>","<0xA2>","▁x","<0xE1>","<0xB5>","<0xA2>","²","▁","≤","▁","∫","₀","^","∞","▁e","⁻","<0xCB>","<0xA3>","▁dx","▁","≈","▁","1"],"offsets":[[0,0],[0,1],[0,1],[1,2],[1,2],[1,2],[2,4],[4,5],[4,5],[4,5],[5,6],[6,7],[7,8],[8,9],[9,10],[10,11],[11,12],[12,13],[13,15],[15,16],[16,17],[16,17],[17,20],[20,21],[21,22],[22,23],[23,24]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":" ∑ᵢ xᵢ² ≤ ∫₀^∞ e⁻ˣ dx ≈ 1","decoded_with_specials":" ∑ᵢ xᵢ² ≤ ∫₀^∞ e⁻ˣ dx ≈ 1"} +{"kind":"sample","source":"fixtures/lang/amh_Ethi.txt[:160]","text":"- ፍርድ ቤቱ በአቶ ዮናታን ተስፋዬ ላይ የ6 አመት ከ6 ወር ጽኑ የእስር ቅጣት አስተላለፈ\n- አንድነት ዶክተር ቴድሮስ ለዓለም ጤና ድርጅት ዋና ዳይሬክተርነት በመመረጣቸው ደስታውን ገለጸ\n- ህንድ መጠነ ሰፊ የድንጋይ ከሰል ሃይል ጣብያዎች የመገንባት እ","ids":[1,448,29871,228,144,144,228,139,176,228,142,184,29871,228,140,167,228,140,180,29871,228,140,163,228,141,163,228,140,185,29871,228,142,177,228,141,150,228,140,182,228,141,152,29871,228,140,179,228,139,184,228,144,142,228,142,175,29871,228,139,142,228,142,176,29871,228,142,171,29953,29871,228,141,163,228,139,155,228,140,184,29871,228,141,171,29953,29871,228,142,139,228,139,176,29871,228,143,192,228,141,148,29871,228,142,171,228,141,168,228,139,184,228,139,176,29871,228,140,136,228,143,166,228,140,184,29871,228,141,163,228,139,184,228,140,179,228,139,142,228,139,139,228,144,139,13,29899,29871,228,141,163,228,141,152,228,142,184,228,141,147,228,140,184,29871,228,142,185,228,141,176,228,140,179,228,139,176,29871,228,140,183,228,142,184,228,139,177,228,139,184,29871,228,139,139,228,142,150,228,139,139,228,139,160,29871,228,143,167,228,141,150,29871,228,142,184,228,139,176,228,143,136,228,140,184,29871,228,142,142,228,141,150,29871,228,142,182,228,142,176,228,139,175,228,141,176,228,140,179,228,139,176,228,141,147,228,140,184,29871,228,140,163,228,139,155,228,139,155,228,139,171,228,143,166,228,140,187,228,142,144,29871,228,142,179,228,139,184,228,140,182,228,142,144,228,141,152,29871,228,143,139,228,139,139,228,143,187,13,29899,29871,228,139,136,228,141,152,228,142,184,29871,228,139,155,228,143,163,228,141,147,29871,228,139,179,228,144,141,29871,228,142,171,228,142,184,228,141,152,228,143,142,228,142,176,29871,228,141,171,228,139,179,228,139,144,29871,228,139,134,228,142,176,228,139,144,29871,228,143,166,228,140,168,228,142,174,228,142,145,228,140,192,29871,228,142,171,228,139,155,228,143,139,228,141,152,228,140,166,228,140,184,29871,228,141,168],"ids_no_specials":[448,29871,228,144,144,228,139,176,228,142,184,29871,228,140,167,228,140,180,29871,228,140,163,228,141,163,228,140,185,29871,228,142,177,228,141,150,228,140,182,228,141,152,29871,228,140,179,228,139,184,228,144,142,228,142,175,29871,228,139,142,228,142,176,29871,228,142,171,29953,29871,228,141,163,228,139,155,228,140,184,29871,228,141,171,29953,29871,228,142,139,228,139,176,29871,228,143,192,228,141,148,29871,228,142,171,228,141,168,228,139,184,228,139,176,29871,228,140,136,228,143,166,228,140,184,29871,228,141,163,228,139,184,228,140,179,228,139,142,228,139,139,228,144,139,13,29899,29871,228,141,163,228,141,152,228,142,184,228,141,147,228,140,184,29871,228,142,185,228,141,176,228,140,179,228,139,176,29871,228,140,183,228,142,184,228,139,177,228,139,184,29871,228,139,139,228,142,150,228,139,139,228,139,160,29871,228,143,167,228,141,150,29871,228,142,184,228,139,176,228,143,136,228,140,184,29871,228,142,142,228,141,150,29871,228,142,182,228,142,176,228,139,175,228,141,176,228,140,179,228,139,176,228,141,147,228,140,184,29871,228,140,163,228,139,155,228,139,155,228,139,171,228,143,166,228,140,187,228,142,144,29871,228,142,179,228,139,184,228,140,182,228,142,144,228,141,152,29871,228,143,139,228,139,139,228,143,187,13,29899,29871,228,139,136,228,141,152,228,142,184,29871,228,139,155,228,143,163,228,141,147,29871,228,139,179,228,144,141,29871,228,142,171,228,142,184,228,141,152,228,143,142,228,142,176,29871,228,141,171,228,139,179,228,139,144,29871,228,139,134,228,142,176,228,139,144,29871,228,143,166,228,140,168,228,142,174,228,142,145,228,140,192,29871,228,142,171,228,139,155,228,143,139,228,141,152,228,140,166,228,140,184,29871,228,141,168],"tokens":["","▁-","▁","<0xE1>","<0x8D>","<0x8D>","<0xE1>","<0x88>","<0xAD>","<0xE1>","<0x8B>","<0xB5>","▁","<0xE1>","<0x89>","<0xA4>","<0xE1>","<0x89>","<0xB1>","▁","<0xE1>","<0x89>","<0xA0>","<0xE1>","<0x8A>","<0xA0>","<0xE1>","<0x89>","<0xB6>","▁","<0xE1>","<0x8B>","<0xAE>","<0xE1>","<0x8A>","<0x93>","<0xE1>","<0x89>","<0xB3>","<0xE1>","<0x8A>","<0x95>","▁","<0xE1>","<0x89>","<0xB0>","<0xE1>","<0x88>","<0xB5>","<0xE1>","<0x8D>","<0x8B>","<0xE1>","<0x8B>","<0xAC>","▁","<0xE1>","<0x88>","<0x8B>","<0xE1>","<0x8B>","<0xAD>","▁","<0xE1>","<0x8B>","<0xA8>","6","▁","<0xE1>","<0x8A>","<0xA0>","<0xE1>","<0x88>","<0x98>","<0xE1>","<0x89>","<0xB5>","▁","<0xE1>","<0x8A>","<0xA8>","6","▁","<0xE1>","<0x8B>","<0x88>","<0xE1>","<0x88>","<0xAD>","▁","<0xE1>","<0x8C>","<0xBD>","<0xE1>","<0x8A>","<0x91>","▁","<0xE1>","<0x8B>","<0xA8>","<0xE1>","<0x8A>","<0xA5>","<0xE1>","<0x88>","<0xB5>","<0xE1>","<0x88>","<0xAD>","▁","<0xE1>","<0x89>","<0x85>","<0xE1>","<0x8C>","<0xA3>","<0xE1>","<0x89>","<0xB5>","▁","<0xE1>","<0x8A>","<0xA0>","<0xE1>","<0x88>","<0xB5>","<0xE1>","<0x89>","<0xB0>","<0xE1>","<0x88>","<0x8B>","<0xE1>","<0x88>","<0x88>","<0xE1>","<0x8D>","<0x88>","<0x0A>","-","▁","<0xE1>","<0x8A>","<0xA0>","<0xE1>","<0x8A>","<0x95>","<0xE1>","<0x8B>","<0xB5>","<0xE1>","<0x8A>","<0x90>","<0xE1>","<0x89>","<0xB5>","▁","<0xE1>","<0x8B>","<0xB6>","<0xE1>","<0x8A>","<0xAD>","<0xE1>","<0x89>","<0xB0>","<0xE1>","<0x88>","<0xAD>","▁","<0xE1>","<0x89>","<0xB4>","<0xE1>","<0x8B>","<0xB5>","<0xE1>","<0x88>","<0xAE>","<0xE1>","<0x88>","<0xB5>","▁","<0xE1>","<0x88>","<0x88>","<0xE1>","<0x8B>","<0x93>","<0xE1>","<0x88>","<0x88>","<0xE1>","<0x88>","<0x9D>","▁","<0xE1>","<0x8C>","<0xA4>","<0xE1>","<0x8A>","<0x93>","▁","<0xE1>","<0x8B>","<0xB5>","<0xE1>","<0x88>","<0xAD>","<0xE1>","<0x8C>","<0x85>","<0xE1>","<0x89>","<0xB5>","▁","<0xE1>","<0x8B>","<0x8B>","<0xE1>","<0x8A>","<0x93>","▁","<0xE1>","<0x8B>","<0xB3>","<0xE1>","<0x8B>","<0xAD>","<0xE1>","<0x88>","<0xAC>","<0xE1>","<0x8A>","<0xAD>","<0xE1>","<0x89>","<0xB0>","<0xE1>","<0x88>","<0xAD>","<0xE1>","<0x8A>","<0x90>","<0xE1>","<0x89>","<0xB5>","▁","<0xE1>","<0x89>","<0xA0>","<0xE1>","<0x88>","<0x98>","<0xE1>","<0x88>","<0x98>","<0xE1>","<0x88>","<0xA8>","<0xE1>","<0x8C>","<0xA3>","<0xE1>","<0x89>","<0xB8>","<0xE1>","<0x8B>","<0x8D>","▁","<0xE1>","<0x8B>","<0xB0>","<0xE1>","<0x88>","<0xB5>","<0xE1>","<0x89>","<0xB3>","<0xE1>","<0x8B>","<0x8D>","<0xE1>","<0x8A>","<0x95>","▁","<0xE1>","<0x8C>","<0x88>","<0xE1>","<0x88>","<0x88>","<0xE1>","<0x8C>","<0xB8>","<0x0A>","-","▁","<0xE1>","<0x88>","<0x85>","<0xE1>","<0x8A>","<0x95>","<0xE1>","<0x8B>","<0xB5>","▁","<0xE1>","<0x88>","<0x98>","<0xE1>","<0x8C>","<0xA0>","<0xE1>","<0x8A>","<0x90>","▁","<0xE1>","<0x88>","<0xB0>","<0xE1>","<0x8D>","<0x8A>","▁","<0xE1>","<0x8B>","<0xA8>","<0xE1>","<0x8B>","<0xB5>","<0xE1>","<0x8A>","<0x95>","<0xE1>","<0x8C>","<0x8B>","<0xE1>","<0x8B>","<0xAD>","▁","<0xE1>","<0x8A>","<0xA8>","<0xE1>","<0x88>","<0xB0>","<0xE1>","<0x88>","<0x8D>","▁","<0xE1>","<0x88>","<0x83>","<0xE1>","<0x8B>","<0xAD>","<0xE1>","<0x88>","<0x8D>","▁","<0xE1>","<0x8C>","<0xA3>","<0xE1>","<0x89>","<0xA5>","<0xE1>","<0x8B>","<0xAB>","<0xE1>","<0x8B>","<0x8E>","<0xE1>","<0x89>","<0xBD>","▁","<0xE1>","<0x8B>","<0xA8>","<0xE1>","<0x88>","<0x98>","<0xE1>","<0x8C>","<0x88>","<0xE1>","<0x8A>","<0x95>","<0xE1>","<0x89>","<0xA3>","<0xE1>","<0x89>","<0xB5>","▁","<0xE1>","<0x8A>","<0xA5>"],"offsets":[[0,0],[0,1],[1,2],[2,3],[2,3],[2,3],[3,4],[3,4],[3,4],[4,5],[4,5],[4,5],[5,6],[6,7],[6,7],[6,7],[7,8],[7,8],[7,8],[8,9],[9,10],[9,10],[9,10],[10,11],[10,11],[10,11],[11,12],[11,12],[11,12],[12,13],[13,14],[13,14],[13,14],[14,15],[14,15],[14,15],[15,16],[15,16],[15,16],[16,17],[16,17],[16,17],[17,18],[18,19],[18,19],[18,19],[19,20],[19,20],[19,20],[20,21],[20,21],[20,21],[21,22],[21,22],[21,22],[22,23],[23,24],[23,24],[23,24],[24,25],[24,25],[24,25],[25,26],[26,27],[26,27],[26,27],[27,28],[28,29],[29,30],[29,30],[29,30],[30,31],[30,31],[30,31],[31,32],[31,32],[31,32],[32,33],[33,34],[33,34],[33,34],[34,35],[35,36],[36,37],[36,37],[36,37],[37,38],[37,38],[37,38],[38,39],[39,40],[39,40],[39,40],[40,41],[40,41],[40,41],[41,42],[42,43],[42,43],[42,43],[43,44],[43,44],[43,44],[44,45],[44,45],[44,45],[45,46],[45,46],[45,46],[46,47],[47,48],[47,48],[47,48],[48,49],[48,49],[48,49],[49,50],[49,50],[49,50],[50,51],[51,52],[51,52],[51,52],[52,53],[52,53],[52,53],[53,54],[53,54],[53,54],[54,55],[54,55],[54,55],[55,56],[55,56],[55,56],[56,57],[56,57],[56,57],[57,58],[58,59],[59,60],[60,61],[60,61],[60,61],[61,62],[61,62],[61,62],[62,63],[62,63],[62,63],[63,64],[63,64],[63,64],[64,65],[64,65],[64,65],[65,66],[66,67],[66,67],[66,67],[67,68],[67,68],[67,68],[68,69],[68,69],[68,69],[69,70],[69,70],[69,70],[70,71],[71,72],[71,72],[71,72],[72,73],[72,73],[72,73],[73,74],[73,74],[73,74],[74,75],[74,75],[74,75],[75,76],[76,77],[76,77],[76,77],[77,78],[77,78],[77,78],[78,79],[78,79],[78,79],[79,80],[79,80],[79,80],[80,81],[81,82],[81,82],[81,82],[82,83],[82,83],[82,83],[83,84],[84,85],[84,85],[84,85],[85,86],[85,86],[85,86],[86,87],[86,87],[86,87],[87,88],[87,88],[87,88],[88,89],[89,90],[89,90],[89,90],[90,91],[90,91],[90,91],[91,92],[92,93],[92,93],[92,93],[93,94],[93,94],[93,94],[94,95],[94,95],[94,95],[95,96],[95,96],[95,96],[96,97],[96,97],[96,97],[97,98],[97,98],[97,98],[98,99],[98,99],[98,99],[99,100],[99,100],[99,100],[100,101],[101,102],[101,102],[101,102],[102,103],[102,103],[102,103],[103,104],[103,104],[103,104],[104,105],[104,105],[104,105],[105,106],[105,106],[105,106],[106,107],[106,107],[106,107],[107,108],[107,108],[107,108],[108,109],[109,110],[109,110],[109,110],[110,111],[110,111],[110,111],[111,112],[111,112],[111,112],[112,113],[112,113],[112,113],[113,114],[113,114],[113,114],[114,115],[115,116],[115,116],[115,116],[116,117],[116,117],[116,117],[117,118],[117,118],[117,118],[118,119],[119,120],[120,121],[121,122],[121,122],[121,122],[122,123],[122,123],[122,123],[123,124],[123,124],[123,124],[124,125],[125,126],[125,126],[125,126],[126,127],[126,127],[126,127],[127,128],[127,128],[127,128],[128,129],[129,130],[129,130],[129,130],[130,131],[130,131],[130,131],[131,132],[132,133],[132,133],[132,133],[133,134],[133,134],[133,134],[134,135],[134,135],[134,135],[135,136],[135,136],[135,136],[136,137],[136,137],[136,137],[137,138],[138,139],[138,139],[138,139],[139,140],[139,140],[139,140],[140,141],[140,141],[140,141],[141,142],[142,143],[142,143],[142,143],[143,144],[143,144],[143,144],[144,145],[144,145],[144,145],[145,146],[146,147],[146,147],[146,147],[147,148],[147,148],[147,148],[148,149],[148,149],[148,149],[149,150],[149,150],[149,150],[150,151],[150,151],[150,151],[151,152],[152,153],[152,153],[152,153],[153,154],[153,154],[153,154],[154,155],[154,155],[154,155],[155,156],[155,156],[155,156],[156,157],[156,157],[156,157],[157,158],[157,158],[157,158],[158,159],[159,160],[159,160],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":" - ፍርድ ቤቱ በአቶ ዮናታን ተስፋዬ ላይ የ6 አመት ከ6 ወር ጽኑ የእስር ቅጣት አስተላለፈ\n- አንድነት ዶክተር ቴድሮስ ለዓለም ጤና ድርጅት ዋና ዳይሬክተርነት በመመረጣቸው ደስታውን ገለጸ\n- ህንድ መጠነ ሰፊ የድንጋይ ከሰል ሃይል ጣብያዎች የመገንባት እ","decoded_with_specials":" - ፍርድ ቤቱ በአቶ ዮናታን ተስፋዬ ላይ የ6 አመት ከ6 ወር ጽኑ የእስር ቅጣት አስተላለፈ\n- አንድነት ዶክተር ቴድሮስ ለዓለም ጤና ድርጅት ዋና ዳይሬክተርነት በመመረጣቸው ደስታውን ገለጸ\n- ህንድ መጠነ ሰፊ የድንጋይ ከሰል ሃይል ጣብያዎች የመገንባት እ"} +{"kind":"sample","source":"fixtures/lang/arb_Arab.txt[:160]","text":"[بالصور] ترقوميا : الموت يطفىء باقة ورد قيد التفتح !!\nالخليل – دوت كوم – من غسان عبد الحميد - لم يخطر في بال أهالي \"ترقوميا\" التي انخلع قلبها وهي تودع التراب جث","ids":[1,518,30177,19233,30360,30171,30156,29962,29871,30195,30156,30265,30171,30159,30163,30112,584,24508,30159,30171,30195,29871,30163,30367,30241,30480,30992,29871,30177,30112,30265,30242,29871,30171,30156,30172,29871,30265,30163,30172,24508,30195,30241,30195,30240,21443,13,19233,30321,30138,30163,30138,785,29871,30172,30171,30195,29871,30283,30171,30159,785,29871,30159,30162,29871,30611,30198,30112,30162,29871,30218,30177,30172,24508,30240,30159,30163,30172,448,29871,30138,30159,29871,30163,30321,30367,30156,29871,30241,30163,29871,30177,19233,29871,30372,30204,19233,30163,376,30195,30156,30265,30171,30159,30163,30112,29908,24508,30195,30163,29871,30112,30162,30321,30138,30218,29871,30265,30138,30177,30204,30112,29871,30171,30204,30163,29871,30195,30171,30172,30218,24508,30195,30156,30112,30177,29871,30270,30839],"ids_no_specials":[518,30177,19233,30360,30171,30156,29962,29871,30195,30156,30265,30171,30159,30163,30112,584,24508,30159,30171,30195,29871,30163,30367,30241,30480,30992,29871,30177,30112,30265,30242,29871,30171,30156,30172,29871,30265,30163,30172,24508,30195,30241,30195,30240,21443,13,19233,30321,30138,30163,30138,785,29871,30172,30171,30195,29871,30283,30171,30159,785,29871,30159,30162,29871,30611,30198,30112,30162,29871,30218,30177,30172,24508,30240,30159,30163,30172,448,29871,30138,30159,29871,30163,30321,30367,30156,29871,30241,30163,29871,30177,19233,29871,30372,30204,19233,30163,376,30195,30156,30265,30171,30159,30163,30112,29908,24508,30195,30163,29871,30112,30162,30321,30138,30218,29871,30265,30138,30177,30204,30112,29871,30171,30204,30163,29871,30195,30171,30172,30218,24508,30195,30156,30112,30177,29871,30270,30839],"tokens":["","▁[","ب","ال","ص","و","ر","]","▁","ت","ر","ق","و","م","ي","ا","▁:","▁ال","م","و","ت","▁","ي","ط","ف","ى","ء","▁","ب","ا","ق","ة","▁","و","ر","د","▁","ق","ي","د","▁ال","ت","ف","ت","ح","▁!!","<0x0A>","ال","خ","ل","ي","ل","▁–","▁","د","و","ت","▁","ك","و","م","▁–","▁","م","ن","▁","غ","س","ا","ن","▁","ع","ب","د","▁ال","ح","م","ي","د","▁-","▁","ل","م","▁","ي","خ","ط","ر","▁","ف","ي","▁","ب","ال","▁","أ","ه","ال","ي","▁\"","ت","ر","ق","و","م","ي","ا","\"","▁ال","ت","ي","▁","ا","ن","خ","ل","ع","▁","ق","ل","ب","ه","ا","▁","و","ه","ي","▁","ت","و","د","ع","▁ال","ت","ر","ا","ب","▁","ج","ث"],"offsets":[[0,0],[0,1],[1,2],[2,4],[4,5],[5,6],[6,7],[7,8],[8,9],[9,10],[10,11],[11,12],[12,13],[13,14],[14,15],[15,16],[16,18],[18,21],[21,22],[22,23],[23,24],[24,25],[25,26],[26,27],[27,28],[28,29],[29,30],[30,31],[31,32],[32,33],[33,34],[34,35],[35,36],[36,37],[37,38],[38,39],[39,40],[40,41],[41,42],[42,43],[43,46],[46,47],[47,48],[48,49],[49,50],[50,53],[53,54],[54,56],[56,57],[57,58],[58,59],[59,60],[60,62],[62,63],[63,64],[64,65],[65,66],[66,67],[67,68],[68,69],[69,70],[70,72],[72,73],[73,74],[74,75],[75,76],[76,77],[77,78],[78,79],[79,80],[80,81],[81,82],[82,83],[83,84],[84,87],[87,88],[88,89],[89,90],[90,91],[91,93],[93,94],[94,95],[95,96],[96,97],[97,98],[98,99],[99,100],[100,101],[101,102],[102,103],[103,104],[104,105],[105,106],[106,108],[108,109],[109,110],[110,111],[111,113],[113,114],[114,116],[116,117],[117,118],[118,119],[119,120],[120,121],[121,122],[122,123],[123,124],[124,127],[127,128],[128,129],[129,130],[130,131],[131,132],[132,133],[133,134],[134,135],[135,136],[136,137],[137,138],[138,139],[139,140],[140,141],[141,142],[142,143],[143,144],[144,145],[145,146],[146,147],[147,148],[148,149],[149,150],[150,153],[153,154],[154,155],[155,156],[156,157],[157,158],[158,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":" [بالصور] ترقوميا : الموت يطفىء باقة ورد قيد التفتح !!\nالخليل – دوت كوم – من غسان عبد الحميد - لم يخطر في بال أهالي \"ترقوميا\" التي انخلع قلبها وهي تودع التراب جث","decoded_with_specials":" [بالصور] ترقوميا : الموت يطفىء باقة ورد قيد التفتح !!\nالخليل – دوت كوم – من غسان عبد الحميد - لم يخطر في بال أهالي \"ترقوميا\" التي انخلع قلبها وهي تودع التراب جث"} +{"kind":"sample","source":"fixtures/lang/ben_Beng.txt[:160]","text":"গ্রহ নীহারিকা\nগ্রহ নীহারিকা (ইংরেজি ভাষায়: Planetary nebula) এক বিশেষ ধরনের গ্যাসীয় নীহারিকা। যেসব তারার ভর কম, নির্দিষ্টভাবে বলতে গেলে যেসব তারার ভর সূর্যের ","ids":[1,29871,31665,30655,30653,31717,29871,30792,31595,31717,30445,30653,30786,30995,30445,13,31665,30655,30653,31717,29871,30792,31595,31717,30445,30653,30786,30995,30445,313,227,169,138,227,169,133,30653,30932,31537,30786,29871,227,169,176,30445,227,169,186,30445,31218,31552,29901,20540,653,452,29890,2497,29897,29871,227,169,146,30995,29871,30962,30786,31686,30932,227,169,186,29871,227,169,170,30653,30792,30932,30653,29871,31665,30655,31218,30445,31148,31595,31218,31552,29871,30792,31595,31717,30445,30653,30786,30995,30445,31776,29871,31218,30932,31148,30962,29871,31150,30445,30653,30445,30653,29871,227,169,176,30653,29871,30995,31006,29892,29871,30792,30786,30653,30655,31204,30786,227,169,186,30655,227,169,162,227,169,176,30445,30962,30932,29871,30962,31055,31150,30932,29871,31665,30932,31055,30932,29871,31218,30932,31148,30962,29871,31150,30445,30653,30445,30653,29871,227,169,176,30653,29871,31148,227,170,133,30653,30655,31218,30932,30653,29871],"ids_no_specials":[29871,31665,30655,30653,31717,29871,30792,31595,31717,30445,30653,30786,30995,30445,13,31665,30655,30653,31717,29871,30792,31595,31717,30445,30653,30786,30995,30445,313,227,169,138,227,169,133,30653,30932,31537,30786,29871,227,169,176,30445,227,169,186,30445,31218,31552,29901,20540,653,452,29890,2497,29897,29871,227,169,146,30995,29871,30962,30786,31686,30932,227,169,186,29871,227,169,170,30653,30792,30932,30653,29871,31665,30655,31218,30445,31148,31595,31218,31552,29871,30792,31595,31717,30445,30653,30786,30995,30445,31776,29871,31218,30932,31148,30962,29871,31150,30445,30653,30445,30653,29871,227,169,176,30653,29871,30995,31006,29892,29871,30792,30786,30653,30655,31204,30786,227,169,186,30655,227,169,162,227,169,176,30445,30962,30932,29871,30962,31055,31150,30932,29871,31665,30932,31055,30932,29871,31218,30932,31148,30962,29871,31150,30445,30653,30445,30653,29871,227,169,176,30653,29871,31148,227,170,133,30653,30655,31218,30932,30653,29871],"tokens":["","▁","গ","্","র","হ","▁","ন","ী","হ","া","র","ি","ক","া","<0x0A>","গ","্","র","হ","▁","ন","ী","হ","া","র","ি","ক","া","▁(","<0xE0>","<0xA6>","<0x87>","<0xE0>","<0xA6>","<0x82>","র","ে","জ","ি","▁","<0xE0>","<0xA6>","<0xAD>","া","<0xE0>","<0xA6>","<0xB7>","া","য","়",":","▁Planet","ary","▁ne","b","ula",")","▁","<0xE0>","<0xA6>","<0x8F>","ক","▁","ব","ি","শ","ে","<0xE0>","<0xA6>","<0xB7>","▁","<0xE0>","<0xA6>","<0xA7>","র","ন","ে","র","▁","গ","্","য","া","স","ী","য","়","▁","ন","ী","হ","া","র","ি","ক","া","।","▁","য","ে","স","ব","▁","ত","া","র","া","র","▁","<0xE0>","<0xA6>","<0xAD>","র","▁","ক","ম",",","▁","ন","ি","র","্","দ","ি","<0xE0>","<0xA6>","<0xB7>","্","<0xE0>","<0xA6>","<0x9F>","<0xE0>","<0xA6>","<0xAD>","া","ব","ে","▁","ব","ল","ত","ে","▁","গ","ে","ল","ে","▁","য","ে","স","ব","▁","ত","া","র","া","র","▁","<0xE0>","<0xA6>","<0xAD>","র","▁","স","<0xE0>","<0xA7>","<0x82>","র","্","য","ে","র","▁"],"offsets":[[0,0],[0,1],[0,1],[1,2],[2,3],[3,4],[4,5],[5,6],[6,7],[7,8],[8,9],[9,10],[10,11],[11,12],[12,13],[13,14],[14,15],[15,16],[16,17],[17,18],[18,19],[19,20],[20,21],[21,22],[22,23],[23,24],[24,25],[25,26],[26,27],[27,29],[29,30],[29,30],[29,30],[30,31],[30,31],[30,31],[31,32],[32,33],[33,34],[34,35],[35,36],[36,37],[36,37],[36,37],[37,38],[38,39],[38,39],[38,39],[39,40],[40,41],[41,42],[42,43],[43,50],[50,53],[53,56],[56,57],[57,60],[60,61],[61,62],[62,63],[62,63],[62,63],[63,64],[64,65],[65,66],[66,67],[67,68],[68,69],[69,70],[69,70],[69,70],[70,71],[71,72],[71,72],[71,72],[72,73],[73,74],[74,75],[75,76],[76,77],[77,78],[78,79],[79,80],[80,81],[81,82],[82,83],[83,84],[84,85],[85,86],[86,87],[87,88],[88,89],[89,90],[90,91],[91,92],[92,93],[93,94],[94,95],[95,96],[96,97],[97,98],[98,99],[99,100],[100,101],[101,102],[102,103],[103,104],[104,105],[105,106],[106,107],[107,108],[107,108],[107,108],[108,109],[109,110],[110,111],[111,112],[112,113],[113,114],[114,115],[115,116],[116,117],[117,118],[118,119],[119,120],[120,121],[120,121],[120,121],[121,122],[122,123],[122,123],[122,123],[123,124],[123,124],[123,124],[124,125],[125,126],[126,127],[127,128],[128,129],[129,130],[130,131],[131,132],[132,133],[133,134],[134,135],[135,136],[136,137],[137,138],[138,139],[139,140],[140,141],[141,142],[142,143],[143,144],[144,145],[145,146],[146,147],[147,148],[148,149],[149,150],[149,150],[149,150],[150,151],[151,152],[152,153],[153,154],[153,154],[153,154],[154,155],[155,156],[156,157],[157,158],[158,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":" গ্রহ নীহারিকা\nগ্রহ নীহারিকা (ইংরেজি ভাষায়: Planetary nebula) এক বিশেষ ধরনের গ্যাসীয় নীহারিকা। যেসব তারার ভর কম, নির্দিষ্টভাবে বলতে গেলে যেসব তারার ভর সূর্যের ","decoded_with_specials":" গ্রহ নীহারিকা\nগ্রহ নীহারিকা (ইংরেজি ভাষায়: Planetary nebula) এক বিশেষ ধরনের গ্যাসীয় নীহারিকা। যেসব তারার ভর কম, নির্দিষ্টভাবে বলতে গেলে যেসব তারার ভর সূর্যের "} +{"kind":"sample","source":"fixtures/lang/cmn_Hani.txt[:160]","text":"強力建議大家未來盡量避免英航阿~~~\n班機原訂航程\n6/9 台北→香港\n6/9 香港→倫敦 (原訂23:15起飛，4:50am抵達)\n6/10 倫敦→斯德哥爾摩 (原訂7:40am起飛，11:05am抵達)\n就在第二段，英航no.25班機在香港機場兩度離開閘口又兩度返航\n第一次因有旅客身體不適 (好，不怪他，消耗時間也","ids":[1,29871,232,191,186,31074,30886,235,176,179,30257,30613,31295,231,193,137,234,158,164,31180,236,132,194,232,136,144,31144,31727,31227,7377,30022,13,234,146,176,31540,30667,235,171,133,31727,31101,13,29953,29914,29929,29871,31037,30662,30121,31113,31449,13,29953,29914,29929,29871,31113,31449,30121,232,131,174,233,152,169,313,30667,235,171,133,29906,29941,29901,29896,29945,31558,31877,30214,29946,29901,29945,29900,314,233,141,184,31883,29897,13,29953,29914,29896,29900,29871,232,131,174,233,152,169,30121,31824,31169,232,150,168,234,139,193,233,148,172,313,30667,235,171,133,29955,29901,29946,29900,314,31558,31877,30214,29896,29896,29901,29900,29945,314,233,141,184,31883,29897,13,31238,30505,30622,30685,31559,30214,31144,31727,1217,29889,29906,29945,234,146,176,31540,30505,31113,31449,31540,31045,232,136,172,30898,236,158,165,31404,236,153,155,30856,232,146,139,232,136,172,30898,31086,31727,13,30622,30287,30936,31570,30417,233,154,136,31915,31687,236,174,151,30413,236,132,172,313,31076,30214,30413,31985,31221,30214,31276,235,131,154,30974,31069,30953],"ids_no_specials":[29871,232,191,186,31074,30886,235,176,179,30257,30613,31295,231,193,137,234,158,164,31180,236,132,194,232,136,144,31144,31727,31227,7377,30022,13,234,146,176,31540,30667,235,171,133,31727,31101,13,29953,29914,29929,29871,31037,30662,30121,31113,31449,13,29953,29914,29929,29871,31113,31449,30121,232,131,174,233,152,169,313,30667,235,171,133,29906,29941,29901,29896,29945,31558,31877,30214,29946,29901,29945,29900,314,233,141,184,31883,29897,13,29953,29914,29896,29900,29871,232,131,174,233,152,169,30121,31824,31169,232,150,168,234,139,193,233,148,172,313,30667,235,171,133,29955,29901,29946,29900,314,31558,31877,30214,29896,29896,29901,29900,29945,314,233,141,184,31883,29897,13,31238,30505,30622,30685,31559,30214,31144,31727,1217,29889,29906,29945,234,146,176,31540,30505,31113,31449,31540,31045,232,136,172,30898,236,158,165,31404,236,153,155,30856,232,146,139,232,136,172,30898,31086,31727,13,30622,30287,30936,31570,30417,233,154,136,31915,31687,236,174,151,30413,236,132,172,313,31076,30214,30413,31985,31221,30214,31276,235,131,154,30974,31069,30953],"tokens":["","▁","<0xE5>","<0xBC>","<0xB7>","力","建","<0xE8>","<0xAD>","<0xB0>","大","家","未","<0xE4>","<0xBE>","<0x86>","<0xE7>","<0x9B>","<0xA1>","量","<0xE9>","<0x81>","<0xBF>","<0xE5>","<0x85>","<0x8D>","英","航","阿","~~","~","<0x0A>","<0xE7>","<0x8F>","<0xAD>","機","原","<0xE8>","<0xA8>","<0x82>","航","程","<0x0A>","6","/","9","▁","台","北","→","香","港","<0x0A>","6","/","9","▁","香","港","→","<0xE5>","<0x80>","<0xAB>","<0xE6>","<0x95>","<0xA6>","▁(","原","<0xE8>","<0xA8>","<0x82>","2","3",":","1","5","起","飛","，","4",":","5","0","am","<0xE6>","<0x8A>","<0xB5>","達",")","<0x0A>","6","/","1","0","▁","<0xE5>","<0x80>","<0xAB>","<0xE6>","<0x95>","<0xA6>","→","斯","德","<0xE5>","<0x93>","<0xA5>","<0xE7>","<0x88>","<0xBE>","<0xE6>","<0x91>","<0xA9>","▁(","原","<0xE8>","<0xA8>","<0x82>","7",":","4","0","am","起","飛","，","1","1",":","0","5","am","<0xE6>","<0x8A>","<0xB5>","達",")","<0x0A>","就","在","第","二","段","，","英","航","no",".","2","5","<0xE7>","<0x8F>","<0xAD>","機","在","香","港","機","場","<0xE5>","<0x85>","<0xA9>","度","<0xE9>","<0x9B>","<0xA2>","開","<0xE9>","<0x96>","<0x98>","口","<0xE5>","<0x8F>","<0x88>","<0xE5>","<0x85>","<0xA9>","度","返","航","<0x0A>","第","一","次","因","有","<0xE6>","<0x97>","<0x85>","客","身","<0xE9>","<0xAB>","<0x94>","不","<0xE9>","<0x81>","<0xA9>","▁(","好","，","不","怪","他","，","消","<0xE8>","<0x80>","<0x97>","時","間","也"],"offsets":[[0,0],[0,1],[0,1],[0,1],[0,1],[1,2],[2,3],[3,4],[3,4],[3,4],[4,5],[5,6],[6,7],[7,8],[7,8],[7,8],[8,9],[8,9],[8,9],[9,10],[10,11],[10,11],[10,11],[11,12],[11,12],[11,12],[12,13],[13,14],[14,15],[15,17],[17,18],[18,19],[19,20],[19,20],[19,20],[20,21],[21,22],[22,23],[22,23],[22,23],[23,24],[24,25],[25,26],[26,27],[27,28],[28,29],[29,30],[30,31],[31,32],[32,33],[33,34],[34,35],[35,36],[36,37],[37,38],[38,39],[39,40],[40,41],[41,42],[42,43],[43,44],[43,44],[43,44],[44,45],[44,45],[44,45],[45,47],[47,48],[48,49],[48,49],[48,49],[49,50],[50,51],[51,52],[52,53],[53,54],[54,55],[55,56],[56,57],[57,58],[58,59],[59,60],[60,61],[61,63],[63,64],[63,64],[63,64],[64,65],[65,66],[66,67],[67,68],[68,69],[69,70],[70,71],[71,72],[72,73],[72,73],[72,73],[73,74],[73,74],[73,74],[74,75],[75,76],[76,77],[77,78],[77,78],[77,78],[78,79],[78,79],[78,79],[79,80],[79,80],[79,80],[80,82],[82,83],[83,84],[83,84],[83,84],[84,85],[85,86],[86,87],[87,88],[88,90],[90,91],[91,92],[92,93],[93,94],[94,95],[95,96],[96,97],[97,98],[98,100],[100,101],[100,101],[100,101],[101,102],[102,103],[103,104],[104,105],[105,106],[106,107],[107,108],[108,109],[109,110],[110,111],[111,112],[112,114],[114,115],[115,116],[116,117],[117,118],[117,118],[117,118],[118,119],[119,120],[120,121],[121,122],[122,123],[123,124],[124,125],[124,125],[124,125],[125,126],[126,127],[126,127],[126,127],[127,128],[128,129],[128,129],[128,129],[129,130],[130,131],[130,131],[130,131],[131,132],[131,132],[131,132],[132,133],[133,134],[134,135],[135,136],[136,137],[137,138],[138,139],[139,140],[140,141],[141,142],[141,142],[141,142],[142,143],[143,144],[144,145],[144,145],[144,145],[145,146],[146,147],[146,147],[146,147],[147,149],[149,150],[150,151],[151,152],[152,153],[153,154],[154,155],[155,156],[156,157],[156,157],[156,157],[157,158],[158,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":" 強力建議大家未來盡量避免英航阿~~~\n班機原訂航程\n6/9 台北→香港\n6/9 香港→倫敦 (原訂23:15起飛，4:50am抵達)\n6/10 倫敦→斯德哥爾摩 (原訂7:40am起飛，11:05am抵達)\n就在第二段，英航no.25班機在香港機場兩度離開閘口又兩度返航\n第一次因有旅客身體不適 (好，不怪他，消耗時間也","decoded_with_specials":" 強力建議大家未來盡量避免英航阿~~~\n班機原訂航程\n6/9 台北→香港\n6/9 香港→倫敦 (原訂23:15起飛，4:50am抵達)\n6/10 倫敦→斯德哥爾摩 (原訂7:40am起飛，11:05am抵達)\n就在第二段，英航no.25班機在香港機場兩度離開閘口又兩度返航\n第一次因有旅客身體不適 (好，不怪他，消耗時間也"} +{"kind":"sample","source":"fixtures/lang/ell_Grek.txt[:160]","text":"Θυμήσου με\nΗ πόλη προσφέρει πολλές ευκαιρίες - πήγαινε στο εμπορικό κέντρο, στο σαλόνι ομορφιάς , σε κλάμπ και διασκέδασε με την ψυχή σου.\nΔημιούργησε μαγευτικά","ids":[1,29871,30546,30192,30167,30280,30164,30123,30192,29871,30167,30151,13,30835,29871,30170,30228,30142,30183,29871,30170,30147,30123,30164,30237,30273,30147,30151,30136,29871,30170,30123,30142,30142,30273,30145,29871,30151,30192,30173,30110,30136,30147,30207,30151,30145,448,29871,30170,30280,30197,30110,30136,30133,30151,29871,30164,30137,30123,29871,30151,30167,30170,30123,30147,30136,30173,30228,29871,30173,30273,30133,30137,30147,30123,29892,29871,30164,30137,30123,29871,30164,30110,30142,30228,30133,30136,29871,30123,30167,30123,30147,30237,30136,30216,30145,1919,29871,30164,30151,29871,30173,30142,30216,30167,30170,29871,30173,30110,30136,29871,30201,30136,30110,30164,30173,30273,30201,30110,30164,30151,29871,30167,30151,29871,30137,30183,30133,29871,30559,30192,30274,30280,29871,30164,30123,30192,29889,13,30293,30183,30167,30136,30123,30308,30147,30197,30183,30164,30151,29871,30167,30110,30197,30151,30192,30137,30136,30173,30216],"ids_no_specials":[29871,30546,30192,30167,30280,30164,30123,30192,29871,30167,30151,13,30835,29871,30170,30228,30142,30183,29871,30170,30147,30123,30164,30237,30273,30147,30151,30136,29871,30170,30123,30142,30142,30273,30145,29871,30151,30192,30173,30110,30136,30147,30207,30151,30145,448,29871,30170,30280,30197,30110,30136,30133,30151,29871,30164,30137,30123,29871,30151,30167,30170,30123,30147,30136,30173,30228,29871,30173,30273,30133,30137,30147,30123,29892,29871,30164,30137,30123,29871,30164,30110,30142,30228,30133,30136,29871,30123,30167,30123,30147,30237,30136,30216,30145,1919,29871,30164,30151,29871,30173,30142,30216,30167,30170,29871,30173,30110,30136,29871,30201,30136,30110,30164,30173,30273,30201,30110,30164,30151,29871,30167,30151,29871,30137,30183,30133,29871,30559,30192,30274,30280,29871,30164,30123,30192,29889,13,30293,30183,30167,30136,30123,30308,30147,30197,30183,30164,30151,29871,30167,30110,30197,30151,30192,30137,30136,30173,30216],"tokens":["","▁","Θ","υ","μ","ή","σ","ο","υ","▁","μ","ε","<0x0A>","Η","▁","π","ό","λ","η","▁","π","ρ","ο","σ","φ","έ","ρ","ε","ι","▁","π","ο","λ","λ","έ","ς","▁","ε","υ","κ","α","ι","ρ","ί","ε","ς","▁-","▁","π","ή","γ","α","ι","ν","ε","▁","σ","τ","ο","▁","ε","μ","π","ο","ρ","ι","κ","ό","▁","κ","έ","ν","τ","ρ","ο",",","▁","σ","τ","ο","▁","σ","α","λ","ό","ν","ι","▁","ο","μ","ο","ρ","φ","ι","ά","ς","▁,","▁","σ","ε","▁","κ","λ","ά","μ","π","▁","κ","α","ι","▁","δ","ι","α","σ","κ","έ","δ","α","σ","ε","▁","μ","ε","▁","τ","η","ν","▁","ψ","υ","χ","ή","▁","σ","ο","υ",".","<0x0A>","Δ","η","μ","ι","ο","ύ","ρ","γ","η","σ","ε","▁","μ","α","γ","ε","υ","τ","ι","κ","ά"],"offsets":[[0,0],[0,1],[0,1],[1,2],[2,3],[3,4],[4,5],[5,6],[6,7],[7,8],[8,9],[9,10],[10,11],[11,12],[12,13],[13,14],[14,15],[15,16],[16,17],[17,18],[18,19],[19,20],[20,21],[21,22],[22,23],[23,24],[24,25],[25,26],[26,27],[27,28],[28,29],[29,30],[30,31],[31,32],[32,33],[33,34],[34,35],[35,36],[36,37],[37,38],[38,39],[39,40],[40,41],[41,42],[42,43],[43,44],[44,46],[46,47],[47,48],[48,49],[49,50],[50,51],[51,52],[52,53],[53,54],[54,55],[55,56],[56,57],[57,58],[58,59],[59,60],[60,61],[61,62],[62,63],[63,64],[64,65],[65,66],[66,67],[67,68],[68,69],[69,70],[70,71],[71,72],[72,73],[73,74],[74,75],[75,76],[76,77],[77,78],[78,79],[79,80],[80,81],[81,82],[82,83],[83,84],[84,85],[85,86],[86,87],[87,88],[88,89],[89,90],[90,91],[91,92],[92,93],[93,94],[94,95],[95,97],[97,98],[98,99],[99,100],[100,101],[101,102],[102,103],[103,104],[104,105],[105,106],[106,107],[107,108],[108,109],[109,110],[110,111],[111,112],[112,113],[113,114],[114,115],[115,116],[116,117],[117,118],[118,119],[119,120],[120,121],[121,122],[122,123],[123,124],[124,125],[125,126],[126,127],[127,128],[128,129],[129,130],[130,131],[131,132],[132,133],[133,134],[134,135],[135,136],[136,137],[137,138],[138,139],[139,140],[140,141],[141,142],[142,143],[143,144],[144,145],[145,146],[146,147],[147,148],[148,149],[149,150],[150,151],[151,152],[152,153],[153,154],[154,155],[155,156],[156,157],[157,158],[158,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":" Θυμήσου με\nΗ πόλη προσφέρει πολλές ευκαιρίες - πήγαινε στο εμπορικό κέντρο, στο σαλόνι ομορφιάς , σε κλάμπ και διασκέδασε με την ψυχή σου.\nΔημιούργησε μαγευτικά","decoded_with_specials":" Θυμήσου με\nΗ πόλη προσφέρει πολλές ευκαιρίες - πήγαινε στο εμπορικό κέντρο, στο σαλόνι ομορφιάς , σε κλάμπ και διασκέδασε με την ψυχή σου.\nΔημιούργησε μαγευτικά"} +{"kind":"sample","source":"fixtures/lang/eng_Latn.txt[:160]","text":"|Viewing Single Post From: Spoilers for the Week of February 11th|\n|Lil||Feb 1 2013, 09:58 AM|\nDon't care about Chloe/Taniel/Jen-Jen. Don't care about Sami, rea","ids":[1,891,1043,292,16740,4918,3645,29901,24674,22058,363,278,15511,310,6339,29871,29896,29896,386,29989,13,29989,29931,309,8876,29943,774,29871,29896,29871,29906,29900,29896,29941,29892,29871,29900,29929,29901,29945,29947,13862,29989,13,10310,29915,29873,2562,1048,678,417,29872,29914,29911,273,709,29914,29967,264,29899,29967,264,29889,3872,29915,29873,2562,1048,3685,29875,29892,337,29874],"ids_no_specials":[891,1043,292,16740,4918,3645,29901,24674,22058,363,278,15511,310,6339,29871,29896,29896,386,29989,13,29989,29931,309,8876,29943,774,29871,29896,29871,29906,29900,29896,29941,29892,29871,29900,29929,29901,29945,29947,13862,29989,13,10310,29915,29873,2562,1048,678,417,29872,29914,29911,273,709,29914,29967,264,29899,29967,264,29889,3872,29915,29873,2562,1048,3685,29875,29892,337,29874],"tokens":["","▁|","View","ing","▁Single","▁Post","▁From",":","▁Spo","ilers","▁for","▁the","▁Week","▁of","▁February","▁","1","1","th","|","<0x0A>","|","L","il","||","F","eb","▁","1","▁","2","0","1","3",",","▁","0","9",":","5","8","▁AM","|","<0x0A>","Don","'","t","▁care","▁about","▁Ch","lo","e","/","T","an","iel","/","J","en","-","J","en",".","▁Don","'","t","▁care","▁about","▁Sam","i",",","▁re","a"],"offsets":[[0,0],[0,1],[1,5],[5,8],[8,15],[15,20],[20,25],[25,26],[26,30],[30,35],[35,39],[39,43],[43,48],[48,51],[51,60],[60,61],[61,62],[62,63],[63,65],[65,66],[66,67],[67,68],[68,69],[69,71],[71,73],[73,74],[74,76],[76,77],[77,78],[78,79],[79,80],[80,81],[81,82],[82,83],[83,84],[84,85],[85,86],[86,87],[87,88],[88,89],[89,90],[90,93],[93,94],[94,95],[95,98],[98,99],[99,100],[100,105],[105,111],[111,114],[114,116],[116,117],[117,118],[118,119],[119,121],[121,124],[124,125],[125,126],[126,128],[128,129],[129,130],[130,132],[132,133],[133,137],[137,138],[138,139],[139,144],[144,150],[150,154],[154,155],[155,156],[156,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":" |Viewing Single Post From: Spoilers for the Week of February 11th|\n|Lil||Feb 1 2013, 09:58 AM|\nDon't care about Chloe/Taniel/Jen-Jen. Don't care about Sami, rea","decoded_with_specials":" |Viewing Single Post From: Spoilers for the Week of February 11th|\n|Lil||Feb 1 2013, 09:58 AM|\nDon't care about Chloe/Taniel/Jen-Jen. Don't care about Sami, rea"} +{"kind":"sample","source":"fixtures/lang/heb_Hebr.txt[:160]","text":"סיכונים – אלמנט מרכזי, חיוני לפעילות מסחרית. להבין מה הסיכון הוא מאוד חשוב. החוויה האנושית עולה כי מי יודע איך לקחת סיכונים בזמן, היא לנצח גדולים. לזכור פוליטיק","ids":[1,29871,30504,30196,30498,30205,30328,30196,30404,785,29871,30253,30249,30285,30328,30639,29871,30285,30236,30498,30776,30196,29892,29871,30428,30196,30205,30328,30196,29871,30249,30471,30324,30196,30249,30205,30286,29871,30285,30504,30428,30236,30196,30286,29889,29871,30249,30235,30276,30196,30447,29871,30285,30235,29871,30235,30504,30196,30498,30205,30447,29871,30235,30205,30253,29871,30285,30253,30205,30336,29871,30428,30294,30205,30276,29889,29871,30235,30428,30205,30205,30196,30235,29871,30235,30253,30328,30205,30294,30196,30286,29871,30324,30205,30249,30235,29871,30498,30196,29871,30285,30196,29871,30196,30205,30336,30324,29871,30253,30196,31339,29871,30249,30433,30428,30286,29871,30504,30196,30498,30205,30328,30196,30404,29871,30276,30776,30285,30447,29892,29871,30235,30196,30253,29871,30249,30328,30692,30428,29871,30554,30336,30205,30249,30196,30404,29889,29871,30249,30776,30498,30205,30236,29871,30471,30205,30249,30196,30639,30196,30433],"ids_no_specials":[29871,30504,30196,30498,30205,30328,30196,30404,785,29871,30253,30249,30285,30328,30639,29871,30285,30236,30498,30776,30196,29892,29871,30428,30196,30205,30328,30196,29871,30249,30471,30324,30196,30249,30205,30286,29871,30285,30504,30428,30236,30196,30286,29889,29871,30249,30235,30276,30196,30447,29871,30285,30235,29871,30235,30504,30196,30498,30205,30447,29871,30235,30205,30253,29871,30285,30253,30205,30336,29871,30428,30294,30205,30276,29889,29871,30235,30428,30205,30205,30196,30235,29871,30235,30253,30328,30205,30294,30196,30286,29871,30324,30205,30249,30235,29871,30498,30196,29871,30285,30196,29871,30196,30205,30336,30324,29871,30253,30196,31339,29871,30249,30433,30428,30286,29871,30504,30196,30498,30205,30328,30196,30404,29871,30276,30776,30285,30447,29892,29871,30235,30196,30253,29871,30249,30328,30692,30428,29871,30554,30336,30205,30249,30196,30404,29889,29871,30249,30776,30498,30205,30236,29871,30471,30205,30249,30196,30639,30196,30433],"tokens":["","▁","ס","י","כ","ו","נ","י","ם","▁–","▁","א","ל","מ","נ","ט","▁","מ","ר","כ","ז","י",",","▁","ח","י","ו","נ","י","▁","ל","פ","ע","י","ל","ו","ת","▁","מ","ס","ח","ר","י","ת",".","▁","ל","ה","ב","י","ן","▁","מ","ה","▁","ה","ס","י","כ","ו","ן","▁","ה","ו","א","▁","מ","א","ו","ד","▁","ח","ש","ו","ב",".","▁","ה","ח","ו","ו","י","ה","▁","ה","א","נ","ו","ש","י","ת","▁","ע","ו","ל","ה","▁","כ","י","▁","מ","י","▁","י","ו","ד","ע","▁","א","י","ך","▁","ל","ק","ח","ת","▁","ס","י","כ","ו","נ","י","ם","▁","ב","ז","מ","ן",",","▁","ה","י","א","▁","ל","נ","צ","ח","▁","ג","ד","ו","ל","י","ם",".","▁","ל","ז","כ","ו","ר","▁","פ","ו","ל","י","ט","י","ק"],"offsets":[[0,0],[0,1],[0,1],[1,2],[2,3],[3,4],[4,5],[5,6],[6,7],[7,9],[9,10],[10,11],[11,12],[12,13],[13,14],[14,15],[15,16],[16,17],[17,18],[18,19],[19,20],[20,21],[21,22],[22,23],[23,24],[24,25],[25,26],[26,27],[27,28],[28,29],[29,30],[30,31],[31,32],[32,33],[33,34],[34,35],[35,36],[36,37],[37,38],[38,39],[39,40],[40,41],[41,42],[42,43],[43,44],[44,45],[45,46],[46,47],[47,48],[48,49],[49,50],[50,51],[51,52],[52,53],[53,54],[54,55],[55,56],[56,57],[57,58],[58,59],[59,60],[60,61],[61,62],[62,63],[63,64],[64,65],[65,66],[66,67],[67,68],[68,69],[69,70],[70,71],[71,72],[72,73],[73,74],[74,75],[75,76],[76,77],[77,78],[78,79],[79,80],[80,81],[81,82],[82,83],[83,84],[84,85],[85,86],[86,87],[87,88],[88,89],[89,90],[90,91],[91,92],[92,93],[93,94],[94,95],[95,96],[96,97],[97,98],[98,99],[99,100],[100,101],[101,102],[102,103],[103,104],[104,105],[105,106],[106,107],[107,108],[108,109],[109,110],[110,111],[111,112],[112,113],[113,114],[114,115],[115,116],[116,117],[117,118],[118,119],[119,120],[120,121],[121,122],[122,123],[123,124],[124,125],[125,126],[126,127],[127,128],[128,129],[129,130],[130,131],[131,132],[132,133],[133,134],[134,135],[135,136],[136,137],[137,138],[138,139],[139,140],[140,141],[141,142],[142,143],[143,144],[144,145],[145,146],[146,147],[147,148],[148,149],[149,150],[150,151],[151,152],[152,153],[153,154],[154,155],[155,156],[156,157],[157,158],[158,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":" סיכונים – אלמנט מרכזי, חיוני לפעילות מסחרית. להבין מה הסיכון הוא מאוד חשוב. החוויה האנושית עולה כי מי יודע איך לקחת סיכונים בזמן, היא לנצח גדולים. לזכור פוליטיק","decoded_with_specials":" סיכונים – אלמנט מרכזי, חיוני לפעילות מסחרית. להבין מה הסיכון הוא מאוד חשוב. החוויה האנושית עולה כי מי יודע איך לקחת סיכונים בזמן, היא לנצח גדולים. לזכור פוליטיק"} +{"kind":"sample","source":"fixtures/lang/hin_Deva.txt[:160]","text":"PHOTOS: न्यूज पढ़ते-पढ़ते अचानक ये क्या करने लगी एंकर!\nकुछ समय पहले एक टीवी चैनल पर एंकर खबर पढ़ रही थी और पीछे की स्क्रीन पर 10 मिनिट का पोर्न वीडियो चलता रहा,","ids":[1,349,29950,2891,3267,29901,29871,30424,30296,30640,31413,30871,29871,30621,227,167,165,31821,30475,30569,29899,30621,227,167,165,31821,30475,30569,29871,31350,31132,30269,30424,30444,29871,30640,30569,29871,30444,30296,30640,30269,29871,30444,30316,30424,30569,29871,30603,30875,30580,29871,227,167,146,30757,30444,30316,29991,13,30444,30702,227,167,158,29871,30489,30485,30640,29871,30621,30714,30603,30569,29871,227,167,146,30444,29871,31329,30580,30610,30580,29871,31132,31678,30424,30603,29871,30621,30316,29871,227,167,146,30757,30444,30316,29871,31667,31012,30316,29871,30621,227,167,165,31821,29871,30316,30714,30580,29871,31927,30580,29871,227,167,151,30316,29871,30621,30580,227,167,158,30569,29871,30444,30580,29871,30489,30296,30444,30296,30316,30580,30424,29871,30621,30316,29871,29896,29900,29871,30485,30436,30424,30436,31329,29871,30444,30269,29871,30621,30799,30316,30296,30424,29871,30610,30580,31447,30436,30640,30799,29871,31132,30603,30475,30269,29871,30316,30714,30269,29892],"ids_no_specials":[349,29950,2891,3267,29901,29871,30424,30296,30640,31413,30871,29871,30621,227,167,165,31821,30475,30569,29899,30621,227,167,165,31821,30475,30569,29871,31350,31132,30269,30424,30444,29871,30640,30569,29871,30444,30296,30640,30269,29871,30444,30316,30424,30569,29871,30603,30875,30580,29871,227,167,146,30757,30444,30316,29991,13,30444,30702,227,167,158,29871,30489,30485,30640,29871,30621,30714,30603,30569,29871,227,167,146,30444,29871,31329,30580,30610,30580,29871,31132,31678,30424,30603,29871,30621,30316,29871,227,167,146,30757,30444,30316,29871,31667,31012,30316,29871,30621,227,167,165,31821,29871,30316,30714,30580,29871,31927,30580,29871,227,167,151,30316,29871,30621,30580,227,167,158,30569,29871,30444,30580,29871,30489,30296,30444,30296,30316,30580,30424,29871,30621,30316,29871,29896,29900,29871,30485,30436,30424,30436,31329,29871,30444,30269,29871,30621,30799,30316,30296,30424,29871,30610,30580,31447,30436,30640,30799,29871,31132,30603,30475,30269,29871,30316,30714,30269,29892],"tokens":["","▁P","H","OT","OS",":","▁","न","्","य","ू","ज","▁","प","<0xE0>","<0xA4>","<0xA2>","़","त","े","-","प","<0xE0>","<0xA4>","<0xA2>","़","त","े","▁","अ","च","ा","न","क","▁","य","े","▁","क","्","य","ा","▁","क","र","न","े","▁","ल","ग","ी","▁","<0xE0>","<0xA4>","<0x8F>","ं","क","र","!","<0x0A>","क","ु","<0xE0>","<0xA4>","<0x9B>","▁","स","म","य","▁","प","ह","ल","े","▁","<0xE0>","<0xA4>","<0x8F>","क","▁","ट","ी","व","ी","▁","च","ै","न","ल","▁","प","र","▁","<0xE0>","<0xA4>","<0x8F>","ं","क","र","▁","ख","ब","र","▁","प","<0xE0>","<0xA4>","<0xA2>","़","▁","र","ह","ी","▁","थ","ी","▁","<0xE0>","<0xA4>","<0x94>","र","▁","प","ी","<0xE0>","<0xA4>","<0x9B>","े","▁","क","ी","▁","स","्","क","्","र","ी","न","▁","प","र","▁","1","0","▁","म","ि","न","ि","ट","▁","क","ा","▁","प","ो","र","्","न","▁","व","ी","ड","ि","य","ो","▁","च","ल","त","ा","▁","र","ह","ा",","],"offsets":[[0,0],[0,1],[1,2],[2,4],[4,6],[6,7],[7,8],[8,9],[9,10],[10,11],[11,12],[12,13],[13,14],[14,15],[15,16],[15,16],[15,16],[16,17],[17,18],[18,19],[19,20],[20,21],[21,22],[21,22],[21,22],[22,23],[23,24],[24,25],[25,26],[26,27],[27,28],[28,29],[29,30],[30,31],[31,32],[32,33],[33,34],[34,35],[35,36],[36,37],[37,38],[38,39],[39,40],[40,41],[41,42],[42,43],[43,44],[44,45],[45,46],[46,47],[47,48],[48,49],[49,50],[49,50],[49,50],[50,51],[51,52],[52,53],[53,54],[54,55],[55,56],[56,57],[57,58],[57,58],[57,58],[58,59],[59,60],[60,61],[61,62],[62,63],[63,64],[64,65],[65,66],[66,67],[67,68],[68,69],[68,69],[68,69],[69,70],[70,71],[71,72],[72,73],[73,74],[74,75],[75,76],[76,77],[77,78],[78,79],[79,80],[80,81],[81,82],[82,83],[83,84],[84,85],[84,85],[84,85],[85,86],[86,87],[87,88],[88,89],[89,90],[90,91],[91,92],[92,93],[93,94],[94,95],[94,95],[94,95],[95,96],[96,97],[97,98],[98,99],[99,100],[100,101],[101,102],[102,103],[103,104],[104,105],[104,105],[104,105],[105,106],[106,107],[107,108],[108,109],[109,110],[109,110],[109,110],[110,111],[111,112],[112,113],[113,114],[114,115],[115,116],[116,117],[117,118],[118,119],[119,120],[120,121],[121,122],[122,123],[123,124],[124,125],[125,126],[126,127],[127,128],[128,129],[129,130],[130,131],[131,132],[132,133],[133,134],[134,135],[135,136],[136,137],[137,138],[138,139],[139,140],[140,141],[141,142],[142,143],[143,144],[144,145],[145,146],[146,147],[147,148],[148,149],[149,150],[150,151],[151,152],[152,153],[153,154],[154,155],[155,156],[156,157],[157,158],[158,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":" PHOTOS: न्यूज पढ़ते-पढ़ते अचानक ये क्या करने लगी एंकर!\nकुछ समय पहले एक टीवी चैनल पर एंकर खबर पढ़ रही थी और पीछे की स्क्रीन पर 10 मिनिट का पोर्न वीडियो चलता रहा,","decoded_with_specials":" PHOTOS: न्यूज पढ़ते-पढ़ते अचानक ये क्या करने लगी एंकर!\nकुछ समय पहले एक टीवी चैनल पर एंकर खबर पढ़ रही थी और पीछे की स्क्रीन पर 10 मिनिट का पोर्न वीडियो चलता रहा,"} +{"kind":"sample","source":"fixtures/lang/jpn_Jpan.txt[:160]","text":"Omni Dallas Parkwest Hotelでは4ツ星ホテルで、アイアンホース・ゴルフコース、Love Field Airport とテキサス・スタジアムから7.8kmかかるのところにあります。 優れたホテルにオープンし、ダラスにある古代の建築の象徴です。\n部屋\n快適なゲストルームには、モダンな設備を備えたプレ","ids":[1,13352,1240,27043,4815,5933,16923,30499,30449,29946,30844,30900,31448,30572,30258,30499,30330,30310,30260,30310,30203,31448,30185,30255,30290,30987,30258,30423,30459,30185,30255,30330,29931,994,8989,18117,29871,30364,30572,30454,30615,30255,30290,30255,30369,30391,30310,30579,30412,30513,29955,29889,29947,8848,30412,30412,30332,30199,30364,30589,31206,30353,30641,30453,30441,30427,30267,29871,232,135,173,30553,30366,31448,30572,30258,30353,30514,30185,30605,30203,30326,30330,30617,30281,30255,30353,30641,30332,30960,30690,30199,30886,234,178,140,30199,31133,232,193,183,30499,30427,30267,13,30636,31654,13,232,194,174,236,132,172,30371,31335,30255,30279,30258,30185,30579,30353,30449,30330,30761,30617,30203,30371,31770,232,133,156,30396,232,133,156,30914,30366,30605,30420],"ids_no_specials":[13352,1240,27043,4815,5933,16923,30499,30449,29946,30844,30900,31448,30572,30258,30499,30330,30310,30260,30310,30203,31448,30185,30255,30290,30987,30258,30423,30459,30185,30255,30330,29931,994,8989,18117,29871,30364,30572,30454,30615,30255,30290,30255,30369,30391,30310,30579,30412,30513,29955,29889,29947,8848,30412,30412,30332,30199,30364,30589,31206,30353,30641,30453,30441,30427,30267,29871,232,135,173,30553,30366,31448,30572,30258,30353,30514,30185,30605,30203,30326,30330,30617,30281,30255,30353,30641,30332,30960,30690,30199,30886,234,178,140,30199,31133,232,193,183,30499,30427,30267,13,30636,31654,13,232,194,174,236,132,172,30371,31335,30255,30279,30258,30185,30579,30353,30449,30330,30761,30617,30203,30371,31770,232,133,156,30396,232,133,156,30914,30366,30605,30420],"tokens":["","▁Om","ni","▁Dallas","▁Park","west","▁Hotel","で","は","4","ツ","星","ホ","テ","ル","で","、","ア","イ","ア","ン","ホ","ー","ス","・","ゴ","ル","フ","コ","ー","ス","、","L","ove","▁Field","▁Airport","▁","と","テ","キ","サ","ス","・","ス","タ","ジ","ア","ム","か","ら","7",".","8","km","か","か","る","の","と","こ","ろ","に","あ","り","ま","す","。","▁","<0xE5>","<0x84>","<0xAA>","れ","た","ホ","テ","ル","に","オ","ー","プ","ン","し","、","ダ","ラ","ス","に","あ","る","古","代","の","建","<0xE7>","<0xAF>","<0x89>","の","象","<0xE5>","<0xBE>","<0xB4>","で","す","。","<0x0A>","部","屋","<0x0A>","<0xE5>","<0xBF>","<0xAB>","<0xE9>","<0x81>","<0xA9>","な","ゲ","ス","ト","ル","ー","ム","に","は","、","モ","ダ","ン","な","設","<0xE5>","<0x82>","<0x99>","を","<0xE5>","<0x82>","<0x99>","え","た","プ","レ"],"offsets":[[0,0],[0,2],[2,4],[4,11],[11,16],[16,20],[20,26],[26,27],[27,28],[28,29],[29,30],[30,31],[31,32],[32,33],[33,34],[34,35],[35,36],[36,37],[37,38],[38,39],[39,40],[40,41],[41,42],[42,43],[43,44],[44,45],[45,46],[46,47],[47,48],[48,49],[49,50],[50,51],[51,52],[52,55],[55,61],[61,69],[69,70],[70,71],[71,72],[72,73],[73,74],[74,75],[75,76],[76,77],[77,78],[78,79],[79,80],[80,81],[81,82],[82,83],[83,84],[84,85],[85,86],[86,88],[88,89],[89,90],[90,91],[91,92],[92,93],[93,94],[94,95],[95,96],[96,97],[97,98],[98,99],[99,100],[100,101],[101,102],[102,103],[102,103],[102,103],[103,104],[104,105],[105,106],[106,107],[107,108],[108,109],[109,110],[110,111],[111,112],[112,113],[113,114],[114,115],[115,116],[116,117],[117,118],[118,119],[119,120],[120,121],[121,122],[122,123],[123,124],[124,125],[125,126],[125,126],[125,126],[126,127],[127,128],[128,129],[128,129],[128,129],[129,130],[130,131],[131,132],[132,133],[133,134],[134,135],[135,136],[136,137],[136,137],[136,137],[137,138],[137,138],[137,138],[138,139],[139,140],[140,141],[141,142],[142,143],[143,144],[144,145],[145,146],[146,147],[147,148],[148,149],[149,150],[150,151],[151,152],[152,153],[153,154],[153,154],[153,154],[154,155],[155,156],[155,156],[155,156],[156,157],[157,158],[158,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":" Omni Dallas Parkwest Hotelでは4ツ星ホテルで、アイアンホース・ゴルフコース、Love Field Airport とテキサス・スタジアムから7.8kmかかるのところにあります。 優れたホテルにオープンし、ダラスにある古代の建築の象徴です。\n部屋\n快適なゲストルームには、モダンな設備を備えたプレ","decoded_with_specials":" Omni Dallas Parkwest Hotelでは4ツ星ホテルで、アイアンホース・ゴルフコース、Love Field Airport とテキサス・スタジアムから7.8kmかかるのところにあります。 優れたホテルにオープンし、ダラスにある古代の建築の象徴です。\n部屋\n快適なゲストルームには、モダンな設備を備えたプレ"} +{"kind":"sample","source":"fixtures/lang/kat_Geor.txt[:160]","text":"ჩვენ მსოფლიოს სხვადასხვა ქვეყანაში ვცხოვრობთ და სხვადასხვა ენაზე ვსაუბრობთ, მაგრამ საერთო მიზანი გვაერთიანებს. ჩვენი მთავარი მიზანი სამყაროს შემოქმედისა და ბიბლ","ids":[1,29871,228,134,172,30678,30355,30632,29871,30677,30483,30541,31978,30493,30277,30541,30483,29871,30483,31130,30678,30272,30894,30272,30483,31130,30678,30272,29871,31271,30678,30355,228,134,170,30272,30632,30272,31246,30277,29871,30678,31828,31130,30541,30678,30456,30541,30798,30838,29871,30894,30272,29871,30483,31130,30678,30272,30894,30272,30483,31130,30678,30272,29871,30355,30632,30272,31693,30355,29871,30678,30483,30272,30796,30798,30456,30541,30798,30838,29892,29871,30677,30272,30961,30456,30272,30677,29871,30483,30272,30355,30456,30838,30541,29871,30677,30277,31693,30272,30632,30277,29871,30961,30678,30272,30355,30456,30838,30277,30272,30632,30355,30798,30483,29889,29871,228,134,172,30678,30355,30632,30277,29871,30677,30838,30272,30678,30272,30456,30277,29871,30677,30277,31693,30272,30632,30277,29871,30483,30272,30677,228,134,170,30272,30456,30541,30483,29871,31246,30355,30677,30541,31271,30677,30355,30894,30277,30483,30272,29871,30894,30272,29871,30798,30277,30798,30493],"ids_no_specials":[29871,228,134,172,30678,30355,30632,29871,30677,30483,30541,31978,30493,30277,30541,30483,29871,30483,31130,30678,30272,30894,30272,30483,31130,30678,30272,29871,31271,30678,30355,228,134,170,30272,30632,30272,31246,30277,29871,30678,31828,31130,30541,30678,30456,30541,30798,30838,29871,30894,30272,29871,30483,31130,30678,30272,30894,30272,30483,31130,30678,30272,29871,30355,30632,30272,31693,30355,29871,30678,30483,30272,30796,30798,30456,30541,30798,30838,29892,29871,30677,30272,30961,30456,30272,30677,29871,30483,30272,30355,30456,30838,30541,29871,30677,30277,31693,30272,30632,30277,29871,30961,30678,30272,30355,30456,30838,30277,30272,30632,30355,30798,30483,29889,29871,228,134,172,30678,30355,30632,30277,29871,30677,30838,30272,30678,30272,30456,30277,29871,30677,30277,31693,30272,30632,30277,29871,30483,30272,30677,228,134,170,30272,30456,30541,30483,29871,31246,30355,30677,30541,31271,30677,30355,30894,30277,30483,30272,29871,30894,30272,29871,30798,30277,30798,30493],"tokens":["","▁","<0xE1>","<0x83>","<0xA9>","ვ","ე","ნ","▁","მ","ს","ო","ფ","ლ","ი","ო","ს","▁","ს","ხ","ვ","ა","დ","ა","ს","ხ","ვ","ა","▁","ქ","ვ","ე","<0xE1>","<0x83>","<0xA7>","ა","ნ","ა","შ","ი","▁","ვ","ც","ხ","ო","ვ","რ","ო","ბ","თ","▁","დ","ა","▁","ს","ხ","ვ","ა","დ","ა","ს","ხ","ვ","ა","▁","ე","ნ","ა","ზ","ე","▁","ვ","ს","ა","უ","ბ","რ","ო","ბ","თ",",","▁","მ","ა","გ","რ","ა","მ","▁","ს","ა","ე","რ","თ","ო","▁","მ","ი","ზ","ა","ნ","ი","▁","გ","ვ","ა","ე","რ","თ","ი","ა","ნ","ე","ბ","ს",".","▁","<0xE1>","<0x83>","<0xA9>","ვ","ე","ნ","ი","▁","მ","თ","ა","ვ","ა","რ","ი","▁","მ","ი","ზ","ა","ნ","ი","▁","ს","ა","მ","<0xE1>","<0x83>","<0xA7>","ა","რ","ო","ს","▁","შ","ე","მ","ო","ქ","მ","ე","დ","ი","ს","ა","▁","დ","ა","▁","ბ","ი","ბ","ლ"],"offsets":[[0,0],[0,1],[0,1],[0,1],[0,1],[1,2],[2,3],[3,4],[4,5],[5,6],[6,7],[7,8],[8,9],[9,10],[10,11],[11,12],[12,13],[13,14],[14,15],[15,16],[16,17],[17,18],[18,19],[19,20],[20,21],[21,22],[22,23],[23,24],[24,25],[25,26],[26,27],[27,28],[28,29],[28,29],[28,29],[29,30],[30,31],[31,32],[32,33],[33,34],[34,35],[35,36],[36,37],[37,38],[38,39],[39,40],[40,41],[41,42],[42,43],[43,44],[44,45],[45,46],[46,47],[47,48],[48,49],[49,50],[50,51],[51,52],[52,53],[53,54],[54,55],[55,56],[56,57],[57,58],[58,59],[59,60],[60,61],[61,62],[62,63],[63,64],[64,65],[65,66],[66,67],[67,68],[68,69],[69,70],[70,71],[71,72],[72,73],[73,74],[74,75],[75,76],[76,77],[77,78],[78,79],[79,80],[80,81],[81,82],[82,83],[83,84],[84,85],[85,86],[86,87],[87,88],[88,89],[89,90],[90,91],[91,92],[92,93],[93,94],[94,95],[95,96],[96,97],[97,98],[98,99],[99,100],[100,101],[101,102],[102,103],[103,104],[104,105],[105,106],[106,107],[107,108],[108,109],[109,110],[110,111],[111,112],[111,112],[111,112],[112,113],[113,114],[114,115],[115,116],[116,117],[117,118],[118,119],[119,120],[120,121],[121,122],[122,123],[123,124],[124,125],[125,126],[126,127],[127,128],[128,129],[129,130],[130,131],[131,132],[132,133],[133,134],[134,135],[135,136],[135,136],[135,136],[136,137],[137,138],[138,139],[139,140],[140,141],[141,142],[142,143],[143,144],[144,145],[145,146],[146,147],[147,148],[148,149],[149,150],[150,151],[151,152],[152,153],[153,154],[154,155],[155,156],[156,157],[157,158],[158,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":" ჩვენ მსოფლიოს სხვადასხვა ქვეყანაში ვცხოვრობთ და სხვადასხვა ენაზე ვსაუბრობთ, მაგრამ საერთო მიზანი გვაერთიანებს. ჩვენი მთავარი მიზანი სამყაროს შემოქმედისა და ბიბლ","decoded_with_specials":" ჩვენ მსოფლიოს სხვადასხვა ქვეყანაში ვცხოვრობთ და სხვადასხვა ენაზე ვსაუბრობთ, მაგრამ საერთო მიზანი გვაერთიანებს. ჩვენი მთავარი მიზანი სამყაროს შემოქმედისა და ბიბლ"} +{"kind":"sample","source":"fixtures/lang/kor_Hang.txt[:160]","text":"전화번호:\n위치: 뉴질랜드 > 남섬 > 말버러 > 블레넘\n가격대 (1박): 117,886 원 - 140,244 원\n4.5성급 — Lugano Motor Lodge 4.5*\n- 예약 옵션:\n- 트립어드바이저는 호텔스닷컴, Expedia, Agoda, Asia Web Direct 및 Boo","ids":[1,29871,31170,31225,238,181,139,31603,29901,13,31724,239,188,155,29901,29871,238,140,183,239,170,139,238,161,159,31493,1405,29871,31754,239,135,175,1405,29871,238,170,147,238,181,135,238,162,175,1405,29871,238,187,151,238,163,139,238,135,155,13,30903,237,181,172,30890,313,29896,31736,1125,29871,29896,29896,29955,29892,29947,29947,29953,29871,31198,448,29871,29896,29946,29900,29892,29906,29946,29946,29871,31198,13,29946,29889,29945,31126,237,187,140,813,365,688,1562,16843,365,17979,29871,29946,29889,29945,29930,13,29899,29871,239,155,139,239,152,192,29871,239,155,184,239,136,155,29901,13,29899,29871,31177,238,169,192,31129,31493,31963,30393,239,163,131,31081,29871,31603,240,136,151,30784,238,142,186,239,190,183,29892,12027,1381,29892,4059,8887,29892,14325,2563,8797,29871,238,179,146,1952,29877],"ids_no_specials":[29871,31170,31225,238,181,139,31603,29901,13,31724,239,188,155,29901,29871,238,140,183,239,170,139,238,161,159,31493,1405,29871,31754,239,135,175,1405,29871,238,170,147,238,181,135,238,162,175,1405,29871,238,187,151,238,163,139,238,135,155,13,30903,237,181,172,30890,313,29896,31736,1125,29871,29896,29896,29955,29892,29947,29947,29953,29871,31198,448,29871,29896,29946,29900,29892,29906,29946,29946,29871,31198,13,29946,29889,29945,31126,237,187,140,813,365,688,1562,16843,365,17979,29871,29946,29889,29945,29930,13,29899,29871,239,155,139,239,152,192,29871,239,155,184,239,136,155,29901,13,29899,29871,31177,238,169,192,31129,31493,31963,30393,239,163,131,31081,29871,31603,240,136,151,30784,238,142,186,239,190,183,29892,12027,1381,29892,4059,8887,29892,14325,2563,8797,29871,238,179,146,1952,29877],"tokens":["","▁","전","화","<0xEB>","<0xB2>","<0x88>","호",":","<0x0A>","위","<0xEC>","<0xB9>","<0x98>",":","▁","<0xEB>","<0x89>","<0xB4>","<0xEC>","<0xA7>","<0x88>","<0xEB>","<0x9E>","<0x9C>","드","▁>","▁","남","<0xEC>","<0x84>","<0xAC>","▁>","▁","<0xEB>","<0xA7>","<0x90>","<0xEB>","<0xB2>","<0x84>","<0xEB>","<0x9F>","<0xAC>","▁>","▁","<0xEB>","<0xB8>","<0x94>","<0xEB>","<0xA0>","<0x88>","<0xEB>","<0x84>","<0x98>","<0x0A>","가","<0xEA>","<0xB2>","<0xA9>","대","▁(","1","박","):","▁","1","1","7",",","8","8","6","▁","원","▁-","▁","1","4","0",",","2","4","4","▁","원","<0x0A>","4",".","5","성","<0xEA>","<0xB8>","<0x89>","▁—","▁L","ug","ano","▁Motor","▁L","odge","▁","4",".","5","*","<0x0A>","-","▁","<0xEC>","<0x98>","<0x88>","<0xEC>","<0x95>","<0xBD>","▁","<0xEC>","<0x98>","<0xB5>","<0xEC>","<0x85>","<0x98>",":","<0x0A>","-","▁","트","<0xEB>","<0xA6>","<0xBD>","어","드","바","이","<0xEC>","<0xA0>","<0x80>","는","▁","호","<0xED>","<0x85>","<0x94>","스","<0xEB>","<0x8B>","<0xB7>","<0xEC>","<0xBB>","<0xB4>",",","▁Exp","edia",",","▁Ag","oda",",","▁Asia","▁Web","▁Direct","▁","<0xEB>","<0xB0>","<0x8F>","▁Bo","o"],"offsets":[[0,0],[0,1],[0,1],[1,2],[2,3],[2,3],[2,3],[3,4],[4,5],[5,6],[6,7],[7,8],[7,8],[7,8],[8,9],[9,10],[10,11],[10,11],[10,11],[11,12],[11,12],[11,12],[12,13],[12,13],[12,13],[13,14],[14,16],[16,17],[17,18],[18,19],[18,19],[18,19],[19,21],[21,22],[22,23],[22,23],[22,23],[23,24],[23,24],[23,24],[24,25],[24,25],[24,25],[25,27],[27,28],[28,29],[28,29],[28,29],[29,30],[29,30],[29,30],[30,31],[30,31],[30,31],[31,32],[32,33],[33,34],[33,34],[33,34],[34,35],[35,37],[37,38],[38,39],[39,41],[41,42],[42,43],[43,44],[44,45],[45,46],[46,47],[47,48],[48,49],[49,50],[50,51],[51,53],[53,54],[54,55],[55,56],[56,57],[57,58],[58,59],[59,60],[60,61],[61,62],[62,63],[63,64],[64,65],[65,66],[66,67],[67,68],[68,69],[68,69],[68,69],[69,71],[71,73],[73,75],[75,78],[78,84],[84,86],[86,90],[90,91],[91,92],[92,93],[93,94],[94,95],[95,96],[96,97],[97,98],[98,99],[98,99],[98,99],[99,100],[99,100],[99,100],[100,101],[101,102],[101,102],[101,102],[102,103],[102,103],[102,103],[103,104],[104,105],[105,106],[106,107],[107,108],[108,109],[108,109],[108,109],[109,110],[110,111],[111,112],[112,113],[113,114],[113,114],[113,114],[114,115],[115,116],[116,117],[117,118],[117,118],[117,118],[118,119],[119,120],[119,120],[119,120],[120,121],[120,121],[120,121],[121,122],[122,126],[126,130],[130,131],[131,134],[134,137],[137,138],[138,143],[143,147],[147,154],[154,155],[155,156],[155,156],[155,156],[156,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":" 전화번호:\n위치: 뉴질랜드 > 남섬 > 말버러 > 블레넘\n가격대 (1박): 117,886 원 - 140,244 원\n4.5성급 — Lugano Motor Lodge 4.5*\n- 예약 옵션:\n- 트립어드바이저는 호텔스닷컴, Expedia, Agoda, Asia Web Direct 및 Boo","decoded_with_specials":" 전화번호:\n위치: 뉴질랜드 > 남섬 > 말버러 > 블레넘\n가격대 (1박): 117,886 원 - 140,244 원\n4.5성급 — Lugano Motor Lodge 4.5*\n- 예약 옵션:\n- 트립어드바이저는 호텔스닷컴, Expedia, Agoda, Asia Web Direct 및 Boo"} +{"kind":"sample","source":"fixtures/lang/rus_Cyrl.txt[:160]","text":"Покупая продукты в супермаркетах, сегодня уже мало кто верит в их качество. Как не сделать свое меню экстремальным и на что следует обращать внимание – читайте!","ids":[1,2195,1382,1843,29970,23092,29951,3327,490,3404,7043,20784,2476,15632,29892,1586,588,16282,20203,29584,1186,702,3340,28717,490,11244,18432,3501,29889,3397,29951,1538,531,12761,1413,3862,29919,26831,30005,18796,11226,28494,4470,606,665,4281,19139,1257,9111,4826,1413,490,7949,1755,785,5787,21821,730,29991],"ids_no_specials":[2195,1382,1843,29970,23092,29951,3327,490,3404,7043,20784,2476,15632,29892,1586,588,16282,20203,29584,1186,702,3340,28717,490,11244,18432,3501,29889,3397,29951,1538,531,12761,1413,3862,29919,26831,30005,18796,11226,28494,4470,606,665,4281,19139,1257,9111,4826,1413,490,7949,1755,785,5787,21821,730,29991],"tokens":["","▁По","ку","па","я","▁проду","к","ты","▁в","▁су","пер","мар","ке","тах",",","▁се","го","дня","▁уже","▁мало","▁к","то","▁ве","рит","▁в","▁их","▁каче","ство",".","▁Ка","к","▁не","▁с","дела","ть","▁сво","е","▁мен","ю","▁эк","стре","маль","ным","▁и","▁на","▁что","▁следу","ет","▁обра","ща","ть","▁в","нима","ние","▁–","▁чи","тай","те","!"],"offsets":[[0,0],[0,2],[2,4],[4,6],[6,7],[7,13],[13,14],[14,16],[16,18],[18,21],[21,24],[24,27],[27,29],[29,32],[32,33],[33,36],[36,38],[38,41],[41,45],[45,50],[50,52],[52,54],[54,57],[57,60],[60,62],[62,65],[65,70],[70,74],[74,75],[75,78],[78,79],[79,82],[82,84],[84,88],[88,90],[90,94],[94,95],[95,99],[99,100],[100,103],[103,107],[107,111],[111,114],[114,116],[116,119],[119,123],[123,129],[129,131],[131,136],[136,138],[138,140],[140,142],[142,146],[146,149],[149,151],[151,154],[154,157],[157,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":" Покупая продукты в супермаркетах, сегодня уже мало кто верит в их качество. Как не сделать свое меню экстремальным и на что следует обращать внимание – читайте!","decoded_with_specials":" Покупая продукты в супермаркетах, сегодня уже мало кто верит в их качество. Как не сделать свое меню экстремальным и на что следует обращать внимание – читайте!"} +{"kind":"sample","source":"fixtures/lang/tam_Taml.txt[:160]","text":"‘ஹலோ கலெக்டர் சாரா… சரக்கு வேணும்… கடை எப்ப திறப்பீங்க?’\nஒரு படத்தில் ஒயின்ஷாப்புக்குள் திருடப் போன வடிவேலு நன்றாக சரக்கடித்து விட்டு, ஒயின்ஷாப் ஓனருக்கு போன் ப","ids":[1,5129,227,177,188,31223,227,178,142,29871,30700,31223,227,178,137,30700,30380,31164,30966,30380,29871,31384,31015,30966,31015,30098,29871,31384,30966,30700,30380,30700,30837,29871,31249,227,178,138,227,177,166,30837,30930,30380,30098,29871,30700,31164,31428,29871,227,177,145,30917,30380,30917,29871,30870,30781,31809,30917,30380,30917,227,178,131,227,177,156,30380,30700,29973,30010,13,227,177,149,30966,30837,29871,30917,31164,30870,30380,30870,30781,31223,30380,29871,227,177,149,31481,30781,31253,30380,227,177,186,31015,30917,30380,30917,30837,30700,30380,30700,30837,31715,30380,29871,30870,30781,30966,30837,31164,30917,30380,29871,30917,227,178,142,31253,29871,31249,31164,30781,31249,227,178,138,31223,30837,29871,31833,31253,30380,31809,31015,30700,29871,31384,30966,30700,30380,30700,31164,30781,30870,30380,30870,30837,29871,31249,30781,31164,30380,31164,30837,29892,29871,227,177,149,31481,30781,31253,30380,227,177,186,31015,30917,30380,29871,227,177,150,31253,30966,30837,30700,30380,30700,30837,29871,30917,227,178,142,31253,30380,29871,30917],"ids_no_specials":[5129,227,177,188,31223,227,178,142,29871,30700,31223,227,178,137,30700,30380,31164,30966,30380,29871,31384,31015,30966,31015,30098,29871,31384,30966,30700,30380,30700,30837,29871,31249,227,178,138,227,177,166,30837,30930,30380,30098,29871,30700,31164,31428,29871,227,177,145,30917,30380,30917,29871,30870,30781,31809,30917,30380,30917,227,178,131,227,177,156,30380,30700,29973,30010,13,227,177,149,30966,30837,29871,30917,31164,30870,30380,30870,30781,31223,30380,29871,227,177,149,31481,30781,31253,30380,227,177,186,31015,30917,30380,30917,30837,30700,30380,30700,30837,31715,30380,29871,30870,30781,30966,30837,31164,30917,30380,29871,30917,227,178,142,31253,29871,31249,31164,30781,31249,227,178,138,31223,30837,29871,31833,31253,30380,31809,31015,30700,29871,31384,30966,30700,30380,30700,31164,30781,30870,30380,30870,30837,29871,31249,30781,31164,30380,31164,30837,29892,29871,227,177,149,31481,30781,31253,30380,227,177,186,31015,30917,30380,29871,227,177,150,31253,30966,30837,30700,30380,30700,30837,29871,30917,227,178,142,31253,30380,29871,30917],"tokens":["","▁‘","<0xE0>","<0xAE>","<0xB9>","ல","<0xE0>","<0xAF>","<0x8B>","▁","க","ல","<0xE0>","<0xAF>","<0x86>","க","்","ட","ர","்","▁","ச","ா","ர","ா","…","▁","ச","ர","க","்","க","ு","▁","வ","<0xE0>","<0xAF>","<0x87>","<0xE0>","<0xAE>","<0xA3>","ு","ம","்","…","▁","க","ட","ை","▁","<0xE0>","<0xAE>","<0x8E>","ப","்","ப","▁","த","ி","ற","ப","்","ப","<0xE0>","<0xAF>","<0x80>","<0xE0>","<0xAE>","<0x99>","்","க","?","’","<0x0A>","<0xE0>","<0xAE>","<0x92>","ர","ு","▁","ப","ட","த","்","த","ி","ல","்","▁","<0xE0>","<0xAE>","<0x92>","ய","ி","ன","்","<0xE0>","<0xAE>","<0xB7>","ா","ப","்","ப","ு","க","்","க","ு","ள","்","▁","த","ி","ர","ு","ட","ப","்","▁","ப","<0xE0>","<0xAF>","<0x8B>","ன","▁","வ","ட","ி","வ","<0xE0>","<0xAF>","<0x87>","ல","ு","▁","ந","ன","்","ற","ா","க","▁","ச","ர","க","்","க","ட","ி","த","்","த","ு","▁","வ","ி","ட","்","ட","ு",",","▁","<0xE0>","<0xAE>","<0x92>","ய","ி","ன","்","<0xE0>","<0xAE>","<0xB7>","ா","ப","்","▁","<0xE0>","<0xAE>","<0x93>","ன","ர","ு","க","்","க","ு","▁","ப","<0xE0>","<0xAF>","<0x8B>","ன","்","▁","ப"],"offsets":[[0,0],[0,1],[1,2],[1,2],[1,2],[2,3],[3,4],[3,4],[3,4],[4,5],[5,6],[6,7],[7,8],[7,8],[7,8],[8,9],[9,10],[10,11],[11,12],[12,13],[13,14],[14,15],[15,16],[16,17],[17,18],[18,19],[19,20],[20,21],[21,22],[22,23],[23,24],[24,25],[25,26],[26,27],[27,28],[28,29],[28,29],[28,29],[29,30],[29,30],[29,30],[30,31],[31,32],[32,33],[33,34],[34,35],[35,36],[36,37],[37,38],[38,39],[39,40],[39,40],[39,40],[40,41],[41,42],[42,43],[43,44],[44,45],[45,46],[46,47],[47,48],[48,49],[49,50],[50,51],[50,51],[50,51],[51,52],[51,52],[51,52],[52,53],[53,54],[54,55],[55,56],[56,57],[57,58],[57,58],[57,58],[58,59],[59,60],[60,61],[61,62],[62,63],[63,64],[64,65],[65,66],[66,67],[67,68],[68,69],[69,70],[70,71],[70,71],[70,71],[71,72],[72,73],[73,74],[74,75],[75,76],[75,76],[75,76],[76,77],[77,78],[78,79],[79,80],[80,81],[81,82],[82,83],[83,84],[84,85],[85,86],[86,87],[87,88],[88,89],[89,90],[90,91],[91,92],[92,93],[93,94],[94,95],[95,96],[96,97],[97,98],[97,98],[97,98],[98,99],[99,100],[100,101],[101,102],[102,103],[103,104],[104,105],[104,105],[104,105],[105,106],[106,107],[107,108],[108,109],[109,110],[110,111],[111,112],[112,113],[113,114],[114,115],[115,116],[116,117],[117,118],[118,119],[119,120],[120,121],[121,122],[122,123],[123,124],[124,125],[125,126],[126,127],[127,128],[128,129],[129,130],[130,131],[131,132],[132,133],[133,134],[134,135],[135,136],[135,136],[135,136],[136,137],[137,138],[138,139],[139,140],[140,141],[140,141],[140,141],[141,142],[142,143],[143,144],[144,145],[145,146],[145,146],[145,146],[146,147],[147,148],[148,149],[149,150],[150,151],[151,152],[152,153],[153,154],[154,155],[155,156],[155,156],[155,156],[156,157],[157,158],[158,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":" ‘ஹலோ கலெக்டர் சாரா… சரக்கு வேணும்… கடை எப்ப திறப்பீங்க?’\nஒரு படத்தில் ஒயின்ஷாப்புக்குள் திருடப் போன வடிவேலு நன்றாக சரக்கடித்து விட்டு, ஒயின்ஷாப் ஓனருக்கு போன் ப","decoded_with_specials":" ‘ஹலோ கலெக்டர் சாரா… சரக்கு வேணும்… கடை எப்ப திறப்பீங்க?’\nஒரு படத்தில் ஒயின்ஷாப்புக்குள் திருடப் போன வடிவேலு நன்றாக சரக்கடித்து விட்டு, ஒயின்ஷாப் ஓனருக்கு போன் ப"} +{"kind":"sample","source":"fixtures/lang/tha_Thai.txt[:160]","text":"นับเจดีย์ดูนะครับว่าครบยี่สิบองค์รึเปล่า คำว่าซาว ทางเหนือแปลว่่ายี่สิบครับ ผมนับดูแล้วก็ครบนะ ลองดู เสร็จแล้วก็เข้าเยี่ยมชมพิพธภัณฑ์ เดินเข้าไปด้านในหน่อยก็จะม","ids":[1,29871,30348,30510,30526,30401,30991,30718,30691,30549,30779,30718,31396,30348,30823,30759,30297,30510,30526,30492,30543,30289,30759,30297,30526,30549,30691,30543,30547,30507,30526,30351,30398,30759,30779,30297,227,187,185,30401,31010,30496,30543,30289,29871,30759,30747,30492,30543,30289,227,187,142,30289,30492,29871,30595,30289,30398,30401,30663,30348,31422,30351,31073,31010,30496,30492,30543,30543,30289,30549,30691,30543,30547,30507,30526,30759,30297,30510,30526,29871,227,187,159,30501,30348,30510,30526,30718,31396,31073,30496,30652,30492,30425,31453,30759,30297,30526,30348,30823,29871,30496,30351,30398,30718,31396,29871,30401,30547,30297,31453,30991,31073,30496,30652,30492,30425,31453,30401,31310,30652,30289,30401,30549,30691,30543,30549,30501,30913,30501,30727,30507,30727,31547,31070,30510,31796,227,187,148,30779,29871,30401,30718,30507,30348,30401,31310,30652,30289,31252,31010,30718,30652,30289,30348,227,188,134,30348,30663,30348,30543,30351,30549,30425,31453,30991,30823,30501],"ids_no_specials":[29871,30348,30510,30526,30401,30991,30718,30691,30549,30779,30718,31396,30348,30823,30759,30297,30510,30526,30492,30543,30289,30759,30297,30526,30549,30691,30543,30547,30507,30526,30351,30398,30759,30779,30297,227,187,185,30401,31010,30496,30543,30289,29871,30759,30747,30492,30543,30289,227,187,142,30289,30492,29871,30595,30289,30398,30401,30663,30348,31422,30351,31073,31010,30496,30492,30543,30543,30289,30549,30691,30543,30547,30507,30526,30759,30297,30510,30526,29871,227,187,159,30501,30348,30510,30526,30718,31396,31073,30496,30652,30492,30425,31453,30759,30297,30526,30348,30823,29871,30496,30351,30398,30718,31396,29871,30401,30547,30297,31453,30991,31073,30496,30652,30492,30425,31453,30401,31310,30652,30289,30401,30549,30691,30543,30549,30501,30913,30501,30727,30507,30727,31547,31070,30510,31796,227,187,148,30779,29871,30401,30718,30507,30348,30401,31310,30652,30289,31252,31010,30718,30652,30289,30348,227,188,134,30348,30663,30348,30543,30351,30549,30425,31453,30991,30823,30501],"tokens":["","▁","น","ั","บ","เ","จ","ด","ี","ย","์","ด","ู","น","ะ","ค","ร","ั","บ","ว","่","า","ค","ร","บ","ย","ี","่","ส","ิ","บ","อ","ง","ค","์","ร","<0xE0>","<0xB8>","<0xB6>","เ","ป","ล","่","า","▁","ค","ำ","ว","่","า","<0xE0>","<0xB8>","<0x8B>","า","ว","▁","ท","า","ง","เ","ห","น","ื","อ","แ","ป","ล","ว","่","่","า","ย","ี","่","ส","ิ","บ","ค","ร","ั","บ","▁","<0xE0>","<0xB8>","<0x9C>","ม","น","ั","บ","ด","ู","แ","ล","้","ว","ก","็","ค","ร","บ","น","ะ","▁","ล","อ","ง","ด","ู","▁","เ","ส","ร","็","จ","แ","ล","้","ว","ก","็","เ","ข","้","า","เ","ย","ี","่","ย","ม","ช","ม","พ","ิ","พ","ธ","ภ","ั","ณ","<0xE0>","<0xB8>","<0x91>","์","▁","เ","ด","ิ","น","เ","ข","้","า","ไ","ป","ด","้","า","น","<0xE0>","<0xB9>","<0x83>","น","ห","น","่","อ","ย","ก","็","จ","ะ","ม"],"offsets":[[0,0],[0,1],[0,1],[1,2],[2,3],[3,4],[4,5],[5,6],[6,7],[7,8],[8,9],[9,10],[10,11],[11,12],[12,13],[13,14],[14,15],[15,16],[16,17],[17,18],[18,19],[19,20],[20,21],[21,22],[22,23],[23,24],[24,25],[25,26],[26,27],[27,28],[28,29],[29,30],[30,31],[31,32],[32,33],[33,34],[34,35],[34,35],[34,35],[35,36],[36,37],[37,38],[38,39],[39,40],[40,41],[41,42],[42,43],[43,44],[44,45],[45,46],[46,47],[46,47],[46,47],[47,48],[48,49],[49,50],[50,51],[51,52],[52,53],[53,54],[54,55],[55,56],[56,57],[57,58],[58,59],[59,60],[60,61],[61,62],[62,63],[63,64],[64,65],[65,66],[66,67],[67,68],[68,69],[69,70],[70,71],[71,72],[72,73],[73,74],[74,75],[75,76],[76,77],[76,77],[76,77],[77,78],[78,79],[79,80],[80,81],[81,82],[82,83],[83,84],[84,85],[85,86],[86,87],[87,88],[88,89],[89,90],[90,91],[91,92],[92,93],[93,94],[94,95],[95,96],[96,97],[97,98],[98,99],[99,100],[100,101],[101,102],[102,103],[103,104],[104,105],[105,106],[106,107],[107,108],[108,109],[109,110],[110,111],[111,112],[112,113],[113,114],[114,115],[115,116],[116,117],[117,118],[118,119],[119,120],[120,121],[121,122],[122,123],[123,124],[124,125],[125,126],[126,127],[127,128],[128,129],[129,130],[130,131],[131,132],[131,132],[131,132],[132,133],[133,134],[134,135],[135,136],[136,137],[137,138],[138,139],[139,140],[140,141],[141,142],[142,143],[143,144],[144,145],[145,146],[146,147],[147,148],[148,149],[148,149],[148,149],[149,150],[150,151],[151,152],[152,153],[153,154],[154,155],[155,156],[156,157],[157,158],[158,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":" นับเจดีย์ดูนะครับว่าครบยี่สิบองค์รึเปล่า คำว่าซาว ทางเหนือแปลว่่ายี่สิบครับ ผมนับดูแล้วก็ครบนะ ลองดู เสร็จแล้วก็เข้าเยี่ยมชมพิพธภัณฑ์ เดินเข้าไปด้านในหน่อยก็จะม","decoded_with_specials":" นับเจดีย์ดูนะครับว่าครบยี่สิบองค์รึเปล่า คำว่าซาว ทางเหนือแปลว่่ายี่สิบครับ ผมนับดูแล้วก็ครบนะ ลองดู เสร็จแล้วก็เข้าเยี่ยมชมพิพธภัณฑ์ เดินเข้าไปด้านในหน่อยก็จะม"} +{"kind":"sample","source":"fixtures/modalities/added_normalized_dense.txt[:160]","text":"FLIBBERJAST CRUNGLEDORF FLIBBERJAST SNORLAXIAN fast BLORPTRONIC ZORPTASTIC split WIDGETRON split stage SNORLAXIAN BLORPTRONIC BLORPTRONIC \n QUIBBLENAUT CRUNGLED","ids":[1,383,5265,29933,13635,29967,28938,15600,3904,29954,20566,1955,29943,383,5265,29933,13635,29967,28938,21989,1955,4375,29990,29902,2190,5172,350,29931,1955,29925,5659,1164,2965,796,1955,29925,6040,1254,2965,6219,399,1367,1692,5659,1164,6219,7408,21989,1955,4375,29990,29902,2190,350,29931,1955,29925,5659,1164,2965,350,29931,1955,29925,5659,1164,2965,29871,13,660,3120,14388,1307,3521,2692,15600,3904,29954,20566],"ids_no_specials":[383,5265,29933,13635,29967,28938,15600,3904,29954,20566,1955,29943,383,5265,29933,13635,29967,28938,21989,1955,4375,29990,29902,2190,5172,350,29931,1955,29925,5659,1164,2965,796,1955,29925,6040,1254,2965,6219,399,1367,1692,5659,1164,6219,7408,21989,1955,4375,29990,29902,2190,350,29931,1955,29925,5659,1164,2965,350,29931,1955,29925,5659,1164,2965,29871,13,660,3120,14388,1307,3521,2692,15600,3904,29954,20566],"tokens":["","▁F","LI","B","BER","J","AST","▁CR","UN","G","LED","OR","F","▁F","LI","B","BER","J","AST","▁SN","OR","LA","X","I","AN","▁fast","▁B","L","OR","P","TR","ON","IC","▁Z","OR","P","TA","ST","IC","▁split","▁W","ID","GE","TR","ON","▁split","▁stage","▁SN","OR","LA","X","I","AN","▁B","L","OR","P","TR","ON","IC","▁B","L","OR","P","TR","ON","IC","▁","<0x0A>","▁Q","UI","BB","LE","NA","UT","▁CR","UN","G","LED"],"offsets":[[0,0],[0,1],[1,3],[3,4],[4,7],[7,8],[8,11],[11,14],[14,16],[16,17],[17,20],[20,22],[22,23],[23,25],[25,27],[27,28],[28,31],[31,32],[32,35],[35,38],[38,40],[40,42],[42,43],[43,44],[44,46],[46,51],[51,53],[53,54],[54,56],[56,57],[57,59],[59,61],[61,63],[63,65],[65,67],[67,68],[68,70],[70,72],[72,74],[74,80],[80,82],[82,84],[84,86],[86,88],[88,90],[90,96],[96,102],[102,105],[105,107],[107,109],[109,110],[110,111],[111,113],[113,115],[115,116],[116,118],[118,119],[119,121],[121,123],[123,125],[125,127],[127,128],[128,130],[130,131],[131,133],[133,135],[135,137],[137,138],[138,139],[139,141],[141,143],[143,145],[145,147],[147,149],[149,151],[151,154],[154,156],[156,157],[157,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":" FLIBBERJAST CRUNGLEDORF FLIBBERJAST SNORLAXIAN fast BLORPTRONIC ZORPTASTIC split WIDGETRON split stage SNORLAXIAN BLORPTRONIC BLORPTRONIC \n QUIBBLENAUT CRUNGLED","decoded_with_specials":" FLIBBERJAST CRUNGLEDORF FLIBBERJAST SNORLAXIAN fast BLORPTRONIC ZORPTASTIC split WIDGETRON split stage SNORLAXIAN BLORPTRONIC BLORPTRONIC \n QUIBBLENAUT CRUNGLED"} +{"kind":"sample","source":"fixtures/modalities/added_normalized_sparse.txt[:160]","text":"ZORPTASTIC for we for decode CRUNGLEDORF here through ZORPTASTIC and the we WIDGETRON model \n normalize bytes and back flows text CRUNGLEDORF WUZZLEFANG chunk b","ids":[1,796,1955,29925,6040,1254,2965,363,591,363,21822,15600,3904,29954,20566,1955,29943,1244,1549,796,1955,29925,6040,1254,2965,322,278,591,399,1367,1692,5659,1164,1904,29871,13,4226,675,6262,322,1250,24536,1426,15600,3904,29954,20566,1955,29943,399,29965,29999,29999,1307,29943,19453,19875,289],"ids_no_specials":[796,1955,29925,6040,1254,2965,363,591,363,21822,15600,3904,29954,20566,1955,29943,1244,1549,796,1955,29925,6040,1254,2965,322,278,591,399,1367,1692,5659,1164,1904,29871,13,4226,675,6262,322,1250,24536,1426,15600,3904,29954,20566,1955,29943,399,29965,29999,29999,1307,29943,19453,19875,289],"tokens":["","▁Z","OR","P","TA","ST","IC","▁for","▁we","▁for","▁decode","▁CR","UN","G","LED","OR","F","▁here","▁through","▁Z","OR","P","TA","ST","IC","▁and","▁the","▁we","▁W","ID","GE","TR","ON","▁model","▁","<0x0A>","▁normal","ize","▁bytes","▁and","▁back","▁flows","▁text","▁CR","UN","G","LED","OR","F","▁W","U","Z","Z","LE","F","ANG","▁chunk","▁b"],"offsets":[[0,0],[0,1],[1,3],[3,4],[4,6],[6,8],[8,10],[10,14],[14,17],[17,21],[21,28],[28,31],[31,33],[33,34],[34,37],[37,39],[39,40],[40,45],[45,53],[53,55],[55,57],[57,58],[58,60],[60,62],[62,64],[64,68],[68,72],[72,75],[75,77],[77,79],[79,81],[81,83],[83,85],[85,91],[91,92],[92,93],[93,100],[100,103],[103,109],[109,113],[113,118],[118,124],[124,129],[129,132],[132,134],[134,135],[135,138],[138,140],[140,141],[141,143],[143,144],[144,145],[145,146],[146,148],[148,149],[149,152],[152,158],[158,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":" ZORPTASTIC for we for decode CRUNGLEDORF here through ZORPTASTIC and the we WIDGETRON model \n normalize bytes and back flows text CRUNGLEDORF WUZZLEFANG chunk b","decoded_with_specials":" ZORPTASTIC for we for decode CRUNGLEDORF here through ZORPTASTIC and the we WIDGETRON model \n normalize bytes and back flows text CRUNGLEDORF WUZZLEFANG chunk b"} +{"kind":"sample","source":"fixtures/modalities/added_special_dense.txt[:160]","text":"fast <|xs2|> chunk <|xs1|> <|xs4|> <|xs3|> <|xs4|> modality language <|xs0|> reads <|xs2|> and and \n <|xs1|> <|xs0|> <|xs1|> <|xs4|> <|xs4|> <|xs1|> back and re","ids":[1,5172,529,29989,10351,29906,29989,29958,19875,529,29989,10351,29896,29989,29958,529,29989,10351,29946,29989,29958,529,29989,10351,29941,29989,29958,529,29989,10351,29946,29989,29958,878,2877,4086,529,29989,10351,29900,29989,29958,13623,529,29989,10351,29906,29989,29958,322,322,29871,13,529,29989,10351,29896,29989,29958,529,29989,10351,29900,29989,29958,529,29989,10351,29896,29989,29958,529,29989,10351,29946,29989,29958,529,29989,10351,29946,29989,29958,529,29989,10351,29896,29989,29958,1250,322,337],"ids_no_specials":[5172,529,29989,10351,29906,29989,29958,19875,529,29989,10351,29896,29989,29958,529,29989,10351,29946,29989,29958,529,29989,10351,29941,29989,29958,529,29989,10351,29946,29989,29958,878,2877,4086,529,29989,10351,29900,29989,29958,13623,529,29989,10351,29906,29989,29958,322,322,29871,13,529,29989,10351,29896,29989,29958,529,29989,10351,29900,29989,29958,529,29989,10351,29896,29989,29958,529,29989,10351,29946,29989,29958,529,29989,10351,29946,29989,29958,529,29989,10351,29896,29989,29958,1250,322,337],"tokens":["","▁fast","▁<","|","xs","2","|",">","▁chunk","▁<","|","xs","1","|",">","▁<","|","xs","4","|",">","▁<","|","xs","3","|",">","▁<","|","xs","4","|",">","▁mod","ality","▁language","▁<","|","xs","0","|",">","▁reads","▁<","|","xs","2","|",">","▁and","▁and","▁","<0x0A>","▁<","|","xs","1","|",">","▁<","|","xs","0","|",">","▁<","|","xs","1","|",">","▁<","|","xs","4","|",">","▁<","|","xs","4","|",">","▁<","|","xs","1","|",">","▁back","▁and","▁re"],"offsets":[[0,0],[0,4],[4,6],[6,7],[7,9],[9,10],[10,11],[11,12],[12,18],[18,20],[20,21],[21,23],[23,24],[24,25],[25,26],[26,28],[28,29],[29,31],[31,32],[32,33],[33,34],[34,36],[36,37],[37,39],[39,40],[40,41],[41,42],[42,44],[44,45],[45,47],[47,48],[48,49],[49,50],[50,54],[54,59],[59,68],[68,70],[70,71],[71,73],[73,74],[74,75],[75,76],[76,82],[82,84],[84,85],[85,87],[87,88],[88,89],[89,90],[90,94],[94,98],[98,99],[99,100],[100,102],[102,103],[103,105],[105,106],[106,107],[107,108],[108,110],[110,111],[111,113],[113,114],[114,115],[115,116],[116,118],[118,119],[119,121],[121,122],[122,123],[123,124],[124,126],[126,127],[127,129],[129,130],[130,131],[131,132],[132,134],[134,135],[135,137],[137,138],[138,139],[139,140],[140,142],[142,143],[143,145],[145,146],[146,147],[147,148],[148,153],[153,157],[157,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":" fast <|xs2|> chunk <|xs1|> <|xs4|> <|xs3|> <|xs4|> modality language <|xs0|> reads <|xs2|> and and \n <|xs1|> <|xs0|> <|xs1|> <|xs4|> <|xs4|> <|xs1|> back and re","decoded_with_specials":" fast <|xs2|> chunk <|xs1|> <|xs4|> <|xs3|> <|xs4|> modality language <|xs0|> reads <|xs2|> and and \n <|xs1|> <|xs0|> <|xs1|> <|xs4|> <|xs4|> <|xs1|> back and re"} +{"kind":"sample","source":"fixtures/modalities/added_special_sparse.txt[:160]","text":"<|xs0|> every for encode <|xs0|> bytes the normalize decode text model <|xs4|> <|xs3|> and \n and <|xs3|> here <|xs1|> again model here text <|xs2|> <|xs2|> lang","ids":[1,529,29989,10351,29900,29989,29958,1432,363,19750,529,29989,10351,29900,29989,29958,6262,278,4226,675,21822,1426,1904,529,29989,10351,29946,29989,29958,529,29989,10351,29941,29989,29958,322,29871,13,322,529,29989,10351,29941,29989,29958,1244,529,29989,10351,29896,29989,29958,1449,1904,1244,1426,529,29989,10351,29906,29989,29958,529,29989,10351,29906,29989,29958,6361],"ids_no_specials":[529,29989,10351,29900,29989,29958,1432,363,19750,529,29989,10351,29900,29989,29958,6262,278,4226,675,21822,1426,1904,529,29989,10351,29946,29989,29958,529,29989,10351,29941,29989,29958,322,29871,13,322,529,29989,10351,29941,29989,29958,1244,529,29989,10351,29896,29989,29958,1449,1904,1244,1426,529,29989,10351,29906,29989,29958,529,29989,10351,29906,29989,29958,6361],"tokens":["","▁<","|","xs","0","|",">","▁every","▁for","▁encode","▁<","|","xs","0","|",">","▁bytes","▁the","▁normal","ize","▁decode","▁text","▁model","▁<","|","xs","4","|",">","▁<","|","xs","3","|",">","▁and","▁","<0x0A>","▁and","▁<","|","xs","3","|",">","▁here","▁<","|","xs","1","|",">","▁again","▁model","▁here","▁text","▁<","|","xs","2","|",">","▁<","|","xs","2","|",">","▁lang"],"offsets":[[0,0],[0,1],[1,2],[2,4],[4,5],[5,6],[6,7],[7,13],[13,17],[17,24],[24,26],[26,27],[27,29],[29,30],[30,31],[31,32],[32,38],[38,42],[42,49],[49,52],[52,59],[59,64],[64,70],[70,72],[72,73],[73,75],[75,76],[76,77],[77,78],[78,80],[80,81],[81,83],[83,84],[84,85],[85,86],[86,90],[90,91],[91,92],[92,96],[96,98],[98,99],[99,101],[101,102],[102,103],[103,104],[104,109],[109,111],[111,112],[112,114],[114,115],[115,116],[116,117],[117,123],[123,129],[129,134],[134,139],[139,141],[141,142],[142,144],[144,145],[145,146],[146,147],[147,149],[149,150],[150,152],[152,153],[153,154],[154,155],[155,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":" <|xs0|> every for encode <|xs0|> bytes the normalize decode text model <|xs4|> <|xs3|> and \n and <|xs3|> here <|xs1|> again model here text <|xs2|> <|xs2|> lang","decoded_with_specials":" <|xs0|> every for encode <|xs0|> bytes the normalize decode text model <|xs4|> <|xs3|> and \n and <|xs3|> here <|xs1|> again model here text <|xs2|> <|xs2|> lang"} +{"kind":"sample","source":"fixtures/modalities/agentic-traces.txt[:160]","text":"Analyze this codebase and summarize what it is and how it works.\nI need to explore the codebase structure by reading the main entry points and configuration fil","ids":[1,11597,29891,911,445,775,3188,322,19138,675,825,372,338,322,920,372,1736,29889,13,29902,817,304,26987,278,775,3188,3829,491,5183,278,1667,6251,3291,322,5285,977],"ids_no_specials":[11597,29891,911,445,775,3188,322,19138,675,825,372,338,322,920,372,1736,29889,13,29902,817,304,26987,278,775,3188,3829,491,5183,278,1667,6251,3291,322,5285,977],"tokens":["","▁Anal","y","ze","▁this","▁code","base","▁and","▁summar","ize","▁what","▁it","▁is","▁and","▁how","▁it","▁works",".","<0x0A>","I","▁need","▁to","▁explore","▁the","▁code","base","▁structure","▁by","▁reading","▁the","▁main","▁entry","▁points","▁and","▁configuration","▁fil"],"offsets":[[0,0],[0,4],[4,5],[5,7],[7,12],[12,17],[17,21],[21,25],[25,32],[32,35],[35,40],[40,43],[43,46],[46,50],[50,54],[54,57],[57,63],[63,64],[64,65],[65,66],[66,71],[71,74],[74,82],[82,86],[86,91],[91,95],[95,105],[105,108],[108,116],[116,120],[120,125],[125,131],[131,138],[138,142],[142,156],[156,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":" Analyze this codebase and summarize what it is and how it works.\nI need to explore the codebase structure by reading the main entry points and configuration fil","decoded_with_specials":" Analyze this codebase and summarize what it is and how it works.\nI need to explore the codebase structure by reading the main entry points and configuration fil"} +{"kind":"sample","source":"fixtures/modalities/agentic_swe.txt[:160]","text":"[system]\nYou are a helpful assistant that can interact with a computer to solve tasks.\n\n[user]\n\n/testbed\n\nI've uploaded a pytho","ids":[1,518,5205,29962,13,3492,526,263,8444,20255,393,508,16254,411,263,6601,304,4505,9595,29889,13,13,29961,1792,29962,13,29966,9009,287,29918,5325,29958,13,29914,1688,2580,13,829,9009,287,29918,5325,29958,13,29902,29915,345,20373,263,282,1541,29877],"ids_no_specials":[518,5205,29962,13,3492,526,263,8444,20255,393,508,16254,411,263,6601,304,4505,9595,29889,13,13,29961,1792,29962,13,29966,9009,287,29918,5325,29958,13,29914,1688,2580,13,829,9009,287,29918,5325,29958,13,29902,29915,345,20373,263,282,1541,29877],"tokens":["","▁[","system","]","<0x0A>","You","▁are","▁a","▁helpful","▁assistant","▁that","▁can","▁interact","▁with","▁a","▁computer","▁to","▁solve","▁tasks",".","<0x0A>","<0x0A>","[","user","]","<0x0A>","<","upload","ed","_","files",">","<0x0A>","/","test","bed","<0x0A>","","<0x0A>","I","'","ve","▁uploaded","▁a","▁p","yth","o"],"offsets":[[0,0],[0,1],[1,7],[7,8],[8,9],[9,12],[12,16],[16,18],[18,26],[26,36],[36,41],[41,45],[45,54],[54,59],[59,61],[61,70],[70,73],[73,79],[79,85],[85,86],[86,87],[87,88],[88,89],[89,93],[93,94],[94,95],[95,96],[96,102],[102,104],[104,105],[105,110],[110,111],[111,112],[112,113],[113,117],[117,120],[120,121],[121,123],[123,129],[129,131],[131,132],[132,137],[137,138],[138,139],[139,140],[140,141],[141,143],[143,152],[152,154],[154,156],[156,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":" [system]\nYou are a helpful assistant that can interact with a computer to solve tasks.\n\n[user]\n\n/testbed\n\nI've uploaded a pytho","decoded_with_specials":" [system]\nYou are a helpful assistant that can interact with a computer to solve tasks.\n\n[user]\n\n/testbed\n\nI've uploaded a pytho"} +{"kind":"sample","source":"fixtures/modalities/code_mixed.txt[:160]","text":"// django-f3f9601cff03b38942e70ce8042bdfdec6002449/django/__init__.py\nfrom django.utils.version import get_version\n\nVERSION = (6, 2, 0, \"alpha\", 0)\n\n__version__","ids":[1,849,9557,29899,29888,29941,29888,29929,29953,29900,29896,29883,600,29900,29941,29890,29941,29947,29929,29946,29906,29872,29955,29900,346,29947,29900,29946,29906,29890,2176,7099,29953,29900,29900,29906,29946,29946,29929,29914,14095,29914,1649,2344,26914,2272,13,3166,9557,29889,13239,29889,3259,1053,679,29918,3259,13,13,16358,353,313,29953,29892,29871,29906,29892,29871,29900,29892,376,2312,613,29871,29900,29897,13,13,1649,3259,1649],"ids_no_specials":[849,9557,29899,29888,29941,29888,29929,29953,29900,29896,29883,600,29900,29941,29890,29941,29947,29929,29946,29906,29872,29955,29900,346,29947,29900,29946,29906,29890,2176,7099,29953,29900,29900,29906,29946,29946,29929,29914,14095,29914,1649,2344,26914,2272,13,3166,9557,29889,13239,29889,3259,1053,679,29918,3259,13,13,16358,353,313,29953,29892,29871,29906,29892,29871,29900,29892,376,2312,613,29871,29900,29897,13,13,1649,3259,1649],"tokens":["","▁//","▁django","-","f","3","f","9","6","0","1","c","ff","0","3","b","3","8","9","4","2","e","7","0","ce","8","0","4","2","b","df","dec","6","0","0","2","4","4","9","/","django","/","__","init","__.","py","<0x0A>","from","▁django",".","utils",".","version","▁import","▁get","_","version","<0x0A>","<0x0A>","VERSION","▁=","▁(","6",",","▁","2",",","▁","0",",","▁\"","alpha","\",","▁","0",")","<0x0A>","<0x0A>","__","version","__"],"offsets":[[0,0],[0,2],[2,9],[9,10],[10,11],[11,12],[12,13],[13,14],[14,15],[15,16],[16,17],[17,18],[18,20],[20,21],[21,22],[22,23],[23,24],[24,25],[25,26],[26,27],[27,28],[28,29],[29,30],[30,31],[31,33],[33,34],[34,35],[35,36],[36,37],[37,38],[38,40],[40,43],[43,44],[44,45],[45,46],[46,47],[47,48],[48,49],[49,50],[50,51],[51,57],[57,58],[58,60],[60,64],[64,67],[67,69],[69,70],[70,74],[74,81],[81,82],[82,87],[87,88],[88,95],[95,102],[102,106],[106,107],[107,114],[114,115],[115,116],[116,123],[123,125],[125,127],[127,128],[128,129],[129,130],[130,131],[131,132],[132,133],[133,134],[134,135],[135,137],[137,142],[142,144],[144,145],[145,146],[146,147],[147,148],[148,149],[149,151],[151,158],[158,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":" // django-f3f9601cff03b38942e70ce8042bdfdec6002449/django/__init__.py\nfrom django.utils.version import get_version\n\nVERSION = (6, 2, 0, \"alpha\", 0)\n\n__version__","decoded_with_specials":" // django-f3f9601cff03b38942e70ce8042bdfdec6002449/django/__init__.py\nfrom django.utils.version import get_version\n\nVERSION = (6, 2, 0, \"alpha\", 0)\n\n__version__"} +{"kind":"sample","source":"fixtures/modalities/math_latex.txt[:160]","text":"Bayes and his Theorem\n\nMy earlier post on Bayesian probability seems to have generated quite a lot of readers, so this lunchtime I thought I’d add a little bit ","ids":[1,6211,267,322,670,30081,28831,13,13,3421,8859,1400,373,6211,18970,6976,2444,304,505,5759,3755,263,3287,310,22176,29892,577,445,301,3322,2230,306,2714,306,30010,29881,788,263,2217,2586,29871],"ids_no_specials":[6211,267,322,670,30081,28831,13,13,3421,8859,1400,373,6211,18970,6976,2444,304,505,5759,3755,263,3287,310,22176,29892,577,445,301,3322,2230,306,2714,306,30010,29881,788,263,2217,2586,29871],"tokens":["","▁Bay","es","▁and","▁his"," ","Theorem","<0x0A>","<0x0A>","My","▁earlier","▁post","▁on","▁Bay","esian","▁probability","▁seems","▁to","▁have","▁generated","▁quite","▁a","▁lot","▁of","▁readers",",","▁so","▁this","▁l","unch","time","▁I","▁thought","▁I","’","d","▁add","▁a","▁little","▁bit","▁"],"offsets":[[0,0],[0,3],[3,5],[5,9],[9,13],[13,14],[14,21],[21,22],[22,23],[23,25],[25,33],[33,38],[38,41],[41,45],[45,50],[50,62],[62,68],[68,71],[71,76],[76,86],[86,92],[92,94],[94,98],[98,101],[101,109],[109,110],[110,113],[113,118],[118,120],[120,124],[124,128],[128,130],[130,138],[138,140],[140,141],[141,142],[142,146],[146,148],[148,155],[155,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":" Bayes and his Theorem\n\nMy earlier post on Bayesian probability seems to have generated quite a lot of readers, so this lunchtime I thought I’d add a little bit ","decoded_with_specials":" Bayes and his Theorem\n\nMy earlier post on Bayesian probability seems to have generated quite a lot of readers, so this lunchtime I thought I’d add a little bit "} +{"kind":"pair","text":"What is the capital of France?","pair":"Paris is the capital of France.","ids":[1,1724,338,278,7483,310,3444,29973,1,3681,338,278,7483,310,3444,29889],"tokens":["","▁What","▁is","▁the","▁capital","▁of","▁France","?","","▁Paris","▁is","▁the","▁capital","▁of","▁France","."],"type_ids":[0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1],"sequence_ids":[null,0,0,0,0,0,0,0,null,1,1,1,1,1,1,1],"special_tokens_mask":[1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0],"offsets":[[0,0],[0,4],[4,7],[7,11],[11,19],[19,22],[22,29],[29,30],[0,0],[0,5],[5,8],[8,12],[12,20],[20,23],[23,30],[30,31]]} +{"kind":"pair","text":"Question in English?","pair":"Ответ на русском языке, с цифрами 123.","ids":[1,894,297,4223,29973,1,14809,7616,665,14251,5540,14264,2476,29892,531,12442,30011,18731,29871,29896,29906,29941,29889],"tokens":["","▁Question","▁in","▁English","?","","▁От","вет","▁на","▁рус","ском","▁язы","ке",",","▁с","▁ци","ф","рами","▁","1","2","3","."],"type_ids":[0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],"sequence_ids":[null,0,0,0,0,null,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],"special_tokens_mask":[1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"offsets":[[0,0],[0,8],[8,11],[11,19],[19,20],[0,0],[0,2],[2,5],[5,8],[8,12],[12,16],[16,20],[20,22],[22,23],[23,25],[25,28],[28,29],[29,33],[33,34],[34,35],[35,36],[36,37],[37,38]]} +{"kind":"digest","file":"fixtures/lang/amh_Ethi.txt","cap_chars":200000,"text_sha256":"e353d32f45b449b76fb65ee48f1522086c02a6bffa607e73ed8c875c03c84908","n_tokens":401839,"ids_sha256":"98e52abf39872f343500f8b2973ed374eadb38648b5f9c44192a3dbe6be9783d"} +{"kind":"digest","file":"fixtures/lang/arb_Arab.txt","cap_chars":200000,"text_sha256":"529587117db0a7abaa2c8f31ab05d7b2f77e84a11068b46e26b2b37774d836c7","n_tokens":179621,"ids_sha256":"f8d0ed66bad23b0d6783e792902163f28f5147d6614dd89cddda7c8dc1db9691"} +{"kind":"digest","file":"fixtures/lang/ben_Beng.txt","cap_chars":200000,"text_sha256":"915afd7dbbea8256bdb6112b16424c83aa47dae63c5cca0ebadfc6c227d2b89f","n_tokens":250933,"ids_sha256":"b7fd31eaf01fa180996abaddec009b62b3b0158c67530ee79fdae1d59f33dcfc"} +{"kind":"digest","file":"fixtures/lang/cmn_Hani.txt","cap_chars":200000,"text_sha256":"1273c0c093a00739be9efea7804b10a295f902359d11663306d097211934c6df","n_tokens":294119,"ids_sha256":"757f47713b1ca669cdecdfbd4f6dd2ee71c1ecf50c5d60c5ce2cbf72a4bfcd8d"} +{"kind":"digest","file":"fixtures/lang/ell_Grek.txt","cap_chars":200000,"text_sha256":"158c8d3e4933091099bf12a1e21f57c7b7ff3f5381d4b25901eb1fa7ca0926f4","n_tokens":195257,"ids_sha256":"7b7bc0492c161753aa0cac3d8f358d2b5d69c4936afcad0111451d682b9d4409"} +{"kind":"digest","file":"fixtures/lang/eng_Latn.txt","cap_chars":200000,"text_sha256":"d998cb3153e3a56c2ecdf49293062e8bc6ca43365b6c91cfed147d73b84c19fb","n_tokens":56269,"ids_sha256":"81db2fab466dac522496e6c46244034199e18dc2716089685304035111a58b82"} +{"kind":"digest","file":"fixtures/lang/heb_Hebr.txt","cap_chars":200000,"text_sha256":"386da8c0e0d416bffbe84b51baf895c67dbce6e0fd153be597933261794300fd","n_tokens":196267,"ids_sha256":"81e812ab77ee01eaee8a6cbc6dac35daec537a2db21df730fdca15af2f10b5f4"} +{"kind":"digest","file":"fixtures/lang/hin_Deva.txt","cap_chars":200000,"text_sha256":"2d114148c81035b1b63c94c7e9e805933c215d8aa9500d916923aec0a124ddde","n_tokens":212250,"ids_sha256":"ce9fa26dbef81bc9c55f08d469333f26ce1f402453f7b6edd53882ef310df617"} +{"kind":"digest","file":"fixtures/lang/jpn_Jpan.txt","cap_chars":200000,"text_sha256":"2e55c036f500d7720d020be2db45c87a49c17d3b7e63b30bada9cef4cb6aa158","n_tokens":233353,"ids_sha256":"9292ed54d582a3818d6208a893e4ef8b9cad4dbe1650f813a2b42f8735f25bcd"} +{"kind":"digest","file":"fixtures/lang/kat_Geor.txt","cap_chars":200000,"text_sha256":"c69a1aa8fb80b41459c98c15c756eb8f39a9151468b1d5b3f9bbffe4d82eb87a","n_tokens":206557,"ids_sha256":"8bd97ecee775dedb489fc7848d0b0442586ff17128567406b0ce5bfec8682fb1"} +{"kind":"digest","file":"fixtures/lang/kor_Hang.txt","cap_chars":200000,"text_sha256":"68659781c288136c1caffcb10eec9130406ae8a7b003c1e327b7e3601eba041d","n_tokens":287647,"ids_sha256":"c50a2616941247934f150afdab22d90d89aac77a24d2712bd36f293934d59893"} +{"kind":"digest","file":"fixtures/lang/rus_Cyrl.txt","cap_chars":200000,"text_sha256":"5f967a82627afbd72c2f21c6e56183dfa5efc986ba34350c1cf9eea6c27a6132","n_tokens":75821,"ids_sha256":"3169ea521c4cc21933851c97801024be5a8aaf54a8a79d2664f7ffff04eb641c"} +{"kind":"digest","file":"fixtures/lang/tam_Taml.txt","cap_chars":200000,"text_sha256":"bf09e1bca55aeab34162d5dc61afec7ac976b4e66eb5833ac5a037305911d165","n_tokens":235114,"ids_sha256":"4058ceb711d91e6624ab7c75b6123f2fdea66faff8e3821c390e2040361257a5"} +{"kind":"digest","file":"fixtures/lang/tha_Thai.txt","cap_chars":200000,"text_sha256":"25209f4750b3d3f04440f98aeff50316a118cb982462a4763106e8747172b79c","n_tokens":207145,"ids_sha256":"0cf1201dad2ca37beea82c3dc91a9a4ecac21d9acc369519b20feeca8e04c983"} +{"kind":"digest","file":"fixtures/modalities/added_normalized_dense.txt","cap_chars":200000,"text_sha256":"9d49b3cacf40962c051a09f1350a0f5dc2fc210556889ed0f341cfe0cf0af4f9","n_tokens":40351,"ids_sha256":"688d2b1c81413ee29fb76ecd176bcfc7241cbbbd7b45b8850eec8f19beb0a1cc"} +{"kind":"digest","file":"fixtures/modalities/added_normalized_sparse.txt","cap_chars":200000,"text_sha256":"55383f52b02a5d9686f9e4347729e5c08e61b9cff77913bd94ceb65c2fdf7fe4","n_tokens":28000,"ids_sha256":"b6234c507e95399106aa3ccf4f49192b9bba910a14a6aa98727b0a0e914af501"} +{"kind":"digest","file":"fixtures/modalities/added_special_dense.txt","cap_chars":200000,"text_sha256":"472f90f6f2f5803626df5d7088f8ef12984de6e88fedf67723887511d35b64b2","n_tokens":52824,"ids_sha256":"23bf1a41df8f47435a9c1206eeabeb95bb326f084f0991d8e299e28d599301a2"} +{"kind":"digest","file":"fixtures/modalities/added_special_sparse.txt","cap_chars":200000,"text_sha256":"9c52602f7c8d009b11377743f95e5bfe9053694fc6b48c3dd6c1c6dc0681ab5a","n_tokens":31354,"ids_sha256":"7b03acbd4766ab469e5d12e20a30305139211e0b126f72c4b0ac987c35833ce7"} +{"kind":"digest","file":"fixtures/modalities/agentic-traces.txt","cap_chars":200000,"text_sha256":"ae9d63c1adc49f572d9af635ac1b0421461e4d1b92c5f35ea572980ff1e2480c","n_tokens":65930,"ids_sha256":"1d4d06f82533d3df11cc1885e8ef165a76c9885927d3f3455c93f45a3b8944e1"} +{"kind":"digest","file":"fixtures/modalities/agentic_swe.txt","cap_chars":200000,"text_sha256":"b2d31e91ff91bb6c9a3aec3832d25acbecf9b58d5c0674e364d3b287182e7253","n_tokens":79514,"ids_sha256":"4225ac679cdb4e01509b8f4e9651053914ef0294cfda8f487569295fdc021e68"} +{"kind":"digest","file":"fixtures/modalities/code_mixed.txt","cap_chars":200000,"text_sha256":"0fa7314727726db056433d36bc1bda021803be7f1d150cae83e362a3ff41821d","n_tokens":90218,"ids_sha256":"7c0bf113dba7618a459b7aa1f12bab335b53be099bb0fb58ec9de00b1cabd504"} +{"kind":"digest","file":"fixtures/modalities/math_latex.txt","cap_chars":200000,"text_sha256":"2d17be8704f1f7b4f2174a054c0b85d6d22039e00be8e4a487927d27f3852e90","n_tokens":60969,"ids_sha256":"64ea277712a321d74e66be144dc1d56324807a3d9cbb204632a070a829b0ce64"} diff --git a/bindings/python/tests/golden/goldens/llama-3.jsonl b/bindings/python/tests/golden/goldens/llama-3.jsonl new file mode 100644 index 000000000..34edeebf5 --- /dev/null +++ b/bindings/python/tests/golden/goldens/llama-3.jsonl @@ -0,0 +1,64 @@ +{"kind":"meta","model":"llama-3","tokenizer_file":"llama-3-tokenizer.json","tokenizers_version":"0.23.1"} +{"kind":"sample","source":"edge:empty","text":"","ids":[128000],"ids_no_specials":[],"tokens":["<|begin_of_text|>"],"offsets":[[0,0]],"type_ids":[0],"special_tokens_mask":[1],"word_ids":[null],"decoded":"","decoded_with_specials":"<|begin_of_text|>"} +{"kind":"sample","source":"edge:spaces","text":" ","ids":[128000,262],"ids_no_specials":[262],"tokens":["<|begin_of_text|>","ĠĠĠ"],"offsets":[[0,0],[0,3]],"type_ids":[0,0],"special_tokens_mask":[1,0],"word_ids":[null,0],"decoded":" ","decoded_with_specials":"<|begin_of_text|> "} +{"kind":"sample","source":"edge:hello","text":"Hello world","ids":[128000,9906,1917],"ids_no_specials":[9906,1917],"tokens":["<|begin_of_text|>","Hello","Ġworld"],"offsets":[[0,0],[0,5],[5,11]],"type_ids":[0,0,0],"special_tokens_mask":[1,0,0],"word_ids":[null,0,1],"decoded":"Hello world","decoded_with_specials":"<|begin_of_text|>Hello world"} +{"kind":"sample","source":"edge:punctuation","text":"Hello, world!! How's it going? (fine; thanks...)","ids":[128000,9906,11,1917,3001,2650,596,433,2133,30,320,63157,26,9523,33674],"ids_no_specials":[9906,11,1917,3001,2650,596,433,2133,30,320,63157,26,9523,33674],"tokens":["<|begin_of_text|>","Hello",",","Ġworld","!!","ĠHow","'s","Ġit","Ġgoing","?","Ġ(","fine",";","Ġthanks","...)"],"offsets":[[0,0],[0,5],[5,6],[6,12],[12,14],[14,18],[18,20],[20,23],[23,29],[29,30],[30,32],[32,36],[36,37],[37,44],[44,48]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,1,2,3,4,5,6,7,8,9,10,11,12,13],"decoded":"Hello, world!! How's it going? (fine; thanks...)","decoded_with_specials":"<|begin_of_text|>Hello, world!! How's it going? (fine; thanks...)"} +{"kind":"sample","source":"edge:whitespace-mix","text":"line one\nline two\r\n\tindented\n trailing ","ids":[128000,1074,832,198,1074,1403,319,197,485,16243,198,220,28848,256],"ids_no_specials":[1074,832,198,1074,1403,319,197,485,16243,198,220,28848,256],"tokens":["<|begin_of_text|>","line","Ġone","Ċ","line","Ġtwo","čĊ","ĉ","ind","ented","Ċ","Ġ","Ġtrailing","ĠĠ"],"offsets":[[0,0],[0,4],[4,8],[8,9],[9,13],[13,17],[17,19],[19,20],[20,23],[23,28],[28,29],[29,30],[30,39],[39,41]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,1,2,3,4,5,6,6,6,7,8,9,10],"decoded":"line one\nline two\r\n\tindented\n trailing ","decoded_with_specials":"<|begin_of_text|>line one\nline two\r\n\tindented\n trailing "} +{"kind":"sample","source":"edge:accents","text":"café naïve résumé — déjà vu","ids":[128000,936,59958,95980,588,9517,1264,978,2001,46939,33614],"ids_no_specials":[936,59958,95980,588,9517,1264,978,2001,46939,33614],"tokens":["<|begin_of_text|>","ca","fé","Ġnaï","ve","Ġré","sum","é","ĠâĢĶ","ĠdéjÃł","Ġvu"],"offsets":[[0,0],[0,2],[2,4],[4,8],[8,10],[10,13],[13,16],[16,17],[17,19],[19,24],[24,27]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,1,1,2,2,2,3,4,5],"decoded":"café naïve résumé — déjà vu","decoded_with_specials":"<|begin_of_text|>café naïve résumé — déjà vu"} +{"kind":"sample","source":"edge:accents-decomposed","text":"café naïve résumé","ids":[128000,936,1897,54939,308,2192,127104,588,312,54939,32423,54939],"ids_no_specials":[936,1897,54939,308,2192,127104,588,312,54939,32423,54939],"tokens":["<|begin_of_text|>","ca","fe","Ìģ","Ġn","ai","ÌĪ","ve","Ġre","Ìģ","sume","Ìģ"],"offsets":[[0,0],[0,2],[2,4],[4,5],[5,7],[7,9],[9,10],[10,12],[12,15],[15,16],[16,20],[20,21]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,1,2,2,3,3,4,5,5,6],"decoded":"café naïve résumé","decoded_with_specials":"<|begin_of_text|>café naïve résumé"} +{"kind":"sample","source":"edge:emoji","text":"🤗 emoji, families 👨‍👩‍👧‍👦, flags 🇫🇷 and skin tones 👍🏽","ids":[128000,9468,97,245,43465,11,8689,62904,101,102470,9468,239,102,102470,9468,239,100,102470,9468,239,99,11,8202,11410,229,104,9468,229,115,323,6930,43076,62904,235,9468,237,121],"ids_no_specials":[9468,97,245,43465,11,8689,62904,101,102470,9468,239,102,102470,9468,239,100,102470,9468,239,99,11,8202,11410,229,104,9468,229,115,323,6930,43076,62904,235,9468,237,121],"tokens":["<|begin_of_text|>","ðŁ","¤","Ĺ","Ġemoji",",","Ġfamilies","ĠðŁij","¨","âĢį","ðŁ","ij","©","âĢį","ðŁ","ij","§","âĢį","ðŁ","ij","¦",",","Ġflags","ĠðŁ","ĩ","«","ðŁ","ĩ","·","Ġand","Ġskin","Ġtones","ĠðŁij","į","ðŁ","ı","½"],"offsets":[[0,0],[0,1],[0,1],[0,1],[1,7],[7,8],[8,17],[17,19],[18,19],[19,20],[20,21],[20,21],[20,21],[21,22],[22,23],[22,23],[22,23],[23,24],[24,25],[24,25],[24,25],[25,26],[26,32],[32,34],[33,34],[33,34],[34,35],[34,35],[34,35],[35,39],[39,44],[44,50],[50,52],[51,52],[52,53],[52,53],[52,53]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,1,2,3,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,5,6,6,6,6,6,6,7,8,9,10,10,10,10,10],"decoded":"🤗 emoji, families 👨‍👩‍👧‍👦, flags 🇫🇷 and skin tones 👍🏽","decoded_with_specials":"<|begin_of_text|>🤗 emoji, families 👨‍👩‍👧‍👦, flags 🇫🇷 and skin tones 👍🏽"} +{"kind":"sample","source":"edge:cjk","text":"漢字とひらがなとカタカナが混ざった文章です。","ids":[128000,115953,19113,19732,104328,33503,29295,26854,19732,71493,47307,71493,96452,29295,107960,75694,100472,83125,38641,1811],"ids_no_specials":[115953,19113,19732,104328,33503,29295,26854,19732,71493,47307,71493,96452,29295,107960,75694,100472,83125,38641,1811],"tokens":["<|begin_of_text|>","æ¼¢","åŃĹ","ãģ¨","ãģ²","ãĤī","ãģĮ","ãģª","ãģ¨","ãĤ«","ãĤ¿","ãĤ«","ãĥĬ","ãģĮ","æ··","ãģĸ","ãģ£ãģŁ","æĸĩç«ł","ãģ§ãģĻ","ãĢĤ"],"offsets":[[0,0],[0,1],[1,2],[2,3],[3,4],[4,5],[5,6],[6,7],[7,8],[8,9],[9,10],[10,11],[11,12],[12,13],[13,14],[14,15],[15,17],[17,19],[19,21],[21,22]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"decoded":"漢字とひらがなとカタカナが混ざった文章です。","decoded_with_specials":"<|begin_of_text|>漢字とひらがなとカタカナが混ざった文章です。"} +{"kind":"sample","source":"edge:korean","text":"한국어 텍스트 조각","ids":[128000,112699,32179,10997,45204,54289,66610,101930],"ids_no_specials":[112699,32179,10997,45204,54289,66610,101930],"tokens":["<|begin_of_text|>","íķľêµŃ","ìĸ´","Ġí","ħį","ìĬ¤íĬ¸","Ġì¡°","ê°ģ"],"offsets":[[0,0],[0,2],[2,3],[3,5],[4,5],[5,7],[7,9],[9,10]],"type_ids":[0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0],"word_ids":[null,0,0,1,1,1,2,2],"decoded":"한국어 텍스트 조각","decoded_with_specials":"<|begin_of_text|>한국어 텍스트 조각"} +{"kind":"sample","source":"edge:rtl","text":"مرحبا بالعالم — שלום עולם","ids":[128000,101244,30925,103645,100700,24102,101952,2001,88898,50391,37769,251,17732,95,37769,250,147,251],"ids_no_specials":[101244,30925,103645,100700,24102,101952,2001,88898,50391,37769,251,17732,95,37769,250,147,251],"tokens":["<|begin_of_text|>","Ùħر","ØŃ","با","ĠباÙĦ","ع","اÙĦÙħ","ĠâĢĶ","Ġש","׾","×ķ×","Ŀ","Ġ×","¢","×ķ×","ľ","×","Ŀ"],"offsets":[[0,0],[0,2],[2,3],[3,5],[5,9],[9,10],[10,13],[13,15],[15,17],[17,18],[18,20],[19,20],[20,22],[21,22],[22,24],[23,24],[24,25],[24,25]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,1,1,1,2,3,3,3,3,4,4,4,4,4,4],"decoded":"مرحبا بالعالم — שלום עולם","decoded_with_specials":"<|begin_of_text|>مرحبا بالعالم — שלום עולם"} +{"kind":"sample","source":"edge:numbers","text":"1234567890, 3.14159, 1,000,000th","ids":[128000,4513,10961,16474,15,11,220,18,13,9335,2946,11,220,16,11,931,11,931,339],"ids_no_specials":[4513,10961,16474,15,11,220,18,13,9335,2946,11,220,16,11,931,11,931,339],"tokens":["<|begin_of_text|>","123","456","789","0",",","Ġ","3",".","141","59",",","Ġ","1",",","000",",","000","th"],"offsets":[[0,0],[0,3],[3,6],[6,9],[9,10],[10,11],[11,12],[12,13],[13,14],[14,17],[17,19],[19,20],[20,21],[21,22],[22,23],[23,26],[26,27],[27,30],[30,32]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17],"decoded":"1234567890, 3.14159, 1,000,000th","decoded_with_specials":"<|begin_of_text|>1234567890, 3.14159, 1,000,000th"} +{"kind":"sample","source":"edge:code","text":"def f(x):\n return x**2 # squared\nprint(f'{f(3)=}')","ids":[128000,755,282,2120,997,262,471,865,334,17,220,674,53363,198,1374,968,25097,69,7,18,11992,68283],"ids_no_specials":[755,282,2120,997,262,471,865,334,17,220,674,53363,198,1374,968,25097,69,7,18,11992,68283],"tokens":["<|begin_of_text|>","def","Ġf","(x","):Ċ","ĠĠĠ","Ġreturn","Ġx","**","2","Ġ","Ġ#","Ġsquared","Ċ","print","(f","'{","f","(","3",")=","}')"],"offsets":[[0,0],[0,3],[3,5],[5,7],[7,10],[10,13],[13,20],[20,22],[22,24],[24,25],[25,26],[26,28],[28,36],[36,37],[37,42],[42,44],[44,46],[46,47],[47,48],[48,49],[49,51],[51,54]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,19],"decoded":"def f(x):\n return x**2 # squared\nprint(f'{f(3)=}')","decoded_with_specials":"<|begin_of_text|>def f(x):\n return x**2 # squared\nprint(f'{f(3)=}')"} +{"kind":"sample","source":"edge:long-word","text":"Donaudampfschifffahrtsgesellschaftskapitänsmützenabzeichen","ids":[128000,35,6863,664,1141,3933,331,333,544,1494,3423,2034,288,70801,4991,391,275,15492,3647,29758,5797,370,3059,29424],"ids_no_specials":[35,6863,664,1141,3933,331,333,544,1494,3423,2034,288,70801,4991,391,275,15492,3647,29758,5797,370,3059,29424],"tokens":["<|begin_of_text|>","D","ona","ud","amp","fs","ch","if","ff","ah","rt","sg","es","ellschaft","sk","ap","it","än","sm","üt","zen","ab","ze","ichen"],"offsets":[[0,0],[0,1],[1,4],[4,6],[6,9],[9,11],[11,13],[13,15],[15,17],[17,19],[19,21],[21,23],[23,25],[25,34],[34,36],[36,38],[38,40],[40,42],[42,44],[44,46],[46,49],[49,51],[51,53],[53,58]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"decoded":"Donaudampfschifffahrtsgesellschaftskapitänsmützenabzeichen","decoded_with_specials":"<|begin_of_text|>Donaudampfschifffahrtsgesellschaftskapitänsmützenabzeichen"} +{"kind":"sample","source":"edge:url-email","text":"https://example.com/a/b?q=1&r=2#frag user.name+tag@example.co.uk","ids":[128000,2485,1129,8858,916,14520,3554,44882,28,16,61717,28,17,2,34298,1217,2710,10,4681,36587,6973,15549],"ids_no_specials":[2485,1129,8858,916,14520,3554,44882,28,16,61717,28,17,2,34298,1217,2710,10,4681,36587,6973,15549],"tokens":["<|begin_of_text|>","https","://","example",".com","/a","/b","?q","=","1","&r","=","2","#","frag","Ġuser",".name","+","tag","@example",".co",".uk"],"offsets":[[0,0],[0,5],[5,8],[8,15],[15,19],[19,21],[21,23],[23,25],[25,26],[26,27],[27,29],[29,30],[30,31],[31,32],[32,36],[36,41],[41,46],[46,47],[47,50],[50,58],[58,61],[61,64]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,1,2,3,4,5,6,7,8,9,10,11,12,12,13,14,15,15,16,17,18],"decoded":"https://example.com/a/b?q=1&r=2#frag user.name+tag@example.co.uk","decoded_with_specials":"<|begin_of_text|>https://example.com/a/b?q=1&r=2#frag user.name+tag@example.co.uk"} +{"kind":"sample","source":"edge:unicode-spaces","text":"a b c　d","ids":[128000,64,115648,105584,66,23249,67],"ids_no_specials":[64,115648,105584,66,23249,67],"tokens":["<|begin_of_text|>","a","Âłb","âĢī","c","ãĢĢ","d"],"offsets":[[0,0],[0,1],[1,3],[3,4],[4,5],[5,6],[6,7]],"type_ids":[0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0],"word_ids":[null,0,1,2,2,3,3],"decoded":"a b c　d","decoded_with_specials":"<|begin_of_text|>a b c　d"} +{"kind":"sample","source":"edge:math","text":"∑ᵢ xᵢ² ≤ ∫₀^∞ e⁻ˣ dx ≈ 1","ids":[128000,22447,239,157,113,95,865,157,113,95,30556,38394,12264,104,90769,61,22447,252,384,53233,119,135,96,14142,118792,220,16],"ids_no_specials":[22447,239,157,113,95,865,157,113,95,30556,38394,12264,104,90769,61,22447,252,384,53233,119,135,96,14142,118792,220,16],"tokens":["<|begin_of_text|>","âĪ","ij","á","µ","¢","Ġx","á","µ","¢","²","Ġâī¤","ĠâĪ","«","âĤĢ","^","âĪ","ŀ","Ġe","âģ","»","Ë","£","Ġdx","ĠâīĪ","Ġ","1"],"offsets":[[0,0],[0,1],[0,1],[1,2],[1,2],[1,2],[2,4],[4,5],[4,5],[4,5],[5,6],[6,8],[8,10],[9,10],[10,11],[11,12],[12,13],[12,13],[13,15],[15,16],[15,16],[16,17],[16,17],[17,20],[20,22],[22,23],[23,24]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,1,1,1,1,2,3,4,4,5,6,6,6,7,8,8,8,8,9,10,11,12],"decoded":"∑ᵢ xᵢ² ≤ ∫₀^∞ e⁻ˣ dx ≈ 1","decoded_with_specials":"<|begin_of_text|>∑ᵢ xᵢ² ≤ ∫₀^∞ e⁻ˣ dx ≈ 1"} +{"kind":"sample","source":"fixtures/lang/amh_Ethi.txt[:160]","text":"- ፍርድ ቤቱ በአቶ ዮናታን ተስፋዬ ላይ የ6 አመት ከ6 ወር ጽኑ የእስር ቅጣት አስተላለፈ\n- አንድነት ዶክተር ቴድሮስ ለዓለም ጤና ድርጅት ዋና ዳይሬክተርነት በመመረጣቸው ደስታውን ገለጸ\n- ህንድ መጠነ ሰፊ የድንጋይ ከሰል ሃይል ጣብያዎች የመገንባት እ","ids":[128000,12,87189,235,235,157,230,255,157,233,113,87189,231,97,157,231,109,87189,231,254,157,232,254,157,231,114,87189,233,106,157,232,241,157,231,111,157,232,243,87189,231,108,157,230,113,157,235,233,157,233,105,87189,230,233,157,233,255,87189,233,101,21,87189,232,254,157,16281,157,231,113,87189,232,101,21,87189,233,230,157,230,255,87189,234,121,157,232,239,87189,233,101,157,232,98,157,230,113,157,230,255,87189,231,227,157,234,96,157,231,113,87189,232,254,157,230,113,157,231,108,157,230,233,157,230,230,157,235,230,198,12,87189,232,254,157,232,243,157,233,113,157,232,238,157,231,113,87189,233,114,157,232,255,157,231,108,157,230,255,87189,104258,157,233,113,157,230,106,157,230,113,87189,230,230,157,233,241,157,230,230,157,230,251,87189,234,97,157,232,241,87189,233,113,157,230,255,157,234,227,157,231,113,87189,233,233,157,232,241,87189,233,111,157,233,255,157,102958,157,232,255,157,231,108,157,230,255,157,232,238,157,231,113,87189,231,254,157,16281,157,16281,157,230,101,157,234,96,157,231,116,157,233,235,87189,233,108,157,230,113,157,231,111,157,233,235,157,232,243,87189,234,230,157,230,230,157,234,116,198,12,87189,230,227,157,232,243,157,233,113,87189,16281,157,234,254,157,232,238,87189,230,108,157,235,232,87189,233,101,157,233,113,157,232,243,157,234,233,157,233,255,87189,232,101,157,230,108,157,230,235,87189,230,225,157,233,255,157,230,235,87189,234,96,157,231,98,157,233,104,157,233,236,157,231,121,87189,233,101,157,16281,157,234,230,157,232,243,157,231,96,157,231,113,87189,232,98],"ids_no_specials":[12,87189,235,235,157,230,255,157,233,113,87189,231,97,157,231,109,87189,231,254,157,232,254,157,231,114,87189,233,106,157,232,241,157,231,111,157,232,243,87189,231,108,157,230,113,157,235,233,157,233,105,87189,230,233,157,233,255,87189,233,101,21,87189,232,254,157,16281,157,231,113,87189,232,101,21,87189,233,230,157,230,255,87189,234,121,157,232,239,87189,233,101,157,232,98,157,230,113,157,230,255,87189,231,227,157,234,96,157,231,113,87189,232,254,157,230,113,157,231,108,157,230,233,157,230,230,157,235,230,198,12,87189,232,254,157,232,243,157,233,113,157,232,238,157,231,113,87189,233,114,157,232,255,157,231,108,157,230,255,87189,104258,157,233,113,157,230,106,157,230,113,87189,230,230,157,233,241,157,230,230,157,230,251,87189,234,97,157,232,241,87189,233,113,157,230,255,157,234,227,157,231,113,87189,233,233,157,232,241,87189,233,111,157,233,255,157,102958,157,232,255,157,231,108,157,230,255,157,232,238,157,231,113,87189,231,254,157,16281,157,16281,157,230,101,157,234,96,157,231,116,157,233,235,87189,233,108,157,230,113,157,231,111,157,233,235,157,232,243,87189,234,230,157,230,230,157,234,116,198,12,87189,230,227,157,232,243,157,233,113,87189,16281,157,234,254,157,232,238,87189,230,108,157,235,232,87189,233,101,157,233,113,157,232,243,157,234,233,157,233,255,87189,232,101,157,230,108,157,230,235,87189,230,225,157,233,255,157,230,235,87189,234,96,157,231,98,157,233,104,157,233,236,157,231,121,87189,233,101,157,16281,157,234,230,157,232,243,157,231,96,157,231,113,87189,232,98],"tokens":["<|begin_of_text|>","-","Ġá","į","į","á","Ī","Ń","á","ĭ","µ","Ġá","ī","¤","á","ī","±","Ġá","ī","ł","á","Ĭ","ł","á","ī","¶","Ġá","ĭ","®","á","Ĭ","ĵ","á","ī","³","á","Ĭ","ķ","Ġá","ī","°","á","Ī","µ","á","į","ĭ","á","ĭ","¬","Ġá","Ī","ĭ","á","ĭ","Ń","Ġá","ĭ","¨","6","Ġá","Ĭ","ł","á","Īĺ","á","ī","µ","Ġá","Ĭ","¨","6","Ġá","ĭ","Ī","á","Ī","Ń","Ġá","Į","½","á","Ĭ","ij","Ġá","ĭ","¨","á","Ĭ","¥","á","Ī","µ","á","Ī","Ń","Ġá","ī","ħ","á","Į","£","á","ī","µ","Ġá","Ĭ","ł","á","Ī","µ","á","ī","°","á","Ī","ĭ","á","Ī","Ī","á","į","Ī","Ċ","-","Ġá","Ĭ","ł","á","Ĭ","ķ","á","ĭ","µ","á","Ĭ","IJ","á","ī","µ","Ġá","ĭ","¶","á","Ĭ","Ń","á","ī","°","á","Ī","Ń","Ġá","ī´","á","ĭ","µ","á","Ī","®","á","Ī","µ","Ġá","Ī","Ī","á","ĭ","ĵ","á","Ī","Ī","á","Ī","Ŀ","Ġá","Į","¤","á","Ĭ","ĵ","Ġá","ĭ","µ","á","Ī","Ń","á","Į","ħ","á","ī","µ","Ġá","ĭ","ĭ","á","Ĭ","ĵ","Ġá","ĭ","³","á","ĭ","Ń","á","Ī¬","á","Ĭ","Ń","á","ī","°","á","Ī","Ń","á","Ĭ","IJ","á","ī","µ","Ġá","ī","ł","á","Īĺ","á","Īĺ","á","Ī","¨","á","Į","£","á","ī","¸","á","ĭ","į","Ġá","ĭ","°","á","Ī","µ","á","ī","³","á","ĭ","į","á","Ĭ","ķ","Ġá","Į","Ī","á","Ī","Ī","á","Į","¸","Ċ","-","Ġá","Ī","ħ","á","Ĭ","ķ","á","ĭ","µ","Ġá","Īĺ","á","Į","ł","á","Ĭ","IJ","Ġá","Ī","°","á","į","Ĭ","Ġá","ĭ","¨","á","ĭ","µ","á","Ĭ","ķ","á","Į","ĭ","á","ĭ","Ń","Ġá","Ĭ","¨","á","Ī","°","á","Ī","į","Ġá","Ī","ĥ","á","ĭ","Ń","á","Ī","į","Ġá","Į","£","á","ī","¥","á","ĭ","«","á","ĭ","İ","á","ī","½","Ġá","ĭ","¨","á","Īĺ","á","Į","Ī","á","Ĭ","ķ","á","ī","£","á","ī","µ","Ġá","Ĭ","¥"],"offsets":[[0,0],[0,1],[1,3],[2,3],[2,3],[3,4],[3,4],[3,4],[4,5],[4,5],[4,5],[5,7],[6,7],[6,7],[7,8],[7,8],[7,8],[8,10],[9,10],[9,10],[10,11],[10,11],[10,11],[11,12],[11,12],[11,12],[12,14],[13,14],[13,14],[14,15],[14,15],[14,15],[15,16],[15,16],[15,16],[16,17],[16,17],[16,17],[17,19],[18,19],[18,19],[19,20],[19,20],[19,20],[20,21],[20,21],[20,21],[21,22],[21,22],[21,22],[22,24],[23,24],[23,24],[24,25],[24,25],[24,25],[25,27],[26,27],[26,27],[27,28],[28,30],[29,30],[29,30],[30,31],[30,31],[31,32],[31,32],[31,32],[32,34],[33,34],[33,34],[34,35],[35,37],[36,37],[36,37],[37,38],[37,38],[37,38],[38,40],[39,40],[39,40],[40,41],[40,41],[40,41],[41,43],[42,43],[42,43],[43,44],[43,44],[43,44],[44,45],[44,45],[44,45],[45,46],[45,46],[45,46],[46,48],[47,48],[47,48],[48,49],[48,49],[48,49],[49,50],[49,50],[49,50],[50,52],[51,52],[51,52],[52,53],[52,53],[52,53],[53,54],[53,54],[53,54],[54,55],[54,55],[54,55],[55,56],[55,56],[55,56],[56,57],[56,57],[56,57],[57,58],[58,59],[59,61],[60,61],[60,61],[61,62],[61,62],[61,62],[62,63],[62,63],[62,63],[63,64],[63,64],[63,64],[64,65],[64,65],[64,65],[65,67],[66,67],[66,67],[67,68],[67,68],[67,68],[68,69],[68,69],[68,69],[69,70],[69,70],[69,70],[70,72],[71,72],[72,73],[72,73],[72,73],[73,74],[73,74],[73,74],[74,75],[74,75],[74,75],[75,77],[76,77],[76,77],[77,78],[77,78],[77,78],[78,79],[78,79],[78,79],[79,80],[79,80],[79,80],[80,82],[81,82],[81,82],[82,83],[82,83],[82,83],[83,85],[84,85],[84,85],[85,86],[85,86],[85,86],[86,87],[86,87],[86,87],[87,88],[87,88],[87,88],[88,90],[89,90],[89,90],[90,91],[90,91],[90,91],[91,93],[92,93],[92,93],[93,94],[93,94],[93,94],[94,95],[94,95],[95,96],[95,96],[95,96],[96,97],[96,97],[96,97],[97,98],[97,98],[97,98],[98,99],[98,99],[98,99],[99,100],[99,100],[99,100],[100,102],[101,102],[101,102],[102,103],[102,103],[103,104],[103,104],[104,105],[104,105],[104,105],[105,106],[105,106],[105,106],[106,107],[106,107],[106,107],[107,108],[107,108],[107,108],[108,110],[109,110],[109,110],[110,111],[110,111],[110,111],[111,112],[111,112],[111,112],[112,113],[112,113],[112,113],[113,114],[113,114],[113,114],[114,116],[115,116],[115,116],[116,117],[116,117],[116,117],[117,118],[117,118],[117,118],[118,119],[119,120],[120,122],[121,122],[121,122],[122,123],[122,123],[122,123],[123,124],[123,124],[123,124],[124,126],[125,126],[126,127],[126,127],[126,127],[127,128],[127,128],[127,128],[128,130],[129,130],[129,130],[130,131],[130,131],[130,131],[131,133],[132,133],[132,133],[133,134],[133,134],[133,134],[134,135],[134,135],[134,135],[135,136],[135,136],[135,136],[136,137],[136,137],[136,137],[137,139],[138,139],[138,139],[139,140],[139,140],[139,140],[140,141],[140,141],[140,141],[141,143],[142,143],[142,143],[143,144],[143,144],[143,144],[144,145],[144,145],[144,145],[145,147],[146,147],[146,147],[147,148],[147,148],[147,148],[148,149],[148,149],[148,149],[149,150],[149,150],[149,150],[150,151],[150,151],[150,151],[151,153],[152,153],[152,153],[153,154],[153,154],[154,155],[154,155],[154,155],[155,156],[155,156],[155,156],[156,157],[156,157],[156,157],[157,158],[157,158],[157,158],[158,160],[159,160],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,7,7,7,8,9,9,9,9,9,9,9,9,10,10,10,11,12,12,12,12,12,12,13,13,13,13,13,13,14,14,14,14,14,14,14,14,14,14,14,14,15,15,15,15,15,15,15,15,15,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,17,18,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,20,20,20,20,20,20,20,20,20,20,20,20,21,21,21,21,21,21,21,21,21,21,21,22,22,22,22,22,22,22,22,22,22,22,22,23,23,23,23,23,23,24,24,24,24,24,24,24,24,24,24,24,24,25,25,25,25,25,25,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,29,29,29,29,29,29,29,29,29,30,31,32,32,32,32,32,32,32,32,32,33,33,33,33,33,33,33,33,34,34,34,34,34,34,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,36,36,36,36,36,36,36,36,36,37,37,37,37,37,37,37,37,37,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,40,40,40],"decoded":"- ፍርድ ቤቱ በአቶ ዮናታን ተስፋዬ ላይ የ6 አመት ከ6 ወር ጽኑ የእስር ቅጣት አስተላለፈ\n- አንድነት ዶክተር ቴድሮስ ለዓለም ጤና ድርጅት ዋና ዳይሬክተርነት በመመረጣቸው ደስታውን ገለጸ\n- ህንድ መጠነ ሰፊ የድንጋይ ከሰል ሃይል ጣብያዎች የመገንባት እ","decoded_with_specials":"<|begin_of_text|>- ፍርድ ቤቱ በአቶ ዮናታን ተስፋዬ ላይ የ6 አመት ከ6 ወር ጽኑ የእስር ቅጣት አስተላለፈ\n- አንድነት ዶክተር ቴድሮስ ለዓለም ጤና ድርጅት ዋና ዳይሬክተርነት በመመረጣቸው ደስታውን ገለጸ\n- ህንድ መጠነ ሰፊ የድንጋይ ከሰል ሃይል ጣብያዎች የመገንባት እ"} +{"kind":"sample","source":"fixtures/lang/arb_Arab.txt[:160]","text":"[بالصور] ترقوميا : الموت يطفىء باقة ورد قيد التفتح !!\nالخليل – دوت كوم – من غسان عبد الحميد - لم يخطر في بال أهالي \"ترقوميا\" التي انخلع قلبها وهي تودع التراب جث","ids":[128000,58,103151,112551,60,101097,28590,100397,101160,551,54579,101414,74374,107505,56157,102972,83711,101581,125589,78803,100526,96057,101080,30925,758,4999,32482,107481,96298,1389,45430,101414,88041,100397,1389,64337,100686,103640,103486,100688,10386,100526,482,102505,74374,36344,103623,78373,100700,64515,16552,102967,330,100337,28590,100397,101160,1,102626,100407,107481,24102,117627,100338,114388,40534,70523,24102,106762,71704,83268,85632],"ids_no_specials":[58,103151,112551,60,101097,28590,100397,101160,551,54579,101414,74374,107505,56157,102972,83711,101581,125589,78803,100526,96057,101080,30925,758,4999,32482,107481,96298,1389,45430,101414,88041,100397,1389,64337,100686,103640,103486,100688,10386,100526,482,102505,74374,36344,103623,78373,100700,64515,16552,102967,330,100337,28590,100397,101160,1,102626,100407,107481,24102,117627,100338,114388,40534,70523,24102,106762,71704,83268,85632],"tokens":["<|begin_of_text|>","[","باÙĦ","صÙĪر","]","Ġتر","ÙĤ","ÙĪÙħ","ÙĬا","Ġ:","ĠاÙĦÙħ","ÙĪت","ĠÙĬ","Ø·Ùģ","Ùī","Ø¡","Ġبا","ÙĤØ©","ĠÙĪرد","ĠÙĤ","ÙĬد","ĠاÙĦت","Ùģت","ØŃ","Ġ!","!Ċ","اÙĦ","Ø®ÙĦ","ÙĬÙĦ","ĠâĢĵ","Ġد","ÙĪت","ĠÙĥ","ÙĪÙħ","ĠâĢĵ","ĠÙħÙĨ","Ġغ","ساÙĨ","Ġعبد","ĠاÙĦØŃ","Ùħ","ÙĬد","Ġ-","ĠÙĦÙħ","ĠÙĬ","Ø®","طر","ĠÙģÙĬ","ĠباÙĦ","ĠØ£","Ùĩ","اÙĦÙĬ","Ġ\"","تر","ÙĤ","ÙĪÙħ","ÙĬا","\"","ĠاÙĦتÙĬ","ĠاÙĨ","Ø®ÙĦ","ع","ĠÙĤÙĦب","Ùĩا","ĠÙĪÙĩÙĬ","Ġت","ÙĪد","ع","ĠاÙĦتر","اب","Ġج","Ø«"],"offsets":[[0,0],[0,1],[1,4],[4,7],[7,8],[8,11],[11,12],[12,14],[14,16],[16,18],[18,22],[22,24],[24,26],[26,28],[28,29],[29,30],[30,33],[33,35],[35,39],[39,41],[41,43],[43,47],[47,49],[49,50],[50,52],[52,54],[54,56],[56,58],[58,60],[60,62],[62,64],[64,66],[66,68],[68,70],[70,72],[72,75],[75,77],[77,80],[80,84],[84,88],[88,89],[89,91],[91,93],[93,96],[96,98],[98,99],[99,101],[101,104],[104,108],[108,110],[110,111],[111,114],[114,116],[116,118],[118,119],[119,121],[121,123],[123,124],[124,129],[129,132],[132,134],[134,135],[135,139],[139,141],[141,145],[145,147],[147,149],[149,150],[150,155],[155,157],[157,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,1,2,2,2,2,3,4,4,5,5,5,5,6,6,7,8,8,9,9,9,10,10,11,11,11,12,13,13,14,14,15,16,17,17,18,19,19,19,20,21,22,22,22,23,24,25,25,25,26,27,27,27,27,28,29,30,30,30,31,31,32,33,33,33,34,34,35,35],"decoded":"[بالصور] ترقوميا : الموت يطفىء باقة ورد قيد التفتح !!\nالخليل – دوت كوم – من غسان عبد الحميد - لم يخطر في بال أهالي \"ترقوميا\" التي انخلع قلبها وهي تودع التراب جث","decoded_with_specials":"<|begin_of_text|>[بالصور] ترقوميا : الموت يطفىء باقة ورد قيد التفتح !!\nالخليل – دوت كوم – من غسان عبد الحميد - لم يخطر في بال أهالي \"ترقوميا\" التي انخلع قلبها وهي تودع التراب جث"} +{"kind":"sample","source":"fixtures/lang/ben_Beng.txt[:160]","text":"গ্রহ নীহারিকা\nগ্রহ নীহারিকা (ইংরেজি ভাষায়: Planetary nebula) এক বিশেষ ধরনের গ্যাসীয় নীহারিকা। যেসব তারার ভর কম, নির্দিষ্টভাবে বলতে গেলে যেসব তারার ভর সূর্যের ","ids":[128000,11372,245,53906,108,11372,117,36278,101,28025,222,11372,117,50228,108,81278,243,42412,198,11372,245,53906,108,11372,117,36278,101,28025,222,11372,117,50228,108,81278,243,42412,320,11372,229,11372,224,73358,60008,11372,250,62456,36278,255,50228,115,50228,107,11372,120,25,9878,16238,81967,5724,8,36278,237,11372,243,36278,105,81278,114,60008,11372,115,36278,100,73358,87648,60008,73358,36278,245,53906,107,50228,116,28025,222,11372,107,11372,120,36278,101,28025,222,11372,117,50228,108,81278,243,42412,100278,36278,107,60008,11372,116,11372,105,36278,97,50228,108,50228,108,36278,255,73358,36278,243,11372,106,11,36278,101,62456,73358,53906,99,81278,115,53906,253,11372,255,50228,105,60008,36278,105,11372,110,11372,97,60008,36278,245,60008,11372,110,60008,36278,107,60008,11372,116,11372,105,36278,97,50228,108,50228,108,36278,255,73358,36278,116,28025,224,73358,53906,107,60008,73358,220],"ids_no_specials":[11372,245,53906,108,11372,117,36278,101,28025,222,11372,117,50228,108,81278,243,42412,198,11372,245,53906,108,11372,117,36278,101,28025,222,11372,117,50228,108,81278,243,42412,320,11372,229,11372,224,73358,60008,11372,250,62456,36278,255,50228,115,50228,107,11372,120,25,9878,16238,81967,5724,8,36278,237,11372,243,36278,105,81278,114,60008,11372,115,36278,100,73358,87648,60008,73358,36278,245,53906,107,50228,116,28025,222,11372,107,11372,120,36278,101,28025,222,11372,117,50228,108,81278,243,42412,100278,36278,107,60008,11372,116,11372,105,36278,97,50228,108,50228,108,36278,255,73358,36278,243,11372,106,11,36278,101,62456,73358,53906,99,81278,115,53906,253,11372,255,50228,105,60008,36278,105,11372,110,11372,97,60008,36278,245,60008,11372,110,60008,36278,107,60008,11372,116,11372,105,36278,97,50228,108,50228,108,36278,255,73358,36278,116,28025,224,73358,53906,107,60008,73358,220],"tokens":["<|begin_of_text|>","à¦","Ĺ","à§įà¦","°","à¦","¹","Ġà¦","¨","à§","Ģ","à¦","¹","াà¦","°","িà¦","ķ","া","Ċ","à¦","Ĺ","à§įà¦","°","à¦","¹","Ġà¦","¨","à§","Ģ","à¦","¹","াà¦","°","িà¦","ķ","া","Ġ(","à¦","ĩ","à¦","Ĥ","র","à§ĩ","à¦","ľ","ি","Ġà¦","Ń","াà¦","·","াà¦","¯","à¦","¼",":","ĠPlan","etary","Ġneb","ula",")","Ġà¦","ı","à¦","ķ","Ġà¦","¬","িà¦","¶","à§ĩ","à¦","·","Ġà¦","§","র","ন","à§ĩ","র","Ġà¦","Ĺ","à§įà¦","¯","াà¦","¸","à§","Ģ","à¦","¯","à¦","¼","Ġà¦","¨","à§","Ģ","à¦","¹","াà¦","°","িà¦","ķ","া","।","Ġà¦","¯","à§ĩ","à¦","¸","à¦","¬","Ġà¦","¤","াà¦","°","াà¦","°","Ġà¦","Ń","র","Ġà¦","ķ","à¦","®",",","Ġà¦","¨","ি","র","à§įà¦","¦","িà¦","·","à§įà¦","Ł","à¦","Ń","াà¦","¬","à§ĩ","Ġà¦","¬","à¦","²","à¦","¤","à§ĩ","Ġà¦","Ĺ","à§ĩ","à¦","²","à§ĩ","Ġà¦","¯","à§ĩ","à¦","¸","à¦","¬","Ġà¦","¤","াà¦","°","াà¦","°","Ġà¦","Ń","র","Ġà¦","¸","à§","Ĥ","র","à§įà¦","¯","à§ĩ","র","Ġ"],"offsets":[[0,0],[0,1],[0,1],[1,3],[2,3],[3,4],[3,4],[4,6],[5,6],[6,7],[6,7],[7,8],[7,8],[8,10],[9,10],[10,12],[11,12],[12,13],[13,14],[14,15],[14,15],[15,17],[16,17],[17,18],[17,18],[18,20],[19,20],[20,21],[20,21],[21,22],[21,22],[22,24],[23,24],[24,26],[25,26],[26,27],[27,29],[29,30],[29,30],[30,31],[30,31],[31,32],[32,33],[33,34],[33,34],[34,35],[35,37],[36,37],[37,39],[38,39],[39,41],[40,41],[41,42],[41,42],[42,43],[43,48],[48,53],[53,57],[57,60],[60,61],[61,63],[62,63],[63,64],[63,64],[64,66],[65,66],[66,68],[67,68],[68,69],[69,70],[69,70],[70,72],[71,72],[72,73],[73,74],[74,75],[75,76],[76,78],[77,78],[78,80],[79,80],[80,82],[81,82],[82,83],[82,83],[83,84],[83,84],[84,85],[84,85],[85,87],[86,87],[87,88],[87,88],[88,89],[88,89],[89,91],[90,91],[91,93],[92,93],[93,94],[94,95],[95,97],[96,97],[97,98],[98,99],[98,99],[99,100],[99,100],[100,102],[101,102],[102,104],[103,104],[104,106],[105,106],[106,108],[107,108],[108,109],[109,111],[110,111],[111,112],[111,112],[112,113],[113,115],[114,115],[115,116],[116,117],[117,119],[118,119],[119,121],[120,121],[121,123],[122,123],[123,124],[123,124],[124,126],[125,126],[126,127],[127,129],[128,129],[129,130],[129,130],[130,131],[130,131],[131,132],[132,134],[133,134],[134,135],[135,136],[135,136],[136,137],[137,139],[138,139],[139,140],[140,141],[140,141],[141,142],[141,142],[142,144],[143,144],[144,146],[145,146],[146,148],[147,148],[148,150],[149,150],[150,151],[151,153],[152,153],[153,154],[153,154],[154,155],[155,157],[156,157],[157,158],[158,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,1,1,1,1,2,2,3,3,3,3,4,4,5,5,6,6,7,7,8,8,8,8,9,9,10,10,10,10,11,11,12,12,13,14,15,15,16,16,16,17,17,17,18,19,19,20,20,21,21,22,22,22,23,23,24,24,25,26,26,26,26,27,27,28,28,29,29,29,30,30,30,30,31,31,32,32,33,33,34,34,35,35,35,35,36,36,37,37,38,38,38,38,39,39,40,40,41,41,42,42,43,43,43,43,43,44,44,45,45,46,46,47,47,47,48,48,48,48,49,50,50,51,51,52,52,53,53,54,54,54,54,55,55,56,57,57,57,57,57,57,58,59,59,60,60,60,61,62,62,63,63,63,63,63,64,64,65,65,66,66,67,67,67,68,68,69,69,69,70,70,71,71,72],"decoded":"গ্রহ নীহারিকা\nগ্রহ নীহারিকা (ইংরেজি ভাষায়: Planetary nebula) এক বিশেষ ধরনের গ্যাসীয় নীহারিকা। যেসব তারার ভর কম, নির্দিষ্টভাবে বলতে গেলে যেসব তারার ভর সূর্যের ","decoded_with_specials":"<|begin_of_text|>গ্রহ নীহারিকা\nগ্রহ নীহারিকা (ইংরেজি ভাষায়: Planetary nebula) এক বিশেষ ধরনের গ্যাসীয় নীহারিকা। যেসব তারার ভর কম, নির্দিষ্টভাবে বলতে গেলে যেসব তারার ভর সূর্যের "} +{"kind":"sample","source":"fixtures/lang/cmn_Hani.txt[:160]","text":"強力建議大家未來盡量避免英航阿~~~\n班機原訂航程\n6/9 台北→香港\n6/9 香港→倫敦 (原訂23:15起飛，4:50am抵達)\n6/10 倫敦→斯德哥爾摩 (原訂7:40am起飛，11:05am抵達)\n就在第二段，英航no.25班機在香港機場兩度離開閘口又兩度返航\n第一次因有旅客身體不適 (好，不怪他，消耗時間也","ids":[128000,104195,8239,32438,104760,109429,39442,102993,16555,94,33857,111098,103048,83947,104743,103478,5940,89241,104615,101513,53229,116952,104743,39607,198,21,14,24,109142,49409,52118,110171,198,21,14,24,123147,52118,119862,121719,320,53229,116952,1419,25,868,72718,107850,3922,19,25,1135,309,116875,104067,340,21,14,605,107443,104,121719,52118,101011,101597,105736,104945,110581,320,53229,116952,22,25,1272,309,72718,107850,3922,806,25,2304,309,116875,104067,340,124978,106135,38574,3922,83947,104743,2201,13,914,104615,101513,19000,110171,101513,75267,109757,27479,106627,87447,34273,246,40526,102420,109757,27479,104196,104743,198,124810,63212,19361,104412,65854,96356,104146,16937,109431,320,53901,102836,107297,43511,3922,33420,120225,82973,75863],"ids_no_specials":[104195,8239,32438,104760,109429,39442,102993,16555,94,33857,111098,103048,83947,104743,103478,5940,89241,104615,101513,53229,116952,104743,39607,198,21,14,24,109142,49409,52118,110171,198,21,14,24,123147,52118,119862,121719,320,53229,116952,1419,25,868,72718,107850,3922,19,25,1135,309,116875,104067,340,21,14,605,107443,104,121719,52118,101011,101597,105736,104945,110581,320,53229,116952,22,25,1272,309,72718,107850,3922,806,25,2304,309,116875,104067,340,124978,106135,38574,3922,83947,104743,2201,13,914,104615,101513,19000,110171,101513,75267,109757,27479,106627,87447,34273,246,40526,102420,109757,27479,104196,104743,198,124810,63212,19361,104412,65854,96356,104146,16937,109431,320,53901,102836,107297,43511,3922,33420,120225,82973,75863],"tokens":["<|begin_of_text|>","å¼·","åĬ","Ľå»º","èŃ°","大家","æľª","ä¾Ĩ","çĽ","¡","éĩı","éģ¿","åħį","èĭ±","èĪª","éĺ¿","~~","~Ċ","çıŃ","æ©Ł","åİŁ","è¨Ĥ","èĪª","ç¨ĭ","Ċ","6","/","9","Ġåı°","åĮĹ","âĨĴ","é¦Ļ港","Ċ","6","/","9","Ġé¦Ļ港","âĨĴ","åĢ«","æķ¦","Ġ(","åİŁ","è¨Ĥ","23",":","15","èµ·","é£Ľ","ï¼Į","4",":","50","am","æĬµ","éģĶ",")Ċ","6","/","10","ĠåĢ","«","æķ¦","âĨĴ","æĸ¯","å¾·","åĵ¥","çĪ¾","æij©","Ġ(","åİŁ","è¨Ĥ","7",":","40","am","èµ·","é£Ľ","ï¼Į","11",":","05","am","æĬµ","éģĶ",")Ċ","å°±åľ¨","第äºĮ","段","ï¼Į","èĭ±","èĪª","no",".","25","çıŃ","æ©Ł","åľ¨","é¦Ļ港","æ©Ł","åł´","åħ©","度","éĽ¢","éĸĭ","éĸ","ĺ","åı£","åıĪ","åħ©","度","è¿Ķ","èĪª","Ċ","第ä¸Ģ次","åĽł","æľī","æĹħ","客","身","é«Ķ","ä¸į","éģ©","Ġ(","好","ï¼Įä¸į","æĢª","ä»ĸ","ï¼Į","æ¶Ī","èĢĹ","æĻĤéĸĵ","ä¹Ł"],"offsets":[[0,0],[0,1],[1,2],[1,3],[3,4],[4,6],[6,7],[7,8],[8,9],[8,9],[9,10],[10,11],[11,12],[12,13],[13,14],[14,15],[15,17],[17,19],[19,20],[20,21],[21,22],[22,23],[23,24],[24,25],[25,26],[26,27],[27,28],[28,29],[29,31],[31,32],[32,33],[33,35],[35,36],[36,37],[37,38],[38,39],[39,42],[42,43],[43,44],[44,45],[45,47],[47,48],[48,49],[49,51],[51,52],[52,54],[54,55],[55,56],[56,57],[57,58],[58,59],[59,61],[61,63],[63,64],[64,65],[65,67],[67,68],[68,69],[69,71],[71,73],[72,73],[73,74],[74,75],[75,76],[76,77],[77,78],[78,79],[79,80],[80,82],[82,83],[83,84],[84,85],[85,86],[86,88],[88,90],[90,91],[91,92],[92,93],[93,95],[95,96],[96,98],[98,100],[100,101],[101,102],[102,104],[104,106],[106,108],[108,109],[109,110],[110,111],[111,112],[112,114],[114,115],[115,117],[117,118],[118,119],[119,120],[120,122],[122,123],[123,124],[124,125],[125,126],[126,127],[127,128],[128,129],[128,129],[129,130],[130,131],[131,132],[132,133],[133,134],[134,135],[135,136],[136,139],[139,140],[140,141],[141,142],[142,143],[143,144],[144,145],[145,146],[146,147],[147,149],[149,150],[150,152],[152,153],[153,154],[154,155],[155,156],[156,157],[157,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,2,2,2,2,2,2,3,4,5,6,7,7,8,8,9,10,11,12,13,14,14,14,15,16,16,17,18,19,20,20,21,22,23,24,25,25,25,26,27,28,29,30,30,30,31,31,31,31,31,31,32,33,33,34,35,36,37,37,37,38,39,40,41,42,42,42,43,44,44,44,45,45,45,45,46,47,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,49,50,50,50,50,50,50,50,50,50,51,52,53,53,53,54,54,54,54,54],"decoded":"強力建議大家未來盡量避免英航阿~~~\n班機原訂航程\n6/9 台北→香港\n6/9 香港→倫敦 (原訂23:15起飛，4:50am抵達)\n6/10 倫敦→斯德哥爾摩 (原訂7:40am起飛，11:05am抵達)\n就在第二段，英航no.25班機在香港機場兩度離開閘口又兩度返航\n第一次因有旅客身體不適 (好，不怪他，消耗時間也","decoded_with_specials":"<|begin_of_text|>強力建議大家未來盡量避免英航阿~~~\n班機原訂航程\n6/9 台北→香港\n6/9 香港→倫敦 (原訂23:15起飛，4:50am抵達)\n6/10 倫敦→斯德哥爾摩 (原訂7:40am起飛，11:05am抵達)\n就在第二段，英航no.25班機在香港機場兩度離開閘口又兩度返航\n第一次因有旅客身體不適 (好，不怪他，消耗時間也"} +{"kind":"sample","source":"fixtures/lang/ell_Grek.txt[:160]","text":"Θυμήσου με\nΗ πόλη προσφέρει πολλές ευκαιρίες - πήγαινε στο εμπορικό κέντρο, στο σαλόνι ομορφιάς , σε κλάμπ και διασκέδασε με την ψυχή σου.\nΔημιούργησε μαγευτικά","ids":[128000,103808,54556,104522,125907,100577,198,100573,109989,101243,101361,106707,114695,105385,100861,101040,107574,119311,101381,100624,482,52845,74030,60474,90002,111987,101101,60247,113828,39179,102029,72738,127449,11,101101,48823,102095,103164,30862,100544,117748,86134,101510,46742,1174,101396,72738,103961,101751,100429,102226,45028,112794,103748,101143,100577,100856,112091,54556,105336,115780,627,101561,42524,103878,100695,101388,42524,101143,105290,60474,102862,103643],"ids_no_specials":[103808,54556,104522,125907,100577,198,100573,109989,101243,101361,106707,114695,105385,100861,101040,107574,119311,101381,100624,482,52845,74030,60474,90002,111987,101101,60247,113828,39179,102029,72738,127449,11,101101,48823,102095,103164,30862,100544,117748,86134,101510,46742,1174,101396,72738,103961,101751,100429,102226,45028,112794,103748,101143,100577,100856,112091,54556,105336,115780,627,101561,42524,103878,100695,101388,42524,101143,105290,60474,102862,103643],"tokens":["<|begin_of_text|>","Îĺ","Ïħ","μή","ÏĥοÏħ","Ġμε","Ċ","ÎĹ","ĠÏĢÏĮ","λη","ĠÏĢÏģο","ÏĥÏĨ","ÎŃÏģει","ĠÏĢο","λλ","ÎŃÏĤ","ĠεÏħ","και","Ïģί","εÏĤ","Ġ-","ĠÏĢ","ή","γ","αι","νε","ĠÏĥÏĦο","Ġε","μÏĢο","Ïģ","ικÏĮ","Ġκ","ÎŃνÏĦÏģο",",","ĠÏĥÏĦο","ĠÏĥ","αλ","ÏĮν","ι","Ġο","μοÏģ","ÏĨ","ιά","ÏĤ","Ġ,","ĠÏĥε","Ġκ","λά","μÏĢ","Ġκαι","Ġδια","Ïĥ","κÎŃ","δα","Ïĥε","Ġμε","ĠÏĦην","ĠÏĪ","Ïħ","Ïĩή","ĠÏĥοÏħ",".Ċ","ÎĶ","η","μι","οÏį","Ïģγ","η","Ïĥε","Ġμα","γ","εÏħ","ÏĦικά"],"offsets":[[0,0],[0,1],[1,2],[2,4],[4,7],[7,10],[10,11],[11,12],[12,15],[15,17],[17,21],[21,23],[23,27],[27,30],[30,32],[32,34],[34,37],[37,40],[40,42],[42,44],[44,46],[46,48],[48,49],[49,50],[50,52],[52,54],[54,58],[58,60],[60,63],[63,64],[64,67],[67,69],[69,74],[74,75],[75,79],[79,81],[81,83],[83,85],[85,86],[86,88],[88,91],[91,92],[92,94],[94,95],[95,97],[97,100],[100,102],[102,104],[104,106],[106,110],[110,114],[114,115],[115,117],[117,119],[119,121],[121,124],[124,128],[128,130],[130,131],[131,133],[133,137],[137,139],[139,140],[140,141],[141,143],[143,145],[145,147],[147,148],[148,150],[150,153],[153,154],[154,156],[156,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,1,2,3,4,4,5,5,5,6,6,6,7,7,7,7,8,9,9,9,9,9,10,11,11,11,11,12,12,13,14,15,15,15,15,16,16,16,16,16,17,18,19,19,19,20,21,21,21,21,21,22,23,24,24,24,25,26,27,27,27,27,27,27,27,28,28,28,28],"decoded":"Θυμήσου με\nΗ πόλη προσφέρει πολλές ευκαιρίες - πήγαινε στο εμπορικό κέντρο, στο σαλόνι ομορφιάς , σε κλάμπ και διασκέδασε με την ψυχή σου.\nΔημιούργησε μαγευτικά","decoded_with_specials":"<|begin_of_text|>Θυμήσου με\nΗ πόλη προσφέρει πολλές ευκαιρίες - πήγαινε στο εμπορικό κέντρο, στο σαλόνι ομορφιάς , σε κλάμπ και διασκέδασε με την ψυχή σου.\nΔημιούργησε μαγευτικά"} +{"kind":"sample","source":"fixtures/lang/eng_Latn.txt[:160]","text":"|Viewing Single Post From: Spoilers for the Week of February 11th|\n|Lil||Feb 1 2013, 09:58 AM|\nDon't care about Chloe/Taniel/Jen-Jen. Don't care about Sami, rea","ids":[128000,91,860,287,11579,3962,5659,25,57049,28257,369,279,10563,315,7552,220,806,339,7511,91,43,321,8651,41691,220,16,220,679,18,11,220,2545,25,2970,6912,7511,8161,956,2512,922,60470,17146,12315,32801,268,12278,268,13,4418,956,2512,922,8388,72,11,312,64],"ids_no_specials":[91,860,287,11579,3962,5659,25,57049,28257,369,279,10563,315,7552,220,806,339,7511,91,43,321,8651,41691,220,16,220,679,18,11,220,2545,25,2970,6912,7511,8161,956,2512,922,60470,17146,12315,32801,268,12278,268,13,4418,956,2512,922,8388,72,11,312,64],"tokens":["<|begin_of_text|>","|","View","ing","ĠSingle","ĠPost","ĠFrom",":","ĠSpo","ilers","Ġfor","Ġthe","ĠWeek","Ġof","ĠFebruary","Ġ","11","th","|Ċ","|","L","il","||","Feb","Ġ","1","Ġ","201","3",",","Ġ","09",":","58","ĠAM","|Ċ","Don","'t","Ġcare","Ġabout","ĠChloe","/T","aniel","/J","en","-J","en",".","ĠDon","'t","Ġcare","Ġabout","ĠSam","i",",","Ġre","a"],"offsets":[[0,0],[0,1],[1,5],[5,8],[8,15],[15,20],[20,25],[25,26],[26,30],[30,35],[35,39],[39,43],[43,48],[48,51],[51,60],[60,61],[61,63],[63,65],[65,67],[67,68],[68,69],[69,71],[71,73],[73,76],[76,77],[77,78],[78,79],[79,82],[82,83],[83,84],[84,85],[85,87],[87,88],[88,90],[90,93],[93,95],[95,98],[98,100],[100,105],[105,111],[111,117],[117,119],[119,124],[124,126],[126,128],[128,130],[130,132],[132,133],[133,137],[137,139],[139,144],[144,150],[150,154],[154,155],[155,156],[156,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,1,2,3,4,5,5,6,7,8,9,10,11,12,13,14,15,15,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,35,36,36,37,37,38,39,40,41,42,43,43,44,45,45],"decoded":"|Viewing Single Post From: Spoilers for the Week of February 11th|\n|Lil||Feb 1 2013, 09:58 AM|\nDon't care about Chloe/Taniel/Jen-Jen. Don't care about Sami, rea","decoded_with_specials":"<|begin_of_text|>|Viewing Single Post From: Spoilers for the Week of February 11th|\n|Lil||Feb 1 2013, 09:58 AM|\nDon't care about Chloe/Taniel/Jen-Jen. Don't care about Sami, rea"} +{"kind":"sample","source":"fixtures/lang/heb_Hebr.txt[:160]","text":"סיכונים – אלמנט מרכזי, חיוני לפעילות מסחרית. להבין מה הסיכון הוא מאוד חשוב. החוויה האנושית עולה כי מי יודע איך לקחת סיכונים בזמן, היא לנצח גדולים. לזכור פוליטיק","ids":[128000,147,94,43336,249,37769,254,43336,251,1389,63060,50391,68406,95526,147,246,92611,51326,147,249,147,244,33545,11,17732,245,33545,37769,254,33545,78062,147,97,87171,43336,250,37769,103,92611,147,94,90993,51326,43336,103,13,78062,47071,76625,43336,253,92611,47071,70446,147,94,43336,249,37769,253,70446,37769,238,92611,59610,37769,241,17732,245,59511,37769,239,13,70446,90993,32793,32793,43336,242,70446,59610,95526,37769,102,43336,103,17732,95,37769,250,47071,17732,249,33545,92611,33545,17732,247,37769,241,87171,63060,43336,248,78062,147,100,90993,55614,17732,94,43336,249,37769,254,43336,251,89631,147,244,68406,147,253,11,70446,43336,238,78062,95526,147,99,90993,17732,240,86884,37769,250,43336,251,13,78062,147,244,147,249,37769,101,17732,97,37769,250,43336,246,43336,100],"ids_no_specials":[147,94,43336,249,37769,254,43336,251,1389,63060,50391,68406,95526,147,246,92611,51326,147,249,147,244,33545,11,17732,245,33545,37769,254,33545,78062,147,97,87171,43336,250,37769,103,92611,147,94,90993,51326,43336,103,13,78062,47071,76625,43336,253,92611,47071,70446,147,94,43336,249,37769,253,70446,37769,238,92611,59610,37769,241,17732,245,59511,37769,239,13,70446,90993,32793,32793,43336,242,70446,59610,95526,37769,102,43336,103,17732,95,37769,250,47071,17732,249,33545,92611,33545,17732,247,37769,241,87171,63060,43336,248,78062,147,100,90993,55614,17732,94,43336,249,37769,254,43336,251,89631,147,244,68406,147,253,11,70446,43336,238,78062,95526,147,99,90993,17732,240,86884,37769,250,43336,251,13,78062,147,244,147,249,37769,101,17732,97,37769,250,43336,246,43336,100],"tokens":["<|begin_of_text|>","×","¡","×Ļ×","Ľ","×ķ×","ł","×Ļ×","Ŀ","ĠâĢĵ","Ġ×IJ","׾","×ŀ","׳","×","ĺ","Ġ×ŀ","ר","×","Ľ","×","ĸ","×Ļ",",","Ġ×","Ĺ","×Ļ","×ķ×","ł","×Ļ","Ġ׾","×","¤","×¢","×Ļ×","ľ","×ķ×","ª","Ġ×ŀ","×","¡","×Ĺ","ר","×Ļ×","ª",".","Ġ׾","×Ķ","×ij","×Ļ×","Ł","Ġ×ŀ","×Ķ","Ġ×Ķ","×","¡","×Ļ×","Ľ","×ķ×","Ł","Ġ×Ķ","×ķ×","IJ","Ġ×ŀ","×IJ","×ķ×","ĵ","Ġ×","Ĺ","ש","×ķ×","ij",".","Ġ×Ķ","×Ĺ","×ķ","×ķ","×Ļ×","Ķ","Ġ×Ķ","×IJ","׳","×ķ×","©","×Ļ×","ª","Ġ×","¢","×ķ×","ľ","×Ķ","Ġ×","Ľ","×Ļ","Ġ×ŀ","×Ļ","Ġ×","Ļ","×ķ×","ĵ","×¢","Ġ×IJ","×Ļ×","ļ","Ġ׾","×","§","×Ĺ","ת","Ġ×","¡","×Ļ×","Ľ","×ķ×","ł","×Ļ×","Ŀ","Ġ×ij","×","ĸ","×ŀ","×","Ł",",","Ġ×Ķ","×Ļ×","IJ","Ġ׾","׳","×","¦","×Ĺ","Ġ×","Ĵ","×ĵ","×ķ×","ľ","×Ļ×","Ŀ",".","Ġ׾","×","ĸ","×","Ľ","×ķ×","¨","Ġ×","¤","×ķ×","ľ","×Ļ×","ĺ","×Ļ×","§"],"offsets":[[0,0],[0,1],[0,1],[1,3],[2,3],[3,5],[4,5],[5,7],[6,7],[7,9],[9,11],[11,12],[12,13],[13,14],[14,15],[14,15],[15,17],[17,18],[18,19],[18,19],[19,20],[19,20],[20,21],[21,22],[22,24],[23,24],[24,25],[25,27],[26,27],[27,28],[28,30],[30,31],[30,31],[31,32],[32,34],[33,34],[34,36],[35,36],[36,38],[38,39],[38,39],[39,40],[40,41],[41,43],[42,43],[43,44],[44,46],[46,47],[47,48],[48,50],[49,50],[50,52],[52,53],[53,55],[55,56],[55,56],[56,58],[57,58],[58,60],[59,60],[60,62],[62,64],[63,64],[64,66],[66,67],[67,69],[68,69],[69,71],[70,71],[71,72],[72,74],[73,74],[74,75],[75,77],[77,78],[78,79],[79,80],[80,82],[81,82],[82,84],[84,85],[85,86],[86,88],[87,88],[88,90],[89,90],[90,92],[91,92],[92,94],[93,94],[94,95],[95,97],[96,97],[97,98],[98,100],[100,101],[101,103],[102,103],[103,105],[104,105],[105,106],[106,108],[108,110],[109,110],[110,112],[112,113],[112,113],[113,114],[114,115],[115,117],[116,117],[117,119],[118,119],[119,121],[120,121],[121,123],[122,123],[123,125],[125,126],[125,126],[126,127],[127,128],[127,128],[128,129],[129,131],[131,133],[132,133],[133,135],[135,136],[136,137],[136,137],[137,138],[138,140],[139,140],[140,141],[141,143],[142,143],[143,145],[144,145],[145,146],[146,148],[148,149],[148,149],[149,150],[149,150],[150,152],[151,152],[152,154],[153,154],[154,156],[155,156],[156,158],[157,158],[158,160],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,1,2,2,2,2,2,2,3,3,3,3,3,3,3,4,5,5,5,5,5,5,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,8,9,9,9,9,9,10,10,11,11,11,11,11,11,11,12,12,12,13,13,13,13,14,14,14,14,14,15,16,16,16,16,16,16,17,17,17,17,17,17,17,18,18,18,18,18,19,19,19,20,20,21,21,21,21,21,22,22,22,23,23,23,23,23,24,24,24,24,24,24,24,24,25,25,25,25,25,25,26,27,27,27,28,28,28,28,28,29,29,29,29,29,29,29,30,31,31,31,31,31,31,31,32,32,32,32,32,32,32,32],"decoded":"סיכונים – אלמנט מרכזי, חיוני לפעילות מסחרית. להבין מה הסיכון הוא מאוד חשוב. החוויה האנושית עולה כי מי יודע איך לקחת סיכונים בזמן, היא לנצח גדולים. לזכור פוליטיק","decoded_with_specials":"<|begin_of_text|>סיכונים – אלמנט מרכזי, חיוני לפעילות מסחרית. להבין מה הסיכון הוא מאוד חשוב. החוויה האנושית עולה כי מי יודע איך לקחת סיכונים בזמן, היא לנצח גדולים. לזכור פוליטיק"} +{"kind":"sample","source":"fixtures/lang/hin_Deva.txt[:160]","text":"PHOTOS: न्यूज पढ़ते-पढ़ते अचानक ये क्या करने लगी एंकर!\nकुछ समय पहले एक टीवी चैनल पर एंकर खबर पढ़ रही थी और पीछे की स्क्रीन पर 10 मिनिट का पोर्न वीडियो चलता रहा,","ids":[128000,11079,59687,25,100282,100305,107440,111354,109604,111492,87262,102088,109604,35470,105966,105977,100393,35470,48909,100305,24810,100855,35470,101573,44747,100404,73414,101185,4999,65804,102716,103483,102348,35470,100549,101002,102646,44747,100443,104071,92911,100406,100404,73414,101185,120416,111354,100395,100697,44747,100518,44747,100358,84736,118215,35470,48909,44747,69258,100915,86133,101273,100406,220,605,92317,100556,102399,48909,24810,84736,101282,101495,100287,104686,100322,55675,116920,24810,100697,101201],"ids_no_specials":[11079,59687,25,100282,100305,107440,111354,109604,111492,87262,102088,109604,35470,105966,105977,100393,35470,48909,100305,24810,100855,35470,101573,44747,100404,73414,101185,4999,65804,102716,103483,102348,35470,100549,101002,102646,44747,100443,104071,92911,100406,100404,73414,101185,120416,111354,100395,100697,44747,100518,44747,100358,84736,118215,35470,48909,44747,69258,100915,86133,101273,100406,220,605,92317,100556,102399,48909,24810,84736,101282,101495,100287,104686,100322,55675,116920,24810,100697,101201],"tokens":["<|begin_of_text|>","PH","OTOS",":","Ġन","à¥įय","à¥Ĥà¤ľ","Ġपढ","़त","à¥ĩ-","प","ढ","़त","à¥ĩ","Ġà¤ħà¤ļ","ानà¤ķ","Ġय","à¥ĩ","Ġà¤ķ","à¥įय","ा","Ġà¤ķरन","à¥ĩ","Ġलà¤Ĺ","à¥Ģ","Ġà¤ı","à¤Ĥ","à¤ķर","!Ċ","à¤ķ","à¥ģà¤Ľ","Ġसमय","Ġपहल","à¥ĩ","Ġà¤ıà¤ķ","Ġà¤Ł","à¥Ģव","à¥Ģ","Ġà¤ļ","à¥Īन","ल","Ġपर","Ġà¤ı","à¤Ĥ","à¤ķर","Ġà¤ĸबर","Ġपढ","़","Ġरह","à¥Ģ","Ġथ","à¥Ģ","Ġà¤Ķर","Ġप","à¥Ģà¤Ľ","à¥ĩ","Ġà¤ķ","à¥Ģ","Ġस","à¥įà¤ķ","à¥įर","à¥Ģन","Ġपर","Ġ","10","Ġम","िन","à¤¿à¤Ł","Ġà¤ķ","ा","Ġप","à¥ĭर","à¥įन","Ġव","à¥Ģड","िय","à¥ĭ","Ġà¤ļलत","ा","Ġरह","ा,"],"offsets":[[0,0],[0,2],[2,6],[6,7],[7,9],[9,11],[11,13],[13,16],[16,18],[18,20],[20,21],[21,22],[22,24],[24,25],[25,28],[28,31],[31,33],[33,34],[34,36],[36,38],[38,39],[39,43],[43,44],[44,47],[47,48],[48,50],[50,51],[51,53],[53,55],[55,56],[56,58],[58,62],[62,66],[66,67],[67,70],[70,72],[72,74],[74,75],[75,77],[77,79],[79,80],[80,83],[83,85],[85,86],[86,88],[88,92],[92,95],[95,96],[96,99],[99,100],[100,102],[102,103],[103,106],[106,108],[108,110],[110,111],[111,113],[113,114],[114,116],[116,118],[118,120],[120,122],[122,125],[125,126],[126,128],[128,130],[130,132],[132,134],[134,136],[136,137],[137,139],[139,141],[141,143],[143,145],[145,147],[147,149],[149,150],[150,154],[154,155],[155,158],[158,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,1,2,3,4,5,6,7,8,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,23,24,25,26,27,28,29,30,31,32,33,34,35,35,36,37,38,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74],"decoded":"PHOTOS: न्यूज पढ़ते-पढ़ते अचानक ये क्या करने लगी एंकर!\nकुछ समय पहले एक टीवी चैनल पर एंकर खबर पढ़ रही थी और पीछे की स्क्रीन पर 10 मिनिट का पोर्न वीडियो चलता रहा,","decoded_with_specials":"<|begin_of_text|>PHOTOS: न्यूज पढ़ते-पढ़ते अचानक ये क्या करने लगी एंकर!\nकुछ समय पहले एक टीवी चैनल पर एंकर खबर पढ़ रही थी और पीछे की स्क्रीन पर 10 मिनिट का पोर्न वीडियो चलता रहा,"} +{"kind":"sample","source":"fixtures/lang/jpn_Jpan.txt[:160]","text":"Omni Dallas Parkwest Hotelでは4ツ星ホテルで、アイアンホース・ゴルフコース、Love Field Airport とテキサス・スタジアムから7.8kmかかるのところにあります。 優れたホテルにオープンし、ダラスにある古代の建築の象徴です。\n部屋\n快適なゲストルームには、モダンな設備を備えたプレ","ids":[128000,64588,7907,19051,5657,11285,14894,77181,19,103223,78519,122770,16556,5486,111090,39880,100560,249,61398,9458,103621,122981,47260,61398,5486,29351,8771,21348,112031,57933,62903,60868,22398,9458,105335,125581,91062,55031,22,13,23,16400,32149,117295,16144,110229,20230,57903,33541,1811,115799,103,103468,122770,20230,90962,107610,16073,15024,5486,99959,112718,117083,102491,31640,16144,127178,16144,47523,17599,112,38641,9174,114834,198,102395,109431,26854,105621,71634,33710,103202,102052,5486,102494,99959,16073,26854,120370,30512,104874,107800,118192],"ids_no_specials":[64588,7907,19051,5657,11285,14894,77181,19,103223,78519,122770,16556,5486,111090,39880,100560,249,61398,9458,103621,122981,47260,61398,5486,29351,8771,21348,112031,57933,62903,60868,22398,9458,105335,125581,91062,55031,22,13,23,16400,32149,117295,16144,110229,20230,57903,33541,1811,115799,103,103468,122770,20230,90962,107610,16073,15024,5486,99959,112718,117083,102491,31640,16144,127178,16144,47523,17599,112,38641,9174,114834,198,102395,109431,26854,105621,71634,33710,103202,102052,5486,102494,99959,16073,26854,120370,30512,104874,107800,118192],"tokens":["<|begin_of_text|>","Om","ni","ĠDallas","ĠPark","west","ĠHotel","ãģ§ãģ¯","4","ãĥĦ","æĺŁ","ãĥĽãĥĨãĥ«","ãģ§","ãĢģ","ãĤ¢ãĤ¤","ãĤ¢","ãĥ³ãĥ","Ľ","ãĥ¼ãĤ¹","ãĥ»","ãĤ´","ãĥ«ãĥķ","ãĤ³","ãĥ¼ãĤ¹","ãĢģ","Love","ĠField","ĠAirport","Ġãģ¨","ãĥĨ","ãĤŃ","ãĤµ","ãĤ¹","ãĥ»","ãĤ¹ãĤ¿","ãĤ¸ãĤ¢","ãĥł","ãģĭãĤī","7",".","8","km","ãģĭ","ãģĭãĤĭ","ãģ®","ãģ¨ãģĵãĤį","ãģ«","ãģĤãĤĬ","ãģ¾ãģĻ","ãĢĤ","ĠåĦ","ª","ãĤĮãģŁ","ãĥĽãĥĨãĥ«","ãģ«","ãĤª","ãĥ¼ãĥĹ","ãĥ³","ãģĹ","ãĢģ","ãĥĢ","ãĥ©ãĤ¹","ãģ«ãģĤãĤĭ","åı¤","代","ãģ®","建ç¯ī","ãģ®","象","å¾","´","ãģ§ãģĻ","ãĢĤĊ","éĥ¨å±ĭ","Ċ","å¿«","éģ©","ãģª","ãĤ²","ãĤ¹ãĥĪ","ãĥ«","ãĥ¼ãĥł","ãģ«ãģ¯","ãĢģ","ãĥ¢","ãĥĢ","ãĥ³","ãģª","è¨ŃåĤĻ","ãĤĴ","åĤĻ","ãģĪãģŁ","ãĥĹãĥ¬"],"offsets":[[0,0],[0,2],[2,4],[4,11],[11,16],[16,20],[20,26],[26,28],[28,29],[29,30],[30,31],[31,34],[34,35],[35,36],[36,38],[38,39],[39,41],[40,41],[41,43],[43,44],[44,45],[45,47],[47,48],[48,50],[50,51],[51,55],[55,61],[61,69],[69,71],[71,72],[72,73],[73,74],[74,75],[75,76],[76,78],[78,80],[80,81],[81,83],[83,84],[84,85],[85,86],[86,88],[88,89],[89,91],[91,92],[92,95],[95,96],[96,98],[98,100],[100,101],[101,103],[102,103],[103,105],[105,108],[108,109],[109,110],[110,112],[112,113],[113,114],[114,115],[115,116],[116,118],[118,121],[121,122],[122,123],[123,124],[124,126],[126,127],[127,128],[128,129],[128,129],[129,131],[131,133],[133,135],[135,136],[136,137],[137,138],[138,139],[139,140],[140,142],[142,143],[143,145],[145,147],[147,148],[148,149],[149,150],[150,151],[151,152],[152,154],[154,155],[155,156],[156,158],[158,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,1,2,2,3,3,4,5,5,5,5,6,6,6,6,6,6,7,7,7,7,7,8,8,9,10,11,11,11,11,11,12,12,12,12,12,13,14,15,16,16,16,16,16,16,16,16,17,18,18,18,18,18,18,18,18,18,19,19,19,19,19,19,19,19,19,19,19,19,19,20,21,22,23,23,23,23,23,23,23,23,24,24,24,24,24,24,24,24,24,24],"decoded":"Omni Dallas Parkwest Hotelでは4ツ星ホテルで、アイアンホース・ゴルフコース、Love Field Airport とテキサス・スタジアムから7.8kmかかるのところにあります。 優れたホテルにオープンし、ダラスにある古代の建築の象徴です。\n部屋\n快適なゲストルームには、モダンな設備を備えたプレ","decoded_with_specials":"<|begin_of_text|>Omni Dallas Parkwest Hotelでは4ツ星ホテルで、アイアンホース・ゴルフコース、Love Field Airport とテキサス・スタジアムから7.8kmかかるのところにあります。 優れたホテルにオープンし、ダラスにある古代の建築の象徴です。\n部屋\n快適なゲストルームには、モダンな設備を備えたプレ"} +{"kind":"sample","source":"fixtures/lang/kat_Geor.txt[:160]","text":"ჩვენ მსოფლიოს სხვადასხვა ქვეყანაში ვცხოვრობთ და სხვადასხვა ენაზე ვსაუბრობთ, მაგრამ საერთო მიზანი გვაერთიანებს. ჩვენი მთავარი მიზანი სამყაროს შემოქმედისა და ბიბლ","ids":[128000,36393,102,36393,243,36393,242,36393,250,220,36393,249,36393,94,157,30489,36393,97,36393,248,36393,246,157,30489,36393,94,220,36393,94,36393,106,36393,243,36393,238,36393,241,36393,238,36393,94,36393,106,36393,243,36393,238,220,36393,98,36393,243,36393,242,36393,100,36393,238,36393,250,36393,238,36393,101,36393,246,220,36393,243,36393,103,36393,106,157,30489,36393,243,36393,254,157,30489,36393,239,36393,245,220,36393,241,36393,238,220,36393,94,36393,106,36393,243,36393,238,36393,241,36393,238,36393,94,36393,106,36393,243,36393,238,220,36393,242,36393,250,36393,238,36393,244,36393,242,220,36393,243,36393,94,36393,238,36393,96,36393,239,36393,254,157,30489,36393,239,36393,245,11,220,36393,249,36393,238,36393,240,36393,254,36393,238,36393,249,220,36393,94,36393,238,36393,242,36393,254,36393,245,157,30489,220,36393,249,36393,246,36393,244,36393,238,36393,250,36393,246,220,36393,240,36393,243,36393,238,36393,242,36393,254,36393,245,36393,246,36393,238,36393,250,36393,242,36393,239,36393,94,13,220,36393,102,36393,243,36393,242,36393,250,36393,246,220,36393,249,36393,245,36393,238,36393,243,36393,238,36393,254,36393,246,220,36393,249,36393,246,36393,244,36393,238,36393,250,36393,246,220,36393,94,36393,238,36393,249,36393,100,36393,238,36393,254,157,30489,36393,94,220,36393,101,36393,242,36393,249,157,30489,36393,98,36393,249,36393,242,36393,241,36393,246,36393,94,36393,238,220,36393,241,36393,238,220,36393,239,36393,246,36393,239,36393,248],"ids_no_specials":[36393,102,36393,243,36393,242,36393,250,220,36393,249,36393,94,157,30489,36393,97,36393,248,36393,246,157,30489,36393,94,220,36393,94,36393,106,36393,243,36393,238,36393,241,36393,238,36393,94,36393,106,36393,243,36393,238,220,36393,98,36393,243,36393,242,36393,100,36393,238,36393,250,36393,238,36393,101,36393,246,220,36393,243,36393,103,36393,106,157,30489,36393,243,36393,254,157,30489,36393,239,36393,245,220,36393,241,36393,238,220,36393,94,36393,106,36393,243,36393,238,36393,241,36393,238,36393,94,36393,106,36393,243,36393,238,220,36393,242,36393,250,36393,238,36393,244,36393,242,220,36393,243,36393,94,36393,238,36393,96,36393,239,36393,254,157,30489,36393,239,36393,245,11,220,36393,249,36393,238,36393,240,36393,254,36393,238,36393,249,220,36393,94,36393,238,36393,242,36393,254,36393,245,157,30489,220,36393,249,36393,246,36393,244,36393,238,36393,250,36393,246,220,36393,240,36393,243,36393,238,36393,242,36393,254,36393,245,36393,246,36393,238,36393,250,36393,242,36393,239,36393,94,13,220,36393,102,36393,243,36393,242,36393,250,36393,246,220,36393,249,36393,245,36393,238,36393,243,36393,238,36393,254,36393,246,220,36393,249,36393,246,36393,244,36393,238,36393,250,36393,246,220,36393,94,36393,238,36393,249,36393,100,36393,238,36393,254,157,30489,36393,94,220,36393,101,36393,242,36393,249,157,30489,36393,98,36393,249,36393,242,36393,241,36393,246,36393,94,36393,238,220,36393,241,36393,238,220,36393,239,36393,246,36393,239,36393,248],"tokens":["<|begin_of_text|>","áĥ","©","áĥ","ķ","áĥ","Ķ","áĥ","ľ","Ġ","áĥ","Ľ","áĥ","¡","á","ĥĿ","áĥ","¤","áĥ","ļ","áĥ","ĺ","á","ĥĿ","áĥ","¡","Ġ","áĥ","¡","áĥ","®","áĥ","ķ","áĥ","IJ","áĥ","ĵ","áĥ","IJ","áĥ","¡","áĥ","®","áĥ","ķ","áĥ","IJ","Ġ","áĥ","¥","áĥ","ķ","áĥ","Ķ","áĥ","§","áĥ","IJ","áĥ","ľ","áĥ","IJ","áĥ","¨","áĥ","ĺ","Ġ","áĥ","ķ","áĥ","ª","áĥ","®","á","ĥĿ","áĥ","ķ","áĥ","ł","á","ĥĿ","áĥ","ij","áĥ","Ĺ","Ġ","áĥ","ĵ","áĥ","IJ","Ġ","áĥ","¡","áĥ","®","áĥ","ķ","áĥ","IJ","áĥ","ĵ","áĥ","IJ","áĥ","¡","áĥ","®","áĥ","ķ","áĥ","IJ","Ġ","áĥ","Ķ","áĥ","ľ","áĥ","IJ","áĥ","ĸ","áĥ","Ķ","Ġ","áĥ","ķ","áĥ","¡","áĥ","IJ","áĥ","£","áĥ","ij","áĥ","ł","á","ĥĿ","áĥ","ij","áĥ","Ĺ",",","Ġ","áĥ","Ľ","áĥ","IJ","áĥ","Ĵ","áĥ","ł","áĥ","IJ","áĥ","Ľ","Ġ","áĥ","¡","áĥ","IJ","áĥ","Ķ","áĥ","ł","áĥ","Ĺ","á","ĥĿ","Ġ","áĥ","Ľ","áĥ","ĺ","áĥ","ĸ","áĥ","IJ","áĥ","ľ","áĥ","ĺ","Ġ","áĥ","Ĵ","áĥ","ķ","áĥ","IJ","áĥ","Ķ","áĥ","ł","áĥ","Ĺ","áĥ","ĺ","áĥ","IJ","áĥ","ľ","áĥ","Ķ","áĥ","ij","áĥ","¡",".","Ġ","áĥ","©","áĥ","ķ","áĥ","Ķ","áĥ","ľ","áĥ","ĺ","Ġ","áĥ","Ľ","áĥ","Ĺ","áĥ","IJ","áĥ","ķ","áĥ","IJ","áĥ","ł","áĥ","ĺ","Ġ","áĥ","Ľ","áĥ","ĺ","áĥ","ĸ","áĥ","IJ","áĥ","ľ","áĥ","ĺ","Ġ","áĥ","¡","áĥ","IJ","áĥ","Ľ","áĥ","§","áĥ","IJ","áĥ","ł","á","ĥĿ","áĥ","¡","Ġ","áĥ","¨","áĥ","Ķ","áĥ","Ľ","á","ĥĿ","áĥ","¥","áĥ","Ľ","áĥ","Ķ","áĥ","ĵ","áĥ","ĺ","áĥ","¡","áĥ","IJ","Ġ","áĥ","ĵ","áĥ","IJ","Ġ","áĥ","ij","áĥ","ĺ","áĥ","ij","áĥ","ļ"],"offsets":[[0,0],[0,1],[0,1],[1,2],[1,2],[2,3],[2,3],[3,4],[3,4],[4,5],[5,6],[5,6],[6,7],[6,7],[7,8],[7,8],[8,9],[8,9],[9,10],[9,10],[10,11],[10,11],[11,12],[11,12],[12,13],[12,13],[13,14],[14,15],[14,15],[15,16],[15,16],[16,17],[16,17],[17,18],[17,18],[18,19],[18,19],[19,20],[19,20],[20,21],[20,21],[21,22],[21,22],[22,23],[22,23],[23,24],[23,24],[24,25],[25,26],[25,26],[26,27],[26,27],[27,28],[27,28],[28,29],[28,29],[29,30],[29,30],[30,31],[30,31],[31,32],[31,32],[32,33],[32,33],[33,34],[33,34],[34,35],[35,36],[35,36],[36,37],[36,37],[37,38],[37,38],[38,39],[38,39],[39,40],[39,40],[40,41],[40,41],[41,42],[41,42],[42,43],[42,43],[43,44],[43,44],[44,45],[45,46],[45,46],[46,47],[46,47],[47,48],[48,49],[48,49],[49,50],[49,50],[50,51],[50,51],[51,52],[51,52],[52,53],[52,53],[53,54],[53,54],[54,55],[54,55],[55,56],[55,56],[56,57],[56,57],[57,58],[57,58],[58,59],[59,60],[59,60],[60,61],[60,61],[61,62],[61,62],[62,63],[62,63],[63,64],[63,64],[64,65],[65,66],[65,66],[66,67],[66,67],[67,68],[67,68],[68,69],[68,69],[69,70],[69,70],[70,71],[70,71],[71,72],[71,72],[72,73],[72,73],[73,74],[73,74],[74,75],[75,76],[76,77],[76,77],[77,78],[77,78],[78,79],[78,79],[79,80],[79,80],[80,81],[80,81],[81,82],[81,82],[82,83],[83,84],[83,84],[84,85],[84,85],[85,86],[85,86],[86,87],[86,87],[87,88],[87,88],[88,89],[88,89],[89,90],[90,91],[90,91],[91,92],[91,92],[92,93],[92,93],[93,94],[93,94],[94,95],[94,95],[95,96],[95,96],[96,97],[97,98],[97,98],[98,99],[98,99],[99,100],[99,100],[100,101],[100,101],[101,102],[101,102],[102,103],[102,103],[103,104],[103,104],[104,105],[104,105],[105,106],[105,106],[106,107],[106,107],[107,108],[107,108],[108,109],[108,109],[109,110],[110,111],[111,112],[111,112],[112,113],[112,113],[113,114],[113,114],[114,115],[114,115],[115,116],[115,116],[116,117],[117,118],[117,118],[118,119],[118,119],[119,120],[119,120],[120,121],[120,121],[121,122],[121,122],[122,123],[122,123],[123,124],[123,124],[124,125],[125,126],[125,126],[126,127],[126,127],[127,128],[127,128],[128,129],[128,129],[129,130],[129,130],[130,131],[130,131],[131,132],[132,133],[132,133],[133,134],[133,134],[134,135],[134,135],[135,136],[135,136],[136,137],[136,137],[137,138],[137,138],[138,139],[138,139],[139,140],[139,140],[140,141],[141,142],[141,142],[142,143],[142,143],[143,144],[143,144],[144,145],[144,145],[145,146],[145,146],[146,147],[146,147],[147,148],[147,148],[148,149],[148,149],[149,150],[149,150],[150,151],[150,151],[151,152],[151,152],[152,153],[153,154],[153,154],[154,155],[154,155],[155,156],[156,157],[156,157],[157,158],[157,158],[158,159],[158,159],[159,160],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,7,7,7,7,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,9,10,10,10,10,10,10,10,10,10,10,10,10,10,11,11,11,11,11,11,11,11,11,11,11,11,11,12,12,12,12,12,12,12,12,12,12,12,12,12,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,14,15,15,15,15,15,15,15,15,15,15,15,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,17,17,17,17,17,17,17,17,17,17,17,17,17,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,20,20,20,20,20,21,21,21,21,21,21,21,21,21],"decoded":"ჩვენ მსოფლიოს სხვადასხვა ქვეყანაში ვცხოვრობთ და სხვადასხვა ენაზე ვსაუბრობთ, მაგრამ საერთო მიზანი გვაერთიანებს. ჩვენი მთავარი მიზანი სამყაროს შემოქმედისა და ბიბლ","decoded_with_specials":"<|begin_of_text|>ჩვენ მსოფლიოს სხვადასხვა ქვეყანაში ვცხოვრობთ და სხვადასხვა ენაზე ვსაუბრობთ, მაგრამ საერთო მიზანი გვაერთიანებს. ჩვენი მთავარი მიზანი სამყაროს შემოქმედისა და ბიბლ"} +{"kind":"sample","source":"fixtures/lang/kor_Hang.txt[:160]","text":"전화번호:\n위치: 뉴질랜드 > 남섬 > 말버러 > 블레넘\n가격대 (1박): 117,886 원 - 140,244 원\n4.5성급 — Lugano Motor Lodge 4.5*\n- 예약 옵션:\n- 트립어드바이저는 호텔스닷컴, Expedia, Agoda, Asia Web Direct 및 Boo","ids":[128000,66965,57390,73148,512,82001,60798,25,111068,103194,109817,871,102484,14901,105,871,101264,80104,61394,871,109327,101164,76242,246,198,115153,67945,320,16,104706,1680,220,8546,11,25399,102467,482,220,6860,11,13719,102467,198,19,13,20,33931,102662,2001,93590,5770,18079,44668,220,19,13,20,5736,12,96717,103168,39623,113,93131,512,12,107534,102365,32179,30446,126906,101464,16969,114497,25941,9019,115,124158,11,7943,4596,11,4701,14320,11,13936,5000,7286,101824,74784],"ids_no_specials":[66965,57390,73148,512,82001,60798,25,111068,103194,109817,871,102484,14901,105,871,101264,80104,61394,871,109327,101164,76242,246,198,115153,67945,320,16,104706,1680,220,8546,11,25399,102467,482,220,6860,11,13719,102467,198,19,13,20,33931,102662,2001,93590,5770,18079,44668,220,19,13,20,5736,12,96717,103168,39623,113,93131,512,12,107534,102365,32179,30446,126906,101464,16969,114497,25941,9019,115,124158,11,7943,4596,11,4701,14320,11,13936,5000,7286,101824,74784],"tokens":["<|begin_of_text|>","ìłĦ","íĻĶ","ë²Īíĺ¸",":Ċ","ìľĦ","ì¹ĺ",":","Ġëī´","ì§Ī","ëŀľëĵľ","Ġ>","ĠëĤ¨","ìĦ","¬","Ġ>","Ġë§IJ","ë²Ħ","룬","Ġ>","Ġë¸Ķ","ëłĪ","ëĦ","ĺ","Ċ","ê°Ģ격","ëĮĢ","Ġ(","1","ë°ķ","):","Ġ","117",",","886","ĠìĽIJ","Ġ-","Ġ","140",",","244","ĠìĽIJ","Ċ","4",".","5","ìĦ±","ê¸ī","ĠâĢĶ","ĠLug","ano","ĠMotor","ĠLodge","Ġ","4",".","5","*Ċ","-","ĠìĺĪ","ìķ½","Ġìĺ","µ","ìħĺ",":Ċ","-","ĠíĬ¸","립","ìĸ´","ëĵľ","ë°ĶìĿ´","ìłĢ","ëĬĶ","Ġíĺ¸íħĶ","ìĬ¤","ëĭ","·","ì»´",",","ĠExp","edia",",","ĠAg","oda",",","ĠAsia","ĠWeb","ĠDirect","Ġë°ı","ĠBoo"],"offsets":[[0,0],[0,1],[1,2],[2,4],[4,6],[6,7],[7,8],[8,9],[9,11],[11,12],[12,14],[14,16],[16,18],[18,19],[18,19],[19,21],[21,23],[23,24],[24,25],[25,27],[27,29],[29,30],[30,31],[30,31],[31,32],[32,34],[34,35],[35,37],[37,38],[38,39],[39,41],[41,42],[42,45],[45,46],[46,49],[49,51],[51,53],[53,54],[54,57],[57,58],[58,61],[61,63],[63,64],[64,65],[65,66],[66,67],[67,68],[68,69],[69,71],[71,75],[75,78],[78,84],[84,90],[90,91],[91,92],[92,93],[93,94],[94,96],[96,97],[97,99],[99,100],[100,102],[101,102],[102,103],[103,105],[105,106],[106,108],[108,109],[109,110],[110,111],[111,113],[113,114],[114,115],[115,118],[118,119],[119,120],[119,120],[120,121],[121,122],[122,126],[126,130],[130,131],[131,134],[134,137],[137,138],[138,143],[143,147],[147,154],[154,156],[156,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,1,2,2,3,4,4,4,5,6,6,6,7,8,8,8,9,10,10,10,10,11,12,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,32,33,34,34,35,36,37,38,39,40,41,42,43,43,44,44,44,45,46,47,47,47,47,47,47,47,48,48,48,48,48,49,50,50,51,52,52,53,54,55,56,57,58],"decoded":"전화번호:\n위치: 뉴질랜드 > 남섬 > 말버러 > 블레넘\n가격대 (1박): 117,886 원 - 140,244 원\n4.5성급 — Lugano Motor Lodge 4.5*\n- 예약 옵션:\n- 트립어드바이저는 호텔스닷컴, Expedia, Agoda, Asia Web Direct 및 Boo","decoded_with_specials":"<|begin_of_text|>전화번호:\n위치: 뉴질랜드 > 남섬 > 말버러 > 블레넘\n가격대 (1박): 117,886 원 - 140,244 원\n4.5성급 — Lugano Motor Lodge 4.5*\n- 예약 옵션:\n- 트립어드바이저는 호텔스닷컴, Expedia, Agoda, Asia Web Direct 및 Boo"} +{"kind":"sample","source":"fixtures/lang/rus_Cyrl.txt[:160]","text":"Покупая продукты в супермаркетах, сегодня уже мало кто верит в их качество. Как не сделать свое меню экстремальным и на что следует обращать внимание – читайте!","ids":[128000,17279,15088,49907,36497,125374,5927,107101,7753,114863,111885,100676,11,115936,91877,112974,107955,64362,9542,5927,101622,7820,50223,52744,13,107234,19175,108297,118164,69844,12182,81757,6735,50681,114259,7740,13373,48489,105895,113754,18482,113842,1389,111422,106606,0],"ids_no_specials":[17279,15088,49907,36497,125374,5927,107101,7753,114863,111885,100676,11,115936,91877,112974,107955,64362,9542,5927,101622,7820,50223,52744,13,107234,19175,108297,118164,69844,12182,81757,6735,50681,114259,7740,13373,48489,105895,113754,18482,113842,1389,111422,106606,0],"tokens":["<|begin_of_text|>","ÐŁ","ок","Ñĥп","аÑı","ĠпÑĢодÑĥкÑĤÑĭ","Ġв","ĠÑģÑĥп","еÑĢ","маÑĢ","кеÑĤ","аÑħ",",","ĠÑģегоднÑı","ĠÑĥже","Ġмало","ĠкÑĤо","ĠвеÑĢ","иÑĤ","Ġв","ĠиÑħ","Ġк","аÑĩ","еÑģÑĤво",".","ĠÐļак","Ġне","ĠÑģделаÑĤÑĮ","ĠÑģвое","Ġмен","Ñİ","ĠÑįк","ÑģÑĤ","ÑĢем","алÑĮнÑĭм","Ġи","Ġна","ĠÑĩÑĤо","ĠÑģледÑĥеÑĤ","ĠобÑĢаÑī","аÑĤÑĮ","Ġвнимание","ĠâĢĵ","ĠÑĩиÑĤ","айÑĤе","!"],"offsets":[[0,0],[0,1],[1,3],[3,5],[5,7],[7,16],[16,18],[18,22],[22,24],[24,27],[27,30],[30,32],[32,33],[33,41],[41,45],[45,50],[50,54],[54,58],[58,60],[60,62],[62,65],[65,67],[67,69],[69,74],[74,75],[75,79],[79,82],[82,90],[90,95],[95,99],[99,100],[100,103],[103,105],[105,108],[108,114],[114,116],[116,119],[119,123],[123,131],[131,137],[137,140],[140,149],[149,151],[151,155],[155,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,1,2,3,3,3,3,3,4,5,6,7,8,9,9,10,11,12,12,12,13,14,15,16,17,18,18,19,19,19,19,20,21,22,23,24,24,25,26,27,27,28],"decoded":"Покупая продукты в супермаркетах, сегодня уже мало кто верит в их качество. Как не сделать свое меню экстремальным и на что следует обращать внимание – читайте!","decoded_with_specials":"<|begin_of_text|>Покупая продукты в супермаркетах, сегодня уже мало кто верит в их качество. Как не сделать свое меню экстремальным и на что следует обращать внимание – читайте!"} +{"kind":"sample","source":"fixtures/lang/tam_Taml.txt[:160]","text":"‘ஹலோ கலெக்டர் சாரா… சரக்கு வேணும்… கடை எப்ப திறப்பீங்க?’\nஒரு படத்தில் ஒயின்ஷாப்புக்குள் திருடப் போன வடிவேலு நன்றாக சரக்கடித்து விட்டு, ஒயின்ஷாப் ஓனருக்கு போன் ப","ids":[128000,14336,20627,117,20627,110,32601,233,71697,243,20627,110,32601,228,20627,243,64500,253,20627,108,47454,71697,248,20627,122,20627,108,20627,122,1981,71697,248,20627,108,20627,243,64500,243,84298,71697,113,32601,229,20627,96,84298,20627,106,47454,1981,71697,243,20627,253,32601,230,71697,236,20627,103,64500,103,71697,97,100112,109,20627,103,64500,103,32601,222,20627,247,64500,243,45258,198,20627,240,20627,108,84298,71697,103,20627,253,20627,97,64500,97,100112,110,47454,71697,240,20627,107,100112,102,64500,115,20627,122,20627,103,64500,103,84298,20627,243,64500,243,84298,20627,111,47454,71697,97,100112,108,84298,20627,253,20627,103,47454,71697,103,32601,233,20627,102,71697,113,20627,253,100112,113,32601,229,20627,110,84298,71697,101,20627,102,64500,109,20627,122,20627,243,71697,248,20627,108,20627,243,64500,243,20627,253,100112,97,64500,97,84298,71697,113,100112,253,64500,253,84298,11,71697,240,20627,107,100112,102,64500,115,20627,122,20627,103,47454,71697,241,20627,102,20627,108,84298,20627,243,64500,243,84298,71697,103,32601,233,20627,102,47454,71697,103],"ids_no_specials":[14336,20627,117,20627,110,32601,233,71697,243,20627,110,32601,228,20627,243,64500,253,20627,108,47454,71697,248,20627,122,20627,108,20627,122,1981,71697,248,20627,108,20627,243,64500,243,84298,71697,113,32601,229,20627,96,84298,20627,106,47454,1981,71697,243,20627,253,32601,230,71697,236,20627,103,64500,103,71697,97,100112,109,20627,103,64500,103,32601,222,20627,247,64500,243,45258,198,20627,240,20627,108,84298,71697,103,20627,253,20627,97,64500,97,100112,110,47454,71697,240,20627,107,100112,102,64500,115,20627,122,20627,103,64500,103,84298,20627,243,64500,243,84298,20627,111,47454,71697,97,100112,108,84298,20627,253,20627,103,47454,71697,103,32601,233,20627,102,71697,113,20627,253,100112,113,32601,229,20627,110,84298,71697,101,20627,102,64500,109,20627,122,20627,243,71697,248,20627,108,20627,243,64500,243,20627,253,100112,97,64500,97,84298,71697,113,100112,253,64500,253,84298,11,71697,240,20627,107,100112,102,64500,115,20627,122,20627,103,47454,71697,241,20627,102,20627,108,84298,20627,243,64500,243,84298,71697,103,32601,233,20627,102,47454,71697,103],"tokens":["<|begin_of_text|>","âĢĺ","à®","¹","à®","²","à¯","ĭ","Ġà®","ķ","à®","²","à¯","Ĩ","à®","ķ","à¯įà®","Ł","à®","°","à¯į","Ġà®","ļ","à®","¾","à®","°","à®","¾","âĢ¦","Ġà®","ļ","à®","°","à®","ķ","à¯įà®","ķ","à¯ģ","Ġà®","µ","à¯","ĩ","à®","£","à¯ģ","à®","®","à¯į","âĢ¦","Ġà®","ķ","à®","Ł","à¯","Ī","Ġà®","İ","à®","ª","à¯įà®","ª","Ġà®","¤","ிà®","±","à®","ª","à¯įà®","ª","à¯","Ģ","à®","Ļ","à¯įà®","ķ","?âĢĻ","Ċ","à®","Ĵ","à®","°","à¯ģ","Ġà®","ª","à®","Ł","à®","¤","à¯įà®","¤","ிà®","²","à¯į","Ġà®","Ĵ","à®","¯","ிà®","©","à¯įà®","·","à®","¾","à®","ª","à¯įà®","ª","à¯ģ","à®","ķ","à¯įà®","ķ","à¯ģ","à®","³","à¯į","Ġà®","¤","ிà®","°","à¯ģ","à®","Ł","à®","ª","à¯į","Ġà®","ª","à¯","ĭ","à®","©","Ġà®","µ","à®","Ł","ிà®","µ","à¯","ĩ","à®","²","à¯ģ","Ġà®","¨","à®","©","à¯įà®","±","à®","¾","à®","ķ","Ġà®","ļ","à®","°","à®","ķ","à¯įà®","ķ","à®","Ł","ிà®","¤","à¯įà®","¤","à¯ģ","Ġà®","µ","ிà®","Ł","à¯įà®","Ł","à¯ģ",",","Ġà®","Ĵ","à®","¯","ிà®","©","à¯įà®","·","à®","¾","à®","ª","à¯į","Ġà®","ĵ","à®","©","à®","°","à¯ģ","à®","ķ","à¯įà®","ķ","à¯ģ","Ġà®","ª","à¯","ĭ","à®","©","à¯į","Ġà®","ª"],"offsets":[[0,0],[0,1],[1,2],[1,2],[2,3],[2,3],[3,4],[3,4],[4,6],[5,6],[6,7],[6,7],[7,8],[7,8],[8,9],[8,9],[9,11],[10,11],[11,12],[11,12],[12,13],[13,15],[14,15],[15,16],[15,16],[16,17],[16,17],[17,18],[17,18],[18,19],[19,21],[20,21],[21,22],[21,22],[22,23],[22,23],[23,25],[24,25],[25,26],[26,28],[27,28],[28,29],[28,29],[29,30],[29,30],[30,31],[31,32],[31,32],[32,33],[33,34],[34,36],[35,36],[36,37],[36,37],[37,38],[37,38],[38,40],[39,40],[40,41],[40,41],[41,43],[42,43],[43,45],[44,45],[45,47],[46,47],[47,48],[47,48],[48,50],[49,50],[50,51],[50,51],[51,52],[51,52],[52,54],[53,54],[54,56],[56,57],[57,58],[57,58],[58,59],[58,59],[59,60],[60,62],[61,62],[62,63],[62,63],[63,64],[63,64],[64,66],[65,66],[66,68],[67,68],[68,69],[69,71],[70,71],[71,72],[71,72],[72,74],[73,74],[74,76],[75,76],[76,77],[76,77],[77,78],[77,78],[78,80],[79,80],[80,81],[81,82],[81,82],[82,84],[83,84],[84,85],[85,86],[85,86],[86,87],[87,89],[88,89],[89,91],[90,91],[91,92],[92,93],[92,93],[93,94],[93,94],[94,95],[95,97],[96,97],[97,98],[97,98],[98,99],[98,99],[99,101],[100,101],[101,102],[101,102],[102,104],[103,104],[104,105],[104,105],[105,106],[105,106],[106,107],[107,109],[108,109],[109,110],[109,110],[110,112],[111,112],[112,113],[112,113],[113,114],[113,114],[114,116],[115,116],[116,117],[116,117],[117,118],[117,118],[118,120],[119,120],[120,121],[120,121],[121,123],[122,123],[123,125],[124,125],[125,126],[126,128],[127,128],[128,130],[129,130],[130,132],[131,132],[132,133],[133,134],[134,136],[135,136],[136,137],[136,137],[137,139],[138,139],[139,141],[140,141],[141,142],[141,142],[142,143],[142,143],[143,144],[144,146],[145,146],[146,147],[146,147],[147,148],[147,148],[148,149],[149,150],[149,150],[150,152],[151,152],[152,153],[153,155],[154,155],[155,156],[155,156],[156,157],[156,157],[157,158],[158,160],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,6,6,7,7,7,7,8,8,8,9,9,9,9,9,9,10,10,11,12,12,13,13,13,13,14,14,14,15,15,16,16,16,16,17,17,18,18,18,18,19,19,20,20,21,21,21,21,22,22,23,23,23,23,24,24,25,25,26,26,26,26,27,28,28,28,28,28,28,29,29,30,30,31,32,32,32,32,33,33,34,34,35,35,35,35,36,36,37,37,37,38,38,39,39,39,40,41,41,42,42,43,43,43,43,43,44,45,45,46,46,46,46,47,47,47,47,48,48,49,49,49,49,50,51,51,51,51,52,52,53,53,53,53,54,54,54,54,54,54,55,55,55,55,56,56,57,57,58,59,59,60,60,61,61,62,62,63,63,63,63,64,64,65,65,66,66,66,66,67,68,68,68,68,68,68,69,69,69,70,70,71,72,72,73,73,73,73,74,75,75],"decoded":"‘ஹலோ கலெக்டர் சாரா… சரக்கு வேணும்… கடை எப்ப திறப்பீங்க?’\nஒரு படத்தில் ஒயின்ஷாப்புக்குள் திருடப் போன வடிவேலு நன்றாக சரக்கடித்து விட்டு, ஒயின்ஷாப் ஓனருக்கு போன் ப","decoded_with_specials":"<|begin_of_text|>‘ஹலோ கலெக்டர் சாரா… சரக்கு வேணும்… கடை எப்ப திறப்பீங்க?’\nஒரு படத்தில் ஒயின்ஷாப்புக்குள் திருடப் போன வடிவேலு நன்றாக சரக்கடித்து விட்டு, ஒயின்ஷாப் ஓனருக்கு போன் ப"} +{"kind":"sample","source":"fixtures/lang/tha_Thai.txt[:160]","text":"นับเจดีย์ดูนะครับว่าครบยี่สิบองค์รึเปล่า คำว่าซาว ทางเหนือแปลว่่ายี่สิบครับ ผมนับดูแล้วก็ครบนะ ลองดู เสร็จแล้วก็เข้าเยี่ยมชมพิพธภัณฑ์ เดินเข้าไปด้านในหน่อยก็จะม","ids":[128000,20795,84646,102071,38133,99030,108013,42686,105323,100749,84646,38313,65841,120887,35609,48271,36748,104422,107027,116297,100470,108550,65841,126109,38313,65841,100665,100707,123087,112688,84681,72409,104191,38313,19138,19138,100353,48271,36748,104422,100749,84646,102377,107108,84646,38133,123395,100974,26265,84382,120887,105323,102386,100297,38133,42686,95582,102897,107262,100418,100974,26265,84382,101041,59095,104607,48271,108424,108597,60984,115537,100623,100658,111007,56870,105401,86145,101041,59095,101279,38133,101290,100427,100832,108380,26265,84382,115591],"ids_no_specials":[20795,84646,102071,38133,99030,108013,42686,105323,100749,84646,38313,65841,120887,35609,48271,36748,104422,107027,116297,100470,108550,65841,126109,38313,65841,100665,100707,123087,112688,84681,72409,104191,38313,19138,19138,100353,48271,36748,104422,100749,84646,102377,107108,84646,38133,123395,100974,26265,84382,120887,105323,102386,100297,38133,42686,95582,102897,107262,100418,100974,26265,84382,101041,59095,104607,48271,108424,108597,60984,115537,100623,100658,111007,56870,105401,86145,101041,59095,101279,38133,101290,100427,100832,108380,26265,84382,115591],"tokens":["<|begin_of_text|>","à¸Ļ","ัà¸ļ","à¹Ģà¸Ī","à¸Ķ","ีย","à¹Įà¸Ķ","ู","à¸Ļะ","à¸Ħร","ัà¸ļ","ว","à¹Īา","à¸Ħรà¸ļ","ย","ีà¹Ī","ส","ิà¸ļ","à¸Ńà¸ĩà¸Ħ","à¹Įร","ึ","à¹Ģà¸Ľà¸¥","à¹Īา","Ġà¸Ħำ","ว","à¹Īา","à¸ĭ","าว","Ġà¸Ĺาà¸ĩ","à¹Ģหà¸Ļ","ืà¸Ń","à¹ģ","à¸Ľà¸¥","ว","à¹Ī","à¹Ī","าย","ีà¹Ī","ส","ิà¸ļ","à¸Ħร","ัà¸ļ","Ġà¸ľ","มà¸Ļ","ัà¸ļ","à¸Ķ","ูà¹ģล","à¹īว","à¸ģ","à¹ĩ","à¸Ħรà¸ļ","à¸Ļะ","Ġล","à¸Ńà¸ĩ","à¸Ķ","ู","Ġà¹Ģ","สร","à¹ĩà¸Ī","à¹ģล","à¹īว","à¸ģ","à¹ĩ","à¹Ģà¸Ĥ","à¹īา","à¹Ģย","ีà¹Ī","ยม","à¸Ĭม","à¸ŀ","ิà¸ŀ","à¸ĺ","à¸ł","ัà¸ĵà¸ij","à¹Į","Ġà¹Ģà¸Ķ","ิà¸Ļ","à¹Ģà¸Ĥ","à¹īา","à¹Ħà¸Ľ","à¸Ķ","à¹īาà¸Ļ","à¹ĥà¸Ļ","หà¸Ļ","à¹Īà¸Ńย","à¸ģ","à¹ĩ","à¸Īะม"],"offsets":[[0,0],[0,1],[1,3],[3,5],[5,6],[6,8],[8,10],[10,11],[11,13],[13,15],[15,17],[17,18],[18,20],[20,23],[23,24],[24,26],[26,27],[27,29],[29,32],[32,34],[34,35],[35,38],[38,40],[40,43],[43,44],[44,46],[46,47],[47,49],[49,53],[53,56],[56,58],[58,59],[59,61],[61,62],[62,63],[63,64],[64,66],[66,68],[68,69],[69,71],[71,73],[73,75],[75,77],[77,79],[79,81],[81,82],[82,85],[85,87],[87,88],[88,89],[89,92],[92,94],[94,96],[96,98],[98,99],[99,100],[100,102],[102,104],[104,106],[106,108],[108,110],[110,111],[111,112],[112,114],[114,116],[116,118],[118,120],[120,122],[122,124],[124,125],[125,127],[127,128],[128,129],[129,132],[132,133],[133,136],[136,138],[138,140],[140,142],[142,144],[144,145],[145,148],[148,150],[150,152],[152,155],[155,156],[156,157],[157,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,1,1,1,2,3,4,4,4,5,5,6,6,6,7,8,9,9,10,11,11,12,13,13,14,14,14,15,15,16,16,16,16,17,17,18,19,20,21,21,22,23,23,24,24,25,26,26,27,27,27,28,28,28,29,30,30,31,31,32,32,33,33,34,34,35,36,36,36,37,37,37,38,39,40,41,41,42,42,42,43,43,43,44,44,45,45],"decoded":"นับเจดีย์ดูนะครับว่าครบยี่สิบองค์รึเปล่า คำว่าซาว ทางเหนือแปลว่่ายี่สิบครับ ผมนับดูแล้วก็ครบนะ ลองดู เสร็จแล้วก็เข้าเยี่ยมชมพิพธภัณฑ์ เดินเข้าไปด้านในหน่อยก็จะม","decoded_with_specials":"<|begin_of_text|>นับเจดีย์ดูนะครับว่าครบยี่สิบองค์รึเปล่า คำว่าซาว ทางเหนือแปลว่่ายี่สิบครับ ผมนับดูแล้วก็ครบนะ ลองดู เสร็จแล้วก็เข้าเยี่ยมชมพิพธภัณฑ์ เดินเข้าไปด้านในหน่อยก็จะม"} +{"kind":"sample","source":"fixtures/modalities/added_normalized_dense.txt[:160]","text":"FLIBBERJAST CRUNGLEDORF FLIBBERJAST SNORLAXIAN fast BLORPTRONIC ZORPTASTIC split WIDGETRON split stage SNORLAXIAN BLORPTRONIC BLORPTRONIC \n QUIBBLENAUT CRUNGLED","ids":[128000,6254,3336,9745,41,6483,12904,61102,13953,878,37,13062,3336,9745,41,6483,18407,878,43,3027,22774,5043,15195,878,48705,715,1341,1901,878,2898,6483,1341,6859,468,38735,42814,6859,6566,18407,878,43,3027,22774,15195,878,48705,715,1341,15195,878,48705,715,1341,720,89630,10306,877,7476,1406,12904,61102,13953],"ids_no_specials":[6254,3336,9745,41,6483,12904,61102,13953,878,37,13062,3336,9745,41,6483,18407,878,43,3027,22774,5043,15195,878,48705,715,1341,1901,878,2898,6483,1341,6859,468,38735,42814,6859,6566,18407,878,43,3027,22774,15195,878,48705,715,1341,15195,878,48705,715,1341,720,89630,10306,877,7476,1406,12904,61102,13953],"tokens":["<|begin_of_text|>","FL","IB","BER","J","AST","ĠCR","UNG","LED","OR","F","ĠFL","IB","BER","J","AST","ĠSN","OR","L","AX","IAN","Ġfast","ĠBL","OR","PTR","ON","IC","ĠZ","OR","PT","AST","IC","Ġsplit","ĠW","IDGET","RON","Ġsplit","Ġstage","ĠSN","OR","L","AX","IAN","ĠBL","OR","PTR","ON","IC","ĠBL","OR","PTR","ON","IC","ĠĊ","ĠQUI","BB","LE","NA","UT","ĠCR","UNG","LED"],"offsets":[[0,0],[0,2],[2,4],[4,7],[7,8],[8,11],[11,14],[14,17],[17,20],[20,22],[22,23],[23,26],[26,28],[28,31],[31,32],[32,35],[35,38],[38,40],[40,41],[41,43],[43,46],[46,51],[51,54],[54,56],[56,59],[59,61],[61,63],[63,65],[65,67],[67,69],[69,72],[72,74],[74,80],[80,82],[82,87],[87,90],[90,96],[96,102],[102,105],[105,107],[107,108],[108,110],[110,113],[113,116],[116,118],[118,121],[121,123],[123,125],[125,128],[128,130],[130,133],[133,135],[135,137],[137,139],[139,143],[143,145],[145,147],[147,149],[149,151],[151,154],[154,157],[157,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,1,1,1,1,1,2,2,2,2,2,3,3,3,3,3,4,5,5,5,5,5,6,6,6,6,6,7,8,8,8,9,10,11,11,11,11,11,12,12,12,12,12,13,13,13,13,13,14,15,15,15,15,15,16,16,16],"decoded":"FLIBBERJAST CRUNGLEDORF FLIBBERJAST SNORLAXIAN fast BLORPTRONIC ZORPTASTIC split WIDGETRON split stage SNORLAXIAN BLORPTRONIC BLORPTRONIC \n QUIBBLENAUT CRUNGLED","decoded_with_specials":"<|begin_of_text|>FLIBBERJAST CRUNGLEDORF FLIBBERJAST SNORLAXIAN fast BLORPTRONIC ZORPTASTIC split WIDGETRON split stage SNORLAXIAN BLORPTRONIC BLORPTRONIC \n QUIBBLENAUT CRUNGLED"} +{"kind":"sample","source":"fixtures/modalities/added_normalized_sparse.txt[:160]","text":"ZORPTASTIC for we for decode CRUNGLEDORF here through ZORPTASTIC and the we WIDGETRON model \n normalize bytes and back flows text CRUNGLEDORF WUZZLEFANG chunk b","ids":[128000,57,878,2898,6483,1341,369,584,369,17322,12904,61102,13953,878,37,1618,1555,1901,878,2898,6483,1341,323,279,584,468,38735,42814,1646,720,22436,5943,323,1203,28555,1495,12904,61102,13953,878,37,468,52,34636,877,37,5330,12143,293],"ids_no_specials":[57,878,2898,6483,1341,369,584,369,17322,12904,61102,13953,878,37,1618,1555,1901,878,2898,6483,1341,323,279,584,468,38735,42814,1646,720,22436,5943,323,1203,28555,1495,12904,61102,13953,878,37,468,52,34636,877,37,5330,12143,293],"tokens":["<|begin_of_text|>","Z","OR","PT","AST","IC","Ġfor","Ġwe","Ġfor","Ġdecode","ĠCR","UNG","LED","OR","F","Ġhere","Ġthrough","ĠZ","OR","PT","AST","IC","Ġand","Ġthe","Ġwe","ĠW","IDGET","RON","Ġmodel","ĠĊ","Ġnormalize","Ġbytes","Ġand","Ġback","Ġflows","Ġtext","ĠCR","UNG","LED","OR","F","ĠW","U","ZZ","LE","F","ANG","Ġchunk","Ġb"],"offsets":[[0,0],[0,1],[1,3],[3,5],[5,8],[8,10],[10,14],[14,17],[17,21],[21,28],[28,31],[31,34],[34,37],[37,39],[39,40],[40,45],[45,53],[53,55],[55,57],[57,59],[59,62],[62,64],[64,68],[68,72],[72,75],[75,77],[77,82],[82,85],[85,91],[91,93],[93,103],[103,109],[109,113],[113,118],[118,124],[124,129],[129,132],[132,135],[135,138],[138,140],[140,141],[141,143],[143,144],[144,146],[146,148],[148,149],[149,152],[152,158],[158,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,0,0,0,1,2,3,4,5,5,5,5,5,6,7,8,8,8,8,8,9,10,11,12,12,12,13,14,15,16,17,18,19,20,21,21,21,21,21,22,22,22,22,22,22,23,24],"decoded":"ZORPTASTIC for we for decode CRUNGLEDORF here through ZORPTASTIC and the we WIDGETRON model \n normalize bytes and back flows text CRUNGLEDORF WUZZLEFANG chunk b","decoded_with_specials":"<|begin_of_text|>ZORPTASTIC for we for decode CRUNGLEDORF here through ZORPTASTIC and the we WIDGETRON model \n normalize bytes and back flows text CRUNGLEDORF WUZZLEFANG chunk b"} +{"kind":"sample","source":"fixtures/modalities/added_special_dense.txt[:160]","text":"fast <|xs2|> chunk <|xs1|> <|xs4|> <|xs3|> <|xs4|> modality language <|xs0|> reads <|xs2|> and and \n <|xs1|> <|xs0|> <|xs1|> <|xs4|> <|xs4|> <|xs1|> back and re","ids":[128000,9533,83739,19072,17,91,29,12143,83739,19072,16,91,29,83739,19072,19,91,29,83739,19072,18,91,29,83739,19072,19,91,29,1491,2786,4221,83739,19072,15,91,29,16181,83739,19072,17,91,29,323,323,720,83739,19072,16,91,29,83739,19072,15,91,29,83739,19072,16,91,29,83739,19072,19,91,29,83739,19072,19,91,29,83739,19072,16,91,29,1203,323,312],"ids_no_specials":[9533,83739,19072,17,91,29,12143,83739,19072,16,91,29,83739,19072,19,91,29,83739,19072,18,91,29,83739,19072,19,91,29,1491,2786,4221,83739,19072,15,91,29,16181,83739,19072,17,91,29,323,323,720,83739,19072,16,91,29,83739,19072,15,91,29,83739,19072,16,91,29,83739,19072,19,91,29,83739,19072,19,91,29,83739,19072,16,91,29,1203,323,312],"tokens":["<|begin_of_text|>","fast","Ġ<|","xs","2","|",">","Ġchunk","Ġ<|","xs","1","|",">","Ġ<|","xs","4","|",">","Ġ<|","xs","3","|",">","Ġ<|","xs","4","|",">","Ġmod","ality","Ġlanguage","Ġ<|","xs","0","|",">","Ġreads","Ġ<|","xs","2","|",">","Ġand","Ġand","ĠĊ","Ġ<|","xs","1","|",">","Ġ<|","xs","0","|",">","Ġ<|","xs","1","|",">","Ġ<|","xs","4","|",">","Ġ<|","xs","4","|",">","Ġ<|","xs","1","|",">","Ġback","Ġand","Ġre"],"offsets":[[0,0],[0,4],[4,7],[7,9],[9,10],[10,11],[11,12],[12,18],[18,21],[21,23],[23,24],[24,25],[25,26],[26,29],[29,31],[31,32],[32,33],[33,34],[34,37],[37,39],[39,40],[40,41],[41,42],[42,45],[45,47],[47,48],[48,49],[49,50],[50,54],[54,59],[59,68],[68,71],[71,73],[73,74],[74,75],[75,76],[76,82],[82,85],[85,87],[87,88],[88,89],[89,90],[90,94],[94,98],[98,100],[100,103],[103,105],[105,106],[106,107],[107,108],[108,111],[111,113],[113,114],[114,115],[115,116],[116,119],[119,121],[121,122],[122,123],[123,124],[124,127],[127,129],[129,130],[130,131],[131,132],[132,135],[135,137],[137,138],[138,139],[139,140],[140,143],[143,145],[145,146],[146,147],[147,148],[148,153],[153,157],[157,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,1,2,3,4,4,5,6,7,8,9,9,10,11,12,13,13,14,15,16,17,17,18,19,20,21,21,22,22,23,24,25,26,27,27,28,29,30,31,32,32,33,34,35,36,37,38,39,39,40,41,42,43,43,44,45,46,47,47,48,49,50,51,51,52,53,54,55,55,56,57,58,59,59,60,61,62],"decoded":"fast <|xs2|> chunk <|xs1|> <|xs4|> <|xs3|> <|xs4|> modality language <|xs0|> reads <|xs2|> and and \n <|xs1|> <|xs0|> <|xs1|> <|xs4|> <|xs4|> <|xs1|> back and re","decoded_with_specials":"<|begin_of_text|>fast <|xs2|> chunk <|xs1|> <|xs4|> <|xs3|> <|xs4|> modality language <|xs0|> reads <|xs2|> and and \n <|xs1|> <|xs0|> <|xs1|> <|xs4|> <|xs4|> <|xs1|> back and re"} +{"kind":"sample","source":"fixtures/modalities/added_special_sparse.txt[:160]","text":"<|xs0|> every for encode <|xs0|> bytes the normalize decode text model <|xs4|> <|xs3|> and \n and <|xs3|> here <|xs1|> again model here text <|xs2|> <|xs2|> lang","ids":[128000,27,91,19072,15,91,29,1475,369,16559,83739,19072,15,91,29,5943,279,22436,17322,1495,1646,83739,19072,19,91,29,83739,19072,18,91,29,323,720,323,83739,19072,18,91,29,1618,83739,19072,16,91,29,1578,1646,1618,1495,83739,19072,17,91,29,83739,19072,17,91,29,8859],"ids_no_specials":[27,91,19072,15,91,29,1475,369,16559,83739,19072,15,91,29,5943,279,22436,17322,1495,1646,83739,19072,19,91,29,83739,19072,18,91,29,323,720,323,83739,19072,18,91,29,1618,83739,19072,16,91,29,1578,1646,1618,1495,83739,19072,17,91,29,83739,19072,17,91,29,8859],"tokens":["<|begin_of_text|>","<","|","xs","0","|",">","Ġevery","Ġfor","Ġencode","Ġ<|","xs","0","|",">","Ġbytes","Ġthe","Ġnormalize","Ġdecode","Ġtext","Ġmodel","Ġ<|","xs","4","|",">","Ġ<|","xs","3","|",">","Ġand","ĠĊ","Ġand","Ġ<|","xs","3","|",">","Ġhere","Ġ<|","xs","1","|",">","Ġagain","Ġmodel","Ġhere","Ġtext","Ġ<|","xs","2","|",">","Ġ<|","xs","2","|",">","Ġlang"],"offsets":[[0,0],[0,1],[1,2],[2,4],[4,5],[5,6],[6,7],[7,13],[13,17],[17,24],[24,27],[27,29],[29,30],[30,31],[31,32],[32,38],[38,42],[42,52],[52,59],[59,64],[64,70],[70,73],[73,75],[75,76],[76,77],[77,78],[78,81],[81,83],[83,84],[84,85],[85,86],[86,90],[90,92],[92,96],[96,99],[99,101],[101,102],[102,103],[103,104],[104,109],[109,112],[112,114],[114,115],[115,116],[116,117],[117,123],[123,129],[129,134],[134,139],[139,142],[142,144],[144,145],[145,146],[146,147],[147,150],[150,152],[152,153],[153,154],[154,155],[155,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,1,2,3,3,4,5,6,7,8,9,10,10,11,12,13,14,15,16,17,18,19,20,20,21,22,23,24,24,25,26,27,28,29,30,31,31,32,33,34,35,36,36,37,38,39,40,41,42,43,44,44,45,46,47,48,48,49],"decoded":"<|xs0|> every for encode <|xs0|> bytes the normalize decode text model <|xs4|> <|xs3|> and \n and <|xs3|> here <|xs1|> again model here text <|xs2|> <|xs2|> lang","decoded_with_specials":"<|begin_of_text|><|xs0|> every for encode <|xs0|> bytes the normalize decode text model <|xs4|> <|xs3|> and \n and <|xs3|> here <|xs1|> again model here text <|xs2|> <|xs2|> lang"} +{"kind":"sample","source":"fixtures/modalities/agentic-traces.txt[:160]","text":"Analyze this codebase and summarize what it is and how it works.\nI need to explore the codebase structure by reading the main entry points and configuration fil","ids":[128000,2127,56956,420,2082,3231,323,63179,1148,433,374,323,1268,433,4375,627,40,1205,311,13488,279,2082,3231,6070,555,5403,279,1925,4441,3585,323,6683,1488],"ids_no_specials":[2127,56956,420,2082,3231,323,63179,1148,433,374,323,1268,433,4375,627,40,1205,311,13488,279,2082,3231,6070,555,5403,279,1925,4441,3585,323,6683,1488],"tokens":["<|begin_of_text|>","An","alyze","Ġthis","Ġcode","base","Ġand","Ġsummarize","Ġwhat","Ġit","Ġis","Ġand","Ġhow","Ġit","Ġworks",".Ċ","I","Ġneed","Ġto","Ġexplore","Ġthe","Ġcode","base","Ġstructure","Ġby","Ġreading","Ġthe","Ġmain","Ġentry","Ġpoints","Ġand","Ġconfiguration","Ġfil"],"offsets":[[0,0],[0,2],[2,7],[7,12],[12,17],[17,21],[21,25],[25,35],[35,40],[40,43],[43,46],[46,50],[50,54],[54,57],[57,63],[63,65],[65,66],[66,71],[71,74],[74,82],[82,86],[86,91],[91,95],[95,105],[105,108],[108,116],[116,120],[120,125],[125,131],[131,138],[138,142],[142,156],[156,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,1,2,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,18,19,20,21,22,23,24,25,26,27,28],"decoded":"Analyze this codebase and summarize what it is and how it works.\nI need to explore the codebase structure by reading the main entry points and configuration fil","decoded_with_specials":"<|begin_of_text|>Analyze this codebase and summarize what it is and how it works.\nI need to explore the codebase structure by reading the main entry points and configuration fil"} +{"kind":"sample","source":"fixtures/modalities/agentic_swe.txt[:160]","text":"[system]\nYou are a helpful assistant that can interact with a computer to solve tasks.\n\n[user]\n\n/testbed\n\nI've uploaded a pytho","ids":[128000,58,9125,933,2675,527,264,11190,18328,430,649,16681,449,264,6500,311,11886,9256,382,54645,933,27,57983,11171,397,12986,2788,198,524,57983,11171,397,40,3077,23700,264,4611,339,78],"ids_no_specials":[58,9125,933,2675,527,264,11190,18328,430,649,16681,449,264,6500,311,11886,9256,382,54645,933,27,57983,11171,397,12986,2788,198,524,57983,11171,397,40,3077,23700,264,4611,339,78],"tokens":["<|begin_of_text|>","[","system","]Ċ","You","Ġare","Ġa","Ġhelpful","Ġassistant","Ġthat","Ġcan","Ġinteract","Ġwith","Ġa","Ġcomputer","Ġto","Ġsolve","Ġtasks",".ĊĊ","[user","]Ċ","<","uploaded","_files",">Ċ","/test","bed","Ċ","Ċ","I","'ve","Ġuploaded","Ġa","Ġpy","th","o"],"offsets":[[0,0],[0,1],[1,7],[7,9],[9,12],[12,16],[16,18],[18,26],[26,36],[36,41],[41,45],[45,54],[54,59],[59,61],[61,70],[70,73],[73,79],[79,85],[85,88],[88,93],[93,95],[95,96],[96,104],[104,110],[110,112],[112,117],[117,120],[120,121],[121,123],[123,131],[131,137],[137,139],[139,140],[140,143],[143,152],[152,154],[154,157],[157,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,19,20,21,22,22,23,24,25,26,27,28,29,30,31,32,32,32],"decoded":"[system]\nYou are a helpful assistant that can interact with a computer to solve tasks.\n\n[user]\n\n/testbed\n\nI've uploaded a pytho","decoded_with_specials":"<|begin_of_text|>[system]\nYou are a helpful assistant that can interact with a computer to solve tasks.\n\n[user]\n\n/testbed\n\nI've uploaded a pytho"} +{"kind":"sample","source":"fixtures/modalities/code_mixed.txt[:160]","text":"// django-f3f9601cff03b38942e70ce8042bdfdec6002449/django/__init__.py\nfrom django.utils.version import get_version\n\nVERSION = (6, 2, 0, \"alpha\", 0)\n\n__version__","ids":[128000,322,8426,2269,18,69,16415,16,58923,2839,65,20422,2983,68,2031,346,20417,17,65,3013,8332,5067,13719,24,3529,5970,80563,2381,19247,3368,198,1527,8426,8576,20049,1179,636,9625,271,18102,284,320,21,11,220,17,11,220,15,11,330,7288,498,220,15,696,565,4464,565],"ids_no_specials":[322,8426,2269,18,69,16415,16,58923,2839,65,20422,2983,68,2031,346,20417,17,65,3013,8332,5067,13719,24,3529,5970,80563,2381,19247,3368,198,1527,8426,8576,20049,1179,636,9625,271,18102,284,320,21,11,220,17,11,220,15,11,330,7288,498,220,15,696,565,4464,565],"tokens":["<|begin_of_text|>","//","Ġdjango","-f","3","f","960","1","cff","03","b","389","42","e","70","ce","804","2","b","df","dec","600","244","9","/d","jango","/__","init","__.","py","Ċ","from","Ġdjango",".utils",".version","Ġimport","Ġget","_version","ĊĊ","VERSION","Ġ=","Ġ(","6",",","Ġ","2",",","Ġ","0",",","Ġ\"","alpha","\",","Ġ","0",")ĊĊ","__","version","__"],"offsets":[[0,0],[0,2],[2,9],[9,11],[11,12],[12,13],[13,16],[16,17],[17,20],[20,22],[22,23],[23,26],[26,28],[28,29],[29,31],[31,33],[33,36],[36,37],[37,38],[38,40],[40,43],[43,46],[46,49],[49,50],[50,52],[52,57],[57,60],[60,64],[64,67],[67,69],[69,70],[70,74],[74,81],[81,87],[87,95],[95,102],[102,106],[106,114],[114,116],[116,123],[123,125],[125,127],[127,128],[128,129],[129,130],[130,131],[131,132],[132,133],[133,134],[134,135],[135,137],[137,142],[142,144],[144,145],[145,146],[146,149],[149,151],[151,158],[158,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,17,17,18,19,20,21,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54],"decoded":"// django-f3f9601cff03b38942e70ce8042bdfdec6002449/django/__init__.py\nfrom django.utils.version import get_version\n\nVERSION = (6, 2, 0, \"alpha\", 0)\n\n__version__","decoded_with_specials":"<|begin_of_text|>// django-f3f9601cff03b38942e70ce8042bdfdec6002449/django/__init__.py\nfrom django.utils.version import get_version\n\nVERSION = (6, 2, 0, \"alpha\", 0)\n\n__version__"} +{"kind":"sample","source":"fixtures/modalities/math_latex.txt[:160]","text":"Bayes and his Theorem\n\nMy earlier post on Bayesian probability seems to have generated quite a lot of readers, so this lunchtime I thought I’d add a little bit ","ids":[128000,23407,288,323,813,4194,791,13475,271,5159,6931,1772,389,99234,19463,5084,311,617,8066,5115,264,2763,315,13016,11,779,420,16163,1712,358,3463,358,7070,923,264,2697,2766,220],"ids_no_specials":[23407,288,323,813,4194,791,13475,271,5159,6931,1772,389,99234,19463,5084,311,617,8066,5115,264,2763,315,13016,11,779,420,16163,1712,358,3463,358,7070,923,264,2697,2766,220],"tokens":["<|begin_of_text|>","Bay","es","Ġand","Ġhis","Âł","The","orem","ĊĊ","My","Ġearlier","Ġpost","Ġon","ĠBayesian","Ġprobability","Ġseems","Ġto","Ġhave","Ġgenerated","Ġquite","Ġa","Ġlot","Ġof","Ġreaders",",","Ġso","Ġthis","Ġlunch","time","ĠI","Ġthought","ĠI","âĢĻd","Ġadd","Ġa","Ġlittle","Ġbit","Ġ"],"offsets":[[0,0],[0,3],[3,5],[5,9],[9,13],[13,14],[14,17],[17,21],[21,23],[23,25],[25,33],[33,38],[38,41],[41,50],[50,62],[62,68],[68,71],[71,76],[76,86],[86,92],[92,94],[94,98],[98,101],[101,109],[109,110],[110,113],[113,118],[118,124],[124,128],[128,130],[130,138],[138,140],[140,142],[142,146],[146,148],[148,155],[155,159],[159,160]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"word_ids":[null,0,0,1,2,3,3,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,23,24,25,26,27,28,29,30,31,32],"decoded":"Bayes and his Theorem\n\nMy earlier post on Bayesian probability seems to have generated quite a lot of readers, so this lunchtime I thought I’d add a little bit ","decoded_with_specials":"<|begin_of_text|>Bayes and his Theorem\n\nMy earlier post on Bayesian probability seems to have generated quite a lot of readers, so this lunchtime I thought I’d add a little bit "} +{"kind":"pair","text":"What is the capital of France?","pair":"Paris is the capital of France.","ids":[128000,3923,374,279,6864,315,9822,30,128000,60704,374,279,6864,315,9822,13],"tokens":["<|begin_of_text|>","What","Ġis","Ġthe","Ġcapital","Ġof","ĠFrance","?","<|begin_of_text|>","Paris","Ġis","Ġthe","Ġcapital","Ġof","ĠFrance","."],"type_ids":[0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1],"sequence_ids":[null,0,0,0,0,0,0,0,null,1,1,1,1,1,1,1],"special_tokens_mask":[1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0],"offsets":[[0,0],[0,4],[4,7],[7,11],[11,19],[19,22],[22,29],[29,30],[0,0],[0,5],[5,8],[8,12],[12,20],[20,23],[23,30],[30,31]]} +{"kind":"pair","text":"Question in English?","pair":"Ответ на русском языке, с цифрами 123.","ids":[128000,14924,304,6498,30,128000,60627,48074,13373,110950,106292,110424,53671,11,5524,39233,62809,112459,220,4513,13],"tokens":["<|begin_of_text|>","Question","Ġin","ĠEnglish","?","<|begin_of_text|>","ÐŀÑĤ","веÑĤ","Ġна","ĠÑĢÑĥÑģ","Ñģком","ĠÑıзÑĭ","ке",",","ĠÑģ","ĠÑĨ","иÑĦ","ÑĢами","Ġ","123","."],"type_ids":[0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],"sequence_ids":[null,0,0,0,0,null,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],"special_tokens_mask":[1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"offsets":[[0,0],[0,8],[8,11],[11,19],[19,20],[0,0],[0,2],[2,5],[5,8],[8,12],[12,16],[16,20],[20,22],[22,23],[23,25],[25,27],[27,29],[29,33],[33,34],[34,37],[37,38]]} +{"kind":"digest","file":"fixtures/lang/amh_Ethi.txt","cap_chars":200000,"text_sha256":"e353d32f45b449b76fb65ee48f1522086c02a6bffa607e73ed8c875c03c84908","n_tokens":367328,"ids_sha256":"3f0b8d79bcda7908310569cf85e109656029a142ca6a7da07a93b5b27a767215"} +{"kind":"digest","file":"fixtures/lang/arb_Arab.txt","cap_chars":200000,"text_sha256":"529587117db0a7abaa2c8f31ab05d7b2f77e84a11068b46e26b2b37774d836c7","n_tokens":77696,"ids_sha256":"07509671b72b29c3f87a43de9f026dfa2b8f7e39c01330ac317ae5038b07615c"} +{"kind":"digest","file":"fixtures/lang/ben_Beng.txt","cap_chars":200000,"text_sha256":"915afd7dbbea8256bdb6112b16424c83aa47dae63c5cca0ebadfc6c227d2b89f","n_tokens":241237,"ids_sha256":"cb2ffc958f402917dd9aa25360ebcecdf450fc9ac3cc582e4ec9a09f23eab81a"} +{"kind":"digest","file":"fixtures/lang/cmn_Hani.txt","cap_chars":200000,"text_sha256":"1273c0c093a00739be9efea7804b10a295f902359d11663306d097211934c6df","n_tokens":166663,"ids_sha256":"c17c85e2d50e1a2cf5b1a6d12cae9bd1fc545e3d70eeededf568536f32e08eed"} +{"kind":"digest","file":"fixtures/lang/ell_Grek.txt","cap_chars":200000,"text_sha256":"158c8d3e4933091099bf12a1e21f57c7b7ff3f5381d4b25901eb1fa7ca0926f4","n_tokens":80080,"ids_sha256":"4018c4556e4f5e32cd19fe779ef9716f7e463deb638b32725a949189f159b840"} +{"kind":"digest","file":"fixtures/lang/eng_Latn.txt","cap_chars":200000,"text_sha256":"d998cb3153e3a56c2ecdf49293062e8bc6ca43365b6c91cfed147d73b84c19fb","n_tokens":46030,"ids_sha256":"16dd6808ab3c649e1e9ed7a264dc03b28848f9f8de8edc96c77b6ff2949cdd65"} +{"kind":"digest","file":"fixtures/lang/heb_Hebr.txt","cap_chars":200000,"text_sha256":"386da8c0e0d416bffbe84b51baf895c67dbce6e0fd153be597933261794300fd","n_tokens":189179,"ids_sha256":"992cfc84bca48b795b3c21e4162c947da6a51c95ed00f697e4b6640d8518db12"} +{"kind":"digest","file":"fixtures/lang/hin_Deva.txt","cap_chars":200000,"text_sha256":"2d114148c81035b1b63c94c7e9e805933c215d8aa9500d916923aec0a124ddde","n_tokens":102917,"ids_sha256":"71f899a65da6eb568e77156fe163842a95ba93388194f831fa5a3533cea46adf"} +{"kind":"digest","file":"fixtures/lang/jpn_Jpan.txt","cap_chars":200000,"text_sha256":"2e55c036f500d7720d020be2db45c87a49c17d3b7e63b30bada9cef4cb6aa158","n_tokens":140357,"ids_sha256":"885b81feaf640f17e1c4cad7f1c8bc21629ef3f623ee256a921589da4f88c69c"} +{"kind":"digest","file":"fixtures/lang/kat_Geor.txt","cap_chars":200000,"text_sha256":"c69a1aa8fb80b41459c98c15c756eb8f39a9151468b1d5b3f9bbffe4d82eb87a","n_tokens":360364,"ids_sha256":"9f0f546eb89abb88779a597d43b93edf9a367ac533c3cef1cd1e0bd9b2ac6662"} +{"kind":"digest","file":"fixtures/lang/kor_Hang.txt","cap_chars":200000,"text_sha256":"68659781c288136c1caffcb10eec9130406ae8a7b003c1e327b7e3601eba041d","n_tokens":123897,"ids_sha256":"1aa81141c083288ad898c9c4c56d09bf29b95c374a64422daa5a79008c7f67c5"} +{"kind":"digest","file":"fixtures/lang/rus_Cyrl.txt","cap_chars":200000,"text_sha256":"5f967a82627afbd72c2f21c6e56183dfa5efc986ba34350c1cf9eea6c27a6132","n_tokens":64005,"ids_sha256":"fc68a14d6c3b6e60f592e170e342c4ca582fd955a9c6dcd8a507397aa647dc33"} +{"kind":"digest","file":"fixtures/lang/tam_Taml.txt","cap_chars":200000,"text_sha256":"bf09e1bca55aeab34162d5dc61afec7ac976b4e66eb5833ac5a037305911d165","n_tokens":265045,"ids_sha256":"556d871d8b49f8ff6f420979088fdce0b49f203f47978347fdbc86f6c58d0b13"} +{"kind":"digest","file":"fixtures/lang/tha_Thai.txt","cap_chars":200000,"text_sha256":"25209f4750b3d3f04440f98aeff50316a118cb982462a4763106e8747172b79c","n_tokens":94619,"ids_sha256":"6d3e4e6824b7eb7d56539730ac6dcf10904370dd122d1ec295d0da316b8c5e2f"} +{"kind":"digest","file":"fixtures/modalities/added_normalized_dense.txt","cap_chars":200000,"text_sha256":"9d49b3cacf40962c051a09f1350a0f5dc2fc210556889ed0f341cfe0cf0af4f9","n_tokens":32015,"ids_sha256":"0624a85cd233281c833f2a10a50c7cd911696a47ad45ef111f9f0ec5f2eb309f"} +{"kind":"digest","file":"fixtures/modalities/added_normalized_sparse.txt","cap_chars":200000,"text_sha256":"55383f52b02a5d9686f9e4347729e5c08e61b9cff77913bd94ceb65c2fdf7fe4","n_tokens":22919,"ids_sha256":"fbea017d3476a990af7c0a975491c2de3716758f3adb7391a0cd42bcda2687aa"} +{"kind":"digest","file":"fixtures/modalities/added_special_dense.txt","cap_chars":200000,"text_sha256":"472f90f6f2f5803626df5d7088f8ef12984de6e88fedf67723887511d35b64b2","n_tokens":43739,"ids_sha256":"21b767e4a938b42cf14eb9c66590a264f9984d70618f9eaa87340bb1d400bcae"} +{"kind":"digest","file":"fixtures/modalities/added_special_sparse.txt","cap_chars":200000,"text_sha256":"9c52602f7c8d009b11377743f95e5bfe9053694fc6b48c3dd6c1c6dc0681ab5a","n_tokens":26451,"ids_sha256":"f7082b1bf1085418238dc1939fa07d54b8aa99e587e849d545a16497aa837b60"} +{"kind":"digest","file":"fixtures/modalities/agentic-traces.txt","cap_chars":200000,"text_sha256":"ae9d63c1adc49f572d9af635ac1b0421461e4d1b92c5f35ea572980ff1e2480c","n_tokens":51447,"ids_sha256":"8ea6f3f159ae6d09b63853003e3aeb9323b369e03470e0e69e9e082c2b40068c"} +{"kind":"digest","file":"fixtures/modalities/agentic_swe.txt","cap_chars":200000,"text_sha256":"b2d31e91ff91bb6c9a3aec3832d25acbecf9b58d5c0674e364d3b287182e7253","n_tokens":59046,"ids_sha256":"3ade814211ce0bb28debc3afd65c9a7054336396348331bee724451305a4bdfd"} +{"kind":"digest","file":"fixtures/modalities/code_mixed.txt","cap_chars":200000,"text_sha256":"0fa7314727726db056433d36bc1bda021803be7f1d150cae83e362a3ff41821d","n_tokens":64525,"ids_sha256":"3ae31e63cdbab57c39249899591d965fe5ece5b9ddb0ad2cafdd075da20b7653"} +{"kind":"digest","file":"fixtures/modalities/math_latex.txt","cap_chars":200000,"text_sha256":"2d17be8704f1f7b4f2174a054c0b85d6d22039e00be8e4a487927d27f3852e90","n_tokens":52200,"ids_sha256":"67c5ed516afac53dd494dff0629832c523bc365e007eb11c745729956a215f16"} diff --git a/bindings/python/tests/golden/goldens/t5-base.jsonl b/bindings/python/tests/golden/goldens/t5-base.jsonl new file mode 100644 index 000000000..22abb37a2 --- /dev/null +++ b/bindings/python/tests/golden/goldens/t5-base.jsonl @@ -0,0 +1,64 @@ +{"kind":"meta","model":"t5-base","tokenizer_file":"t5-base.json","tokenizers_version":"0.23.1"} +{"kind":"sample","source":"edge:empty","text":"","ids":[1],"ids_no_specials":[],"tokens":[""],"offsets":[[0,0]],"type_ids":[0],"special_tokens_mask":[1],"word_ids":[null],"decoded":"","decoded_with_specials":""} +{"kind":"sample","source":"edge:spaces","text":" ","ids":[1],"ids_no_specials":[],"tokens":[""],"offsets":[[0,0]],"type_ids":[0],"special_tokens_mask":[1],"word_ids":[null],"decoded":"","decoded_with_specials":""} +{"kind":"sample","source":"edge:hello","text":"Hello world","ids":[8774,296,1],"ids_no_specials":[8774,296],"tokens":["▁Hello","▁world",""],"offsets":[[0,5],[6,11],[0,0]],"type_ids":[0,0,0],"special_tokens_mask":[0,0,1],"word_ids":[0,1,null],"decoded":"Hello world","decoded_with_specials":"Hello world"} +{"kind":"sample","source":"edge:punctuation","text":"Hello, world!! How's it going? (fine; thanks...)","ids":[8774,6,296,1603,571,31,7,34,352,58,41,13536,117,2049,11439,1],"ids_no_specials":[8774,6,296,1603,571,31,7,34,352,58,41,13536,117,2049,11439],"tokens":["▁Hello",",","▁world","!!","▁How","'","s","▁it","▁going","?","▁(","fine",";","▁thanks","...)",""],"offsets":[[0,5],[5,6],[7,12],[12,14],[15,18],[18,19],[19,20],[21,23],[24,29],[29,30],[31,32],[32,36],[36,37],[38,44],[44,48],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[0,0,1,1,2,2,2,3,4,4,5,5,5,6,6,null],"decoded":"Hello, world!! How's it going? (fine; thanks...)","decoded_with_specials":"Hello, world!! How's it going? (fine; thanks...)"} +{"kind":"sample","source":"edge:whitespace-mix","text":"line one\nline two\r\n\tindented\n trailing ","ids":[689,80,689,192,16,537,1054,5032,53,1],"ids_no_specials":[689,80,689,192,16,537,1054,5032,53],"tokens":["▁line","▁one","▁line","▁two","▁in","den","ted","▁trail","ing",""],"offsets":[[0,4],[5,8],[9,13],[14,17],[20,22],[22,25],[25,28],[31,36],[36,39],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,1],"word_ids":[0,1,2,3,4,4,4,5,5,null],"decoded":"line one line two indented trailing","decoded_with_specials":"line one line two indented trailing"} +{"kind":"sample","source":"edge:accents","text":"café naïve résumé — déjà vu","ids":[11949,3,29,9,2,162,1417,4078,154,3,318,4315,9056,1],"ids_no_specials":[11949,3,29,9,2,162,1417,4078,154,3,318,4315,9056],"tokens":["▁café","▁","n","a","ï","ve","▁ré","sum","é","▁","—","▁déjà","▁vu",""],"offsets":[[0,4],[5,6],[5,6],[6,7],[7,8],[8,10],[11,13],[13,16],[16,17],[18,19],[18,19],[20,24],[25,27],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[0,1,1,1,1,1,2,2,2,3,3,4,5,null],"decoded":"café nave résumé — déjà vu","decoded_with_specials":"café nave résumé — déjà vu"} +{"kind":"sample","source":"edge:accents-decomposed","text":"café naïve résumé","ids":[11949,3,29,9,2,162,1417,4078,154,1],"ids_no_specials":[11949,3,29,9,2,162,1417,4078,154],"tokens":["▁café","▁","n","a","ï","ve","▁ré","sum","é",""],"offsets":[[0,4],[6,7],[6,7],[7,8],[8,9],[10,12],[13,15],[16,19],[19,20],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,1],"word_ids":[0,1,1,1,1,1,2,2,2,null],"decoded":"café nave résumé","decoded_with_specials":"café nave résumé"} +{"kind":"sample","source":"edge:emoji","text":"🤗 emoji, families 👨‍👩‍👧‍👦, flags 🇫🇷 and skin tones 👍🏽","ids":[3,2,3,15,51,21892,6,1791,3,2,3,2,3,2,3,2,6,5692,7,3,2,11,1133,12,1496,3,2,1],"ids_no_specials":[3,2,3,15,51,21892,6,1791,3,2,3,2,3,2,3,2,6,5692,7,3,2,11,1133,12,1496,3,2],"tokens":["▁","🤗","▁","e","m","oji",",","▁families","▁","👨","▁","👩","▁","👧","▁","👦",",","▁flag","s","▁","🇫🇷","▁and","▁skin","▁to","nes","▁","👍🏽",""],"offsets":[[0,1],[0,1],[2,3],[2,3],[3,4],[4,7],[7,8],[9,17],[18,19],[18,19],[20,21],[20,21],[22,23],[22,23],[24,25],[24,25],[25,26],[27,31],[31,32],[33,34],[33,35],[36,39],[40,44],[45,47],[47,50],[51,52],[51,53],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[0,0,1,1,1,1,1,2,3,3,4,4,5,5,6,6,6,7,7,8,8,9,10,11,11,12,12,null],"decoded":" emoji, families , flags and skin tones ","decoded_with_specials":" emoji, families , flags and skin tones "} +{"kind":"sample","source":"edge:cjk","text":"漢字とひらがなとカタカナが混ざった文章です。","ids":[3,2,1],"ids_no_specials":[3,2],"tokens":["▁","漢字とひらがなとカタカナが混ざった文章です。",""],"offsets":[[0,1],[0,22],[0,0]],"type_ids":[0,0,0],"special_tokens_mask":[0,0,1],"word_ids":[0,0,null],"decoded":"","decoded_with_specials":""} +{"kind":"sample","source":"edge:korean","text":"한국어 텍스트 조각","ids":[3,2,3,2,3,2,1],"ids_no_specials":[3,2,3,2,3,2],"tokens":["▁","한국어","▁","텍스트","▁","조각",""],"offsets":[[0,1],[0,3],[4,5],[4,7],[8,9],[8,10],[0,0]],"type_ids":[0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,1],"word_ids":[0,0,1,1,2,2,null],"decoded":" ","decoded_with_specials":" "} +{"kind":"sample","source":"edge:rtl","text":"مرحبا بالعالم — שלום עולם","ids":[3,2,3,2,3,318,3,2,3,2,1],"ids_no_specials":[3,2,3,2,3,318,3,2,3,2],"tokens":["▁","مرحبا","▁","بالعالم","▁","—","▁","שלום","▁","עולם",""],"offsets":[[0,1],[0,5],[6,7],[6,13],[14,15],[14,15],[16,17],[16,20],[21,22],[21,25],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,1],"word_ids":[0,0,1,1,2,2,3,3,4,4,null],"decoded":" — ","decoded_with_specials":" — "} +{"kind":"sample","source":"edge:numbers","text":"1234567890, 3.14159, 1,000,000th","ids":[586,3710,4834,3940,2394,6,1877,2534,27904,6,209,23916,189,1],"ids_no_specials":[586,3710,4834,3940,2394,6,1877,2534,27904,6,209,23916,189],"tokens":["▁12","34","56","78","90",",","▁3.","14","159",",","▁1",",000,000","th",""],"offsets":[[0,2],[2,4],[4,6],[6,8],[8,10],[10,11],[12,14],[14,16],[16,19],[19,20],[21,22],[22,30],[30,32],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[0,0,0,0,0,0,1,1,1,1,2,2,2,null],"decoded":"1234567890, 3.14159, 1,000,000th","decoded_with_specials":"1234567890, 3.14159, 1,000,000th"} +{"kind":"sample","source":"edge:code","text":"def f(x):\n return x**2 # squared\nprint(f'{f(3)=}')","ids":[20,89,3,89,599,226,61,10,1205,3,226,19844,357,1713,2812,26,2281,599,89,31,2,89,17867,2423,2,31,61,1],"ids_no_specials":[20,89,3,89,599,226,61,10,1205,3,226,19844,357,1713,2812,26,2281,599,89,31,2,89,17867,2423,2,31,61],"tokens":["▁de","f","▁","f","(","x",")",":","▁return","▁","x","**","2","▁#","▁square","d","▁print","(","f","'","{","f","(3)","=","}","'",")",""],"offsets":[[0,2],[2,3],[4,5],[4,5],[5,6],[6,7],[7,8],[8,9],[14,20],[21,22],[21,22],[22,24],[24,25],[27,28],[29,35],[35,36],[37,42],[42,43],[43,44],[44,45],[45,46],[46,47],[47,50],[50,51],[51,52],[52,53],[53,54],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[0,0,1,1,1,1,1,1,2,3,3,3,3,4,5,5,6,6,6,6,6,6,6,6,6,6,6,null],"decoded":"def f(x): return x**2 # squared print(f'f(3)=')","decoded_with_specials":"def f(x): return x**2 # squared print(f'f(3)=')"} +{"kind":"sample","source":"edge:long-word","text":"Donaudampfschifffahrtsgesellschaftskapitänsmützenabzeichen","ids":[1008,402,26,9,8078,26453,8255,7,16852,10717,5230,6125,7,51,17022,35,9,115,16549,1],"ids_no_specials":[1008,402,26,9,8078,26453,8255,7,16852,10717,5230,6125,7,51,17022,35,9,115,16549],"tokens":["▁Don","au","d","a","mpf","schiff","fahrt","s","gesellschaft","ska","pit","än","s","m","ütz","en","a","b","zeichen",""],"offsets":[[0,3],[3,5],[5,6],[6,7],[7,10],[10,16],[16,21],[21,22],[22,34],[34,37],[37,40],[40,42],[42,43],[43,44],[44,47],[47,49],[49,50],[50,51],[51,58],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,null],"decoded":"Donaudampfschifffahrtsgesellschaftskapitänsmützenabzeichen","decoded_with_specials":"Donaudampfschifffahrtsgesellschaftskapitänsmützenabzeichen"} +{"kind":"sample","source":"edge:url-email","text":"https://example.com/a/b?q=1&r=2#frag user.name+tag@example.co.uk","ids":[4893,1303,994,9,9208,5,287,87,9,87,115,58,1824,2423,536,184,52,2423,357,4663,20791,1139,5,4350,1220,2408,1741,994,9,9208,5,509,5,1598,1],"ids_no_specials":[4893,1303,994,9,9208,5,287,87,9,87,115,58,1824,2423,536,184,52,2423,357,4663,20791,1139,5,4350,1220,2408,1741,994,9,9208,5,509,5,1598],"tokens":["▁https","://","ex","a","mple",".","com","/","a","/","b","?","q","=","1","&","r","=","2","#","frag","▁user",".","name","+","tag","@","ex","a","mple",".","co",".","uk",""],"offsets":[[0,5],[5,8],[8,10],[10,11],[11,15],[15,16],[16,19],[19,20],[20,21],[21,22],[22,23],[23,24],[24,25],[25,26],[26,27],[27,28],[28,29],[29,30],[30,31],[31,32],[32,36],[37,41],[41,42],[42,46],[46,47],[47,50],[50,51],[51,53],[53,54],[54,58],[58,59],[59,61],[61,62],[62,64],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,null],"decoded":"https://example.com/a/b?q=1&r=2#frag user.name+tag@example.co.uk","decoded_with_specials":"https://example.com/a/b?q=1&r=2#frag user.name+tag@example.co.uk"} +{"kind":"sample","source":"edge:unicode-spaces","text":"a b c　d","ids":[3,9,3,115,3,75,3,26,1],"ids_no_specials":[3,9,3,115,3,75,3,26],"tokens":["▁","a","▁","b","▁","c","▁","d",""],"offsets":[[0,1],[0,1],[2,3],[2,3],[4,5],[4,5],[6,7],[6,7],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,1],"word_ids":[0,0,1,1,2,2,3,3,null],"decoded":"a b c d","decoded_with_specials":"a b c d"} +{"kind":"sample","source":"edge:math","text":"∑ᵢ xᵢ² ≤ ∫₀^∞ e⁻ˣ dx ≈ 1","ids":[3,2,23,3,226,23,357,3,2,3,2,632,2,3,15,2,226,3,26,226,3,2,209,1],"ids_no_specials":[3,2,23,3,226,23,357,3,2,3,2,632,2,3,15,2,226,3,26,226,3,2,209],"tokens":["▁","∑","i","▁","x","i","2","▁","≤","▁","∫","0","^∞","▁","e","−","x","▁","d","x","▁","≈","▁1",""],"offsets":[[0,1],[0,1],[1,2],[3,4],[3,4],[4,5],[5,6],[7,8],[7,8],[9,10],[9,10],[10,11],[11,13],[14,15],[14,15],[15,16],[16,17],[18,19],[18,19],[19,20],[21,22],[21,22],[23,24],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[0,0,0,1,1,1,1,2,2,3,3,3,3,4,4,4,4,5,5,5,6,6,7,null],"decoded":"i xi2 0 ex dx 1","decoded_with_specials":"i xi2 0 ex dx 1"} +{"kind":"sample","source":"fixtures/lang/amh_Ethi.txt[:160]","text":"- ፍርድ ቤቱ በአቶ ዮናታን ተስፋዬ ላይ የ6 አመት ከ6 ወር ጽኑ የእስር ቅጣት አስተላለፈ\n- አንድነት ዶክተር ቴድሮስ ለዓለም ጤና ድርጅት ዋና ዳይሬክተርነት በመመረጣቸው ደስታውን ገለጸ\n- ህንድ መጠነ ሰፊ የድንጋይ ከሰል ሃይል ጣብያዎች የመገንባት እ","ids":[3,18,3,2,3,2,3,2,3,2,3,2,3,2,3,2,948,3,2,3,2,948,3,2,3,2,3,2,3,2,3,2,3,18,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,18,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,1],"ids_no_specials":[3,18,3,2,3,2,3,2,3,2,3,2,3,2,3,2,948,3,2,3,2,948,3,2,3,2,3,2,3,2,3,2,3,18,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,18,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2],"tokens":["▁","-","▁","ፍርድ","▁","ቤቱ","▁","በአቶ","▁","ዮናታን","▁","ተስፋዬ","▁","ላይ","▁","የ","6","▁","አመት","▁","ከ","6","▁","ወር","▁","ጽኑ","▁","የእስር","▁","ቅጣት","▁","አስተላለፈ","▁","-","▁","አንድነት","▁","ዶክተር","▁","ቴድሮስ","▁","ለዓለም","▁","ጤና","▁","ድርጅት","▁","ዋና","▁","ዳይሬክተርነት","▁","በመመረጣቸው","▁","ደስታውን","▁","ገለጸ","▁","-","▁","ህንድ","▁","መጠነ","▁","ሰፊ","▁","የድንጋይ","▁","ከሰል","▁","ሃይል","▁","ጣብያዎች","▁","የመገንባት","▁","እ",""],"offsets":[[0,1],[0,1],[2,3],[2,5],[6,7],[6,8],[9,10],[9,12],[13,14],[13,17],[18,19],[18,22],[23,24],[23,25],[26,27],[26,27],[27,28],[29,30],[29,32],[33,34],[33,34],[34,35],[36,37],[36,38],[39,40],[39,41],[42,43],[42,46],[47,48],[47,50],[51,52],[51,57],[58,59],[58,59],[60,61],[60,65],[66,67],[66,70],[71,72],[71,75],[76,77],[76,80],[81,82],[81,83],[84,85],[84,88],[89,90],[89,91],[92,93],[92,100],[101,102],[101,108],[109,110],[109,114],[115,116],[115,118],[119,120],[119,120],[121,122],[121,124],[125,126],[125,128],[129,130],[129,131],[132,133],[132,137],[138,139],[138,141],[142,143],[142,145],[146,147],[146,151],[152,153],[152,158],[159,160],[159,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,7,8,8,9,9,9,10,10,11,11,12,12,13,13,14,14,15,15,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,32,32,33,33,34,34,35,35,36,36,null],"decoded":"- 6 6 - - ","decoded_with_specials":"- 6 6 - - "} +{"kind":"sample","source":"fixtures/lang/arb_Arab.txt[:160]","text":"[بالصور] ترقوميا : الموت يطفىء باقة ورد قيد التفتح !!\nالخليل – دوت كوم – من غسان عبد الحميد - لم يخطر في بال أهالي \"ترقوميا\" التي انخلع قلبها وهي تودع التراب جث","ids":[784,2,908,3,2,3,10,3,2,3,2,3,2,3,2,3,2,3,2,3,1603,3,2,3,104,3,2,3,2,3,104,3,2,3,2,3,2,3,2,3,18,3,2,3,2,3,2,3,2,3,2,96,2,121,3,2,3,2,3,2,3,2,3,2,3,2,3,2,1],"ids_no_specials":[784,2,908,3,2,3,10,3,2,3,2,3,2,3,2,3,2,3,2,3,1603,3,2,3,104,3,2,3,2,3,104,3,2,3,2,3,2,3,2,3,18,3,2,3,2,3,2,3,2,3,2,96,2,121,3,2,3,2,3,2,3,2,3,2,3,2,3,2],"tokens":["▁[","بالصور","]","▁","ترقوميا","▁",":","▁","الموت","▁","يطفىء","▁","باقة","▁","ورد","▁","قيد","▁","التفتح","▁","!!","▁","الخليل","▁","–","▁","دوت","▁","كوم","▁","–","▁","من","▁","غسان","▁","عبد","▁","الحميد","▁","-","▁","لم","▁","يخطر","▁","في","▁","بال","▁","أهالي","▁\"","ترقوميا","\"","▁","التي","▁","انخلع","▁","قلبها","▁","وهي","▁","تودع","▁","التراب","▁","جث",""],"offsets":[[0,1],[1,7],[7,8],[9,10],[9,16],[17,18],[17,18],[19,20],[19,24],[25,26],[25,30],[31,32],[31,35],[36,37],[36,39],[40,41],[40,43],[44,45],[44,50],[51,52],[51,53],[54,55],[54,60],[61,62],[61,62],[63,64],[63,66],[67,68],[67,70],[71,72],[71,72],[73,74],[73,75],[76,77],[76,80],[81,82],[81,84],[85,86],[85,91],[92,93],[92,93],[94,95],[94,96],[97,98],[97,101],[102,103],[102,104],[105,106],[105,108],[109,110],[109,114],[115,116],[116,123],[123,124],[125,126],[125,129],[130,131],[130,135],[136,137],[136,141],[142,143],[142,145],[146,147],[146,150],[151,152],[151,157],[158,159],[158,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,24,24,25,25,25,26,26,27,27,28,28,29,29,30,30,31,31,32,32,null],"decoded":"[] : !! – – - \"\" ","decoded_with_specials":"[] : !! – – - \"\" "} +{"kind":"sample","source":"fixtures/lang/ben_Beng.txt[:160]","text":"গ্রহ নীহারিকা\nগ্রহ নীহারিকা (ইংরেজি ভাষায়: Planetary nebula) এক বিশেষ ধরনের গ্যাসীয় নীহারিকা। যেসব তারার ভর কম, নির্দিষ্টভাবে বলতে গেলে যেসব তারার ভর সূর্যের ","ids":[3,2,3,2,3,2,3,2,41,2,3,2,10,12601,1208,3,29,15,6724,9,61,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,6,3,2,3,2,3,2,3,2,3,2,3,2,3,2,1],"ids_no_specials":[3,2,3,2,3,2,3,2,41,2,3,2,10,12601,1208,3,29,15,6724,9,61,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,6,3,2,3,2,3,2,3,2,3,2,3,2,3,2],"tokens":["▁","গ্রহ","▁","নীহারিকা","▁","গ্রহ","▁","নীহারিকা","▁(","ইংরেজি","▁","ভাষায়",":","▁Planet","ary","▁","n","e","bul","a",")","▁","এক","▁","বিশেষ","▁","ধরনের","▁","গ্যাসীয়","▁","নীহারিকা।","▁","যেসব","▁","তারার","▁","ভর","▁","কম",",","▁","নির্দিষ্টভাবে","▁","বলতে","▁","গেলে","▁","যেসব","▁","তারার","▁","ভর","▁","সূর্যের",""],"offsets":[[0,1],[0,4],[5,6],[5,13],[14,15],[14,18],[19,20],[19,27],[28,29],[29,35],[36,37],[36,42],[42,43],[44,50],[50,53],[54,55],[54,55],[55,56],[56,59],[59,60],[60,61],[62,63],[62,64],[65,66],[65,70],[71,72],[71,76],[77,78],[77,85],[86,87],[86,95],[96,97],[96,100],[101,102],[101,106],[107,108],[107,109],[110,111],[110,112],[112,113],[114,115],[114,127],[128,129],[128,132],[133,134],[133,137],[138,139],[138,142],[143,144],[143,148],[149,150],[149,151],[152,153],[152,159],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[0,0,1,1,2,2,3,3,4,4,5,5,5,6,6,7,7,7,7,7,7,8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,16,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,null],"decoded":" ( : Planetary nebula) , ","decoded_with_specials":" ( : Planetary nebula) , "} +{"kind":"sample","source":"fixtures/lang/cmn_Hani.txt[:160]","text":"強力建議大家未來盡量避免英航阿~~~\n班機原訂航程\n6/9 台北→香港\n6/9 香港→倫敦 (原訂23:15起飛，4:50am抵達)\n6/10 倫敦→斯德哥爾摩 (原訂7:40am起飛，11:05am抵達)\n就在第二段，英航no.25班機在香港機場兩度離開閘口又兩度返航\n第一次因有旅客身體不適 (好，不怪他，消耗時間也","ids":[3,2,3,2,431,87,1298,3,2,431,87,1298,3,2,41,2,2773,10,1808,2,6,591,10,1752,265,2,61,431,11476,3,2,41,2,940,10,2445,265,2,6,2596,10,3076,265,2,61,3,2,6,2,29,32,5,1828,2,3,2,41,2,6,2,6,2,1],"ids_no_specials":[3,2,3,2,431,87,1298,3,2,431,87,1298,3,2,41,2,2773,10,1808,2,6,591,10,1752,265,2,61,431,11476,3,2,41,2,940,10,2445,265,2,6,2596,10,3076,265,2,61,3,2,6,2,29,32,5,1828,2,3,2,41,2,6,2,6,2],"tokens":["▁","強力建議大家未來盡量避免英航阿~~~","▁","班機原訂航程","▁6","/","9","▁","台北→香港","▁6","/","9","▁","香港→倫敦","▁(","原訂","23",":","15","起飛",",","4",":","50","am","抵達",")","▁6","/10","▁","倫敦→斯德哥爾摩","▁(","原訂","7",":","40","am","起飛",",","11",":","05","am","抵達",")","▁","就在第二段",",","英航","n","o",".","25","班機在香港機場兩度離開閘口又兩度返航","▁","第一次因有旅客身體不適","▁(","好",",","不怪他",",","消耗時間也",""],"offsets":[[0,1],[0,18],[19,20],[19,25],[26,27],[27,28],[28,29],[30,31],[30,35],[36,37],[37,38],[38,39],[40,41],[40,45],[46,47],[47,49],[49,51],[51,52],[52,54],[54,56],[56,57],[57,58],[58,59],[59,61],[61,63],[63,65],[65,66],[67,68],[68,71],[72,73],[72,80],[81,82],[82,84],[84,85],[85,86],[86,88],[88,90],[90,92],[92,93],[93,95],[95,96],[96,98],[98,100],[100,102],[102,103],[104,105],[104,109],[109,110],[110,112],[112,113],[113,114],[114,115],[115,117],[117,135],[136,137],[136,147],[148,149],[149,150],[150,151],[151,154],[154,155],[155,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[0,0,1,1,2,2,2,3,3,4,4,4,5,5,6,6,6,6,6,6,6,6,6,6,6,6,6,7,7,8,8,9,9,9,9,9,9,9,9,9,9,9,9,9,9,10,10,10,10,10,10,10,10,10,11,11,12,12,12,12,12,12,null],"decoded":" 6/9 6/9 (23:15,4:50am) 6/10 (7:40am,11:05am) ,no.25 (,,","decoded_with_specials":" 6/9 6/9 (23:15,4:50am) 6/10 (7:40am,11:05am) ,no.25 (,,"} +{"kind":"sample","source":"fixtures/lang/ell_Grek.txt[:160]","text":"Θυμήσου με\nΗ πόλη προσφέρει πολλές ευκαιρίες - πήγαινε στο εμπορικό κέντρο, στο σαλόνι ομορφιάς , σε κλάμπ και διασκέδασε με την ψυχή σου.\nΔημιούργησε μαγευτικά","ids":[3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,18,3,2,3,2,3,2,3,2,6,3,2,3,2,3,2,3,6,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,5,3,2,3,2,1],"ids_no_specials":[3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,18,3,2,3,2,3,2,3,2,6,3,2,3,2,3,2,3,6,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,5,3,2,3,2],"tokens":["▁","Θυμήσου","▁","με","▁","Η","▁","πόλη","▁","προσφέρει","▁","πολλές","▁","ευκαιρίες","▁","-","▁","πήγαινε","▁","στο","▁","εμπορικό","▁","κέντρο",",","▁","στο","▁","σαλόνι","▁","ομορφιάς","▁",",","▁","σε","▁","κλάμπ","▁","και","▁","διασκέδασε","▁","με","▁","την","▁","ψυχή","▁","σου",".","▁","Δημιούργησε","▁","μαγευτικά",""],"offsets":[[0,1],[0,7],[8,9],[8,10],[11,12],[11,12],[13,14],[13,17],[18,19],[18,27],[28,29],[28,34],[35,36],[35,44],[45,46],[45,46],[47,48],[47,54],[55,56],[55,58],[59,60],[59,67],[68,69],[68,74],[74,75],[76,77],[76,79],[80,81],[80,86],[87,88],[87,95],[96,97],[96,97],[98,99],[98,100],[101,102],[101,106],[107,108],[107,110],[111,112],[111,121],[122,123],[122,124],[125,126],[125,128],[129,130],[129,133],[134,135],[134,137],[137,138],[139,140],[139,150],[151,152],[151,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,11,12,12,13,13,14,14,15,15,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,23,24,24,25,25,null],"decoded":" - , , . ","decoded_with_specials":" - , , . "} +{"kind":"sample","source":"fixtures/lang/eng_Latn.txt[:160]","text":"|Viewing Single Post From: Spoilers for the Week of February 11th|\n|Lil||Feb 1 2013, 09:58 AM|\nDon't care about Chloe/Taniel/Jen-Jen. Don't care about Sami, rea","ids":[1820,15270,53,7871,1844,1029,10,8927,173,277,21,8,6551,13,2083,850,189,9175,1820,434,173,9175,9175,371,15,115,209,7218,14146,10,3449,5422,9175,1008,31,17,124,81,4004,40,32,15,87,382,2738,15,40,87,683,35,18,683,35,5,1008,31,17,124,81,3084,23,6,3,864,1],"ids_no_specials":[1820,15270,53,7871,1844,1029,10,8927,173,277,21,8,6551,13,2083,850,189,9175,1820,434,173,9175,9175,371,15,115,209,7218,14146,10,3449,5422,9175,1008,31,17,124,81,4004,40,32,15,87,382,2738,15,40,87,683,35,18,683,35,5,1008,31,17,124,81,3084,23,6,3,864],"tokens":["▁|","View","ing","▁Single","▁Post","▁From",":","▁Spo","il","ers","▁for","▁the","▁Week","▁of","▁February","▁11","th","|","▁|","L","il","|","|","F","e","b","▁1","▁2013,","▁09",":","58","▁AM","|","▁Don","'","t","▁care","▁about","▁Ch","l","o","e","/","T","ani","e","l","/","J","en","-","J","en",".","▁Don","'","t","▁care","▁about","▁Sam","i",",","▁","rea",""],"offsets":[[0,1],[1,5],[5,8],[9,15],[16,20],[21,25],[25,26],[27,30],[30,32],[32,35],[36,39],[40,43],[44,48],[49,51],[52,60],[61,63],[63,65],[65,66],[67,68],[68,69],[69,71],[71,72],[72,73],[73,74],[74,75],[75,76],[77,78],[79,84],[85,87],[87,88],[88,90],[91,93],[93,94],[95,98],[98,99],[99,100],[101,105],[106,111],[112,114],[114,115],[115,116],[116,117],[117,118],[118,119],[119,122],[122,123],[123,124],[124,125],[125,126],[126,128],[128,129],[129,130],[130,132],[132,133],[134,137],[137,138],[138,139],[140,144],[145,150],[151,154],[154,155],[155,156],[157,158],[157,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[0,0,0,1,2,3,3,4,4,4,5,6,7,8,9,10,10,10,11,11,11,11,11,11,11,11,12,13,14,14,14,15,15,16,16,16,17,18,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,20,20,20,21,22,23,23,23,24,24,null],"decoded":"|Viewing Single Post From: Spoilers for the Week of February 11th| |Lil||Feb 1 2013, 09:58 AM| Don't care about Chloe/Taniel/Jen-Jen. Don't care about Sami, rea","decoded_with_specials":"|Viewing Single Post From: Spoilers for the Week of February 11th| |Lil||Feb 1 2013, 09:58 AM| Don't care about Chloe/Taniel/Jen-Jen. Don't care about Sami, rea"} +{"kind":"sample","source":"fixtures/lang/heb_Hebr.txt[:160]","text":"סיכונים – אלמנט מרכזי, חיוני לפעילות מסחרית. להבין מה הסיכון הוא מאוד חשוב. החוויה האנושית עולה כי מי יודע איך לקחת סיכונים בזמן, היא לנצח גדולים. לזכור פוליטיק","ids":[3,2,3,104,3,2,3,2,6,3,2,3,2,3,2,5,3,2,3,2,3,2,3,2,3,2,3,2,5,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,6,3,2,3,2,3,2,5,3,2,3,2,1],"ids_no_specials":[3,2,3,104,3,2,3,2,6,3,2,3,2,3,2,5,3,2,3,2,3,2,3,2,3,2,3,2,5,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,6,3,2,3,2,3,2,5,3,2,3,2],"tokens":["▁","סיכונים","▁","–","▁","אלמנט","▁","מרכזי",",","▁","חיוני","▁","לפעילות","▁","מסחרית",".","▁","להבין","▁","מה","▁","הסיכון","▁","הוא","▁","מאוד","▁","חשוב",".","▁","החוויה","▁","האנושית","▁","עולה","▁","כי","▁","מי","▁","יודע","▁","איך","▁","לקחת","▁","סיכונים","▁","בזמן",",","▁","היא","▁","לנצח","▁","גדולים",".","▁","לזכור","▁","פוליטיק",""],"offsets":[[0,1],[0,7],[8,9],[8,9],[10,11],[10,15],[16,17],[16,21],[21,22],[23,24],[23,28],[29,30],[29,36],[37,38],[37,43],[43,44],[45,46],[45,50],[51,52],[51,53],[54,55],[54,60],[61,62],[61,64],[65,66],[65,69],[70,71],[70,74],[74,75],[76,77],[76,82],[83,84],[83,90],[91,92],[91,95],[96,97],[96,98],[99,100],[99,101],[102,103],[102,106],[107,108],[107,110],[111,112],[111,115],[116,117],[116,123],[124,125],[124,128],[128,129],[130,131],[130,133],[134,135],[134,138],[139,140],[139,145],[145,146],[147,148],[147,152],[153,154],[153,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[0,0,1,1,2,2,3,3,3,4,4,5,5,6,6,6,7,7,8,8,9,9,10,10,11,11,12,12,12,13,13,14,14,15,15,16,16,17,17,18,18,19,19,20,20,21,21,22,22,22,23,23,24,24,25,25,25,26,26,27,27,null],"decoded":" – , . . , . ","decoded_with_specials":" – , . . , . "} +{"kind":"sample","source":"fixtures/lang/hin_Deva.txt[:160]","text":"PHOTOS: न्यूज पढ़ते-पढ़ते अचानक ये क्या करने लगी एंकर!\nकुछ समय पहले एक टीवी चैनल पर एंकर खबर पढ़ रही थी और पीछे की स्क्रीन पर 10 मिनिट का पोर्न वीडियो चलता रहा,","ids":[3,8023,22262,134,10,3,2,3,2,18,2,3,2,3,2,3,2,3,2,3,2,3,2,55,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,335,3,2,3,2,3,2,3,2,3,2,3,2,6,1],"ids_no_specials":[3,8023,22262,134,10,3,2,3,2,18,2,3,2,3,2,3,2,3,2,3,2,3,2,55,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,335,3,2,3,2,3,2,3,2,3,2,3,2,6],"tokens":["▁","PH","OTO","S",":","▁","न्यूज","▁","पढ़ते","-","पढ़ते","▁","अचानक","▁","ये","▁","क्या","▁","करने","▁","लगी","▁","एंकर","!","▁","कुछ","▁","समय","▁","पहले","▁","एक","▁","टीवी","▁","चैनल","▁","पर","▁","एंकर","▁","खबर","▁","पढ़","▁","रही","▁","थी","▁","और","▁","पीछे","▁","की","▁","स्क्रीन","▁","पर","▁10","▁","मिनिट","▁","का","▁","पोर्न","▁","वीडियो","▁","चलता","▁","रहा",",",""],"offsets":[[0,1],[0,2],[2,5],[5,6],[6,7],[8,9],[8,13],[14,15],[14,19],[19,20],[20,25],[26,27],[26,31],[32,33],[32,34],[35,36],[35,39],[40,41],[40,44],[45,46],[45,48],[49,50],[49,53],[53,54],[55,56],[55,58],[59,60],[59,62],[63,64],[63,67],[68,69],[68,70],[71,72],[71,75],[76,77],[76,80],[81,82],[81,83],[84,85],[84,88],[89,90],[89,92],[93,94],[93,96],[97,98],[97,100],[101,102],[101,103],[104,105],[104,106],[107,108],[107,111],[112,113],[112,114],[115,116],[115,122],[123,124],[123,125],[126,128],[129,130],[129,134],[135,136],[135,137],[138,139],[138,143],[144,145],[144,150],[151,152],[151,155],[156,157],[156,159],[159,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[0,0,0,0,0,1,1,2,2,2,2,3,3,4,4,5,5,6,6,7,7,8,8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,24,24,25,25,26,27,27,28,28,29,29,30,30,31,31,32,32,32,null],"decoded":"PHOTOS: - ! 10 ,","decoded_with_specials":"PHOTOS: - ! 10 ,"} +{"kind":"sample","source":"fixtures/lang/jpn_Jpan.txt[:160]","text":"Omni Dallas Parkwest Hotelでは4ツ星ホテルで、アイアンホース・ゴルフコース、Love Field Airport とテキサス・スタジアムから7.8kmかかるのところにあります。 優れたホテルにオープンし、ダラスにある古代の建築の象徴です。\n部屋\n快適なゲストルームには、モダンな設備を備えたプレ","ids":[24377,9628,1061,12425,2282,2,591,2,20808,7257,5735,3,2,940,5,927,5848,2,3,2,3,2,3,2,1],"ids_no_specials":[24377,9628,1061,12425,2282,2,591,2,20808,7257,5735,3,2,940,5,927,5848,2,3,2,3,2,3,2],"tokens":["▁Omni","▁Dallas","▁Park","west","▁Hotel","では","4","ツ星ホテルで、アイアンホース・ゴルフコース、","Love","▁Field","▁Airport","▁","とテキサス・スタジアムから","7",".","8","km","かかるのところにあります。","▁","優れたホテルにオープンし、ダラスにある古代の建築の象徴です。","▁","部屋","▁","快適なゲストルームには、モダンな設備を備えたプレ",""],"offsets":[[0,4],[5,11],[12,16],[16,20],[21,26],[26,28],[28,29],[29,51],[51,55],[56,61],[62,69],[70,71],[70,83],[83,84],[84,85],[85,86],[86,88],[88,101],[102,103],[102,132],[133,134],[133,135],[136,137],[136,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[0,1,2,2,3,3,3,3,3,4,5,6,6,6,6,6,6,6,7,7,8,8,9,9,null],"decoded":"Omni Dallas Parkwest Hotel4Love Field Airport 7.8km ","decoded_with_specials":"Omni Dallas Parkwest Hotel4Love Field Airport 7.8km "} +{"kind":"sample","source":"fixtures/lang/kat_Geor.txt[:160]","text":"ჩვენ მსოფლიოს სხვადასხვა ქვეყანაში ვცხოვრობთ და სხვადასხვა ენაზე ვსაუბრობთ, მაგრამ საერთო მიზანი გვაერთიანებს. ჩვენი მთავარი მიზანი სამყაროს შემოქმედისა და ბიბლ","ids":[3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,6,3,2,3,2,3,2,3,2,5,3,2,3,2,3,2,3,2,3,2,3,2,3,2,1],"ids_no_specials":[3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,6,3,2,3,2,3,2,3,2,5,3,2,3,2,3,2,3,2,3,2,3,2,3,2],"tokens":["▁","ჩვენ","▁","მსოფლიოს","▁","სხვადასხვა","▁","ქვეყანაში","▁","ვცხოვრობთ","▁","და","▁","სხვადასხვა","▁","ენაზე","▁","ვსაუბრობთ",",","▁","მაგრამ","▁","საერთო","▁","მიზანი","▁","გვაერთიანებს",".","▁","ჩვენი","▁","მთავარი","▁","მიზანი","▁","სამყაროს","▁","შემოქმედისა","▁","და","▁","ბიბლ",""],"offsets":[[0,1],[0,4],[5,6],[5,13],[14,15],[14,24],[25,26],[25,34],[35,36],[35,44],[45,46],[45,47],[48,49],[48,58],[59,60],[59,64],[65,66],[65,74],[74,75],[76,77],[76,82],[83,84],[83,89],[90,91],[90,96],[97,98],[97,109],[109,110],[111,112],[111,116],[117,118],[117,124],[125,126],[125,131],[132,133],[132,140],[141,142],[141,152],[153,154],[153,155],[156,157],[156,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,8,9,9,10,10,11,11,12,12,12,13,13,14,14,15,15,16,16,17,17,18,18,19,19,null],"decoded":" , . ","decoded_with_specials":" , . "} +{"kind":"sample","source":"fixtures/lang/kor_Hang.txt[:160]","text":"전화번호:\n위치: 뉴질랜드 > 남섬 > 말버러 > 블레넘\n가격대 (1박): 117,886 원 - 140,244 원\n4.5성급 — Lugano Motor Lodge 4.5*\n- 예약 옵션:\n- 트립어드바이저는 호텔스닷컴, Expedia, Agoda, Asia Web Direct 및 Boo","ids":[3,2,10,3,2,10,3,2,2490,3,2,2490,3,2,2490,3,2,3,2,4077,2,61,10,3,20275,6,927,3840,3,2,3,18,11397,6,357,3628,3,2,3,12451,2,3,318,2318,2565,32,5083,14265,3,12451,1935,3,18,3,2,3,2,10,3,18,3,2,3,2,6,1881,24477,6,71,839,26,9,6,3826,1620,7143,3,2,1491,32,1],"ids_no_specials":[3,2,10,3,2,10,3,2,2490,3,2,2490,3,2,2490,3,2,3,2,4077,2,61,10,3,20275,6,927,3840,3,2,3,18,11397,6,357,3628,3,2,3,12451,2,3,318,2318,2565,32,5083,14265,3,12451,1935,3,18,3,2,3,2,10,3,18,3,2,3,2,6,1881,24477,6,71,839,26,9,6,3826,1620,7143,3,2,1491,32],"tokens":["▁","전화번호",":","▁","위치",":","▁","뉴질랜드","▁>","▁","남섬","▁>","▁","말버러","▁>","▁","블레넘","▁","가격대","▁(1","박",")",":","▁","117",",","8","86","▁","원","▁","-","▁140",",","2","44","▁","원","▁","4.5","성급","▁","—","▁Lu","gan","o","▁Motor","▁Lodge","▁","4.5","*","▁","-","▁","예약","▁","옵션",":","▁","-","▁","트립어드바이저는","▁","호텔스닷컴",",","▁Ex","pedia",",","▁A","go","d","a",",","▁Asia","▁Web","▁Direct","▁","및","▁Bo","o",""],"offsets":[[0,1],[0,4],[4,5],[6,7],[6,8],[8,9],[10,11],[10,14],[15,16],[17,18],[17,19],[20,21],[22,23],[22,25],[26,27],[28,29],[28,31],[32,33],[32,35],[36,38],[38,39],[39,40],[40,41],[42,43],[42,45],[45,46],[46,47],[47,49],[50,51],[50,51],[52,53],[52,53],[54,57],[57,58],[58,59],[59,61],[62,63],[62,63],[64,65],[64,67],[67,69],[70,71],[70,71],[72,74],[74,77],[77,78],[79,84],[85,90],[91,92],[91,94],[94,95],[96,97],[96,97],[98,99],[98,100],[101,102],[101,103],[103,104],[105,106],[105,106],[107,108],[107,115],[116,117],[116,121],[121,122],[123,125],[125,130],[130,131],[132,133],[133,135],[135,136],[136,137],[137,138],[139,143],[144,147],[148,154],[155,156],[155,156],[157,159],[159,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[0,0,0,1,1,1,2,2,3,4,4,5,6,6,7,8,8,9,9,10,10,10,10,11,11,11,11,11,12,12,13,13,14,14,14,14,15,15,16,16,16,17,17,18,18,18,19,20,21,21,21,22,22,23,23,24,24,24,25,25,26,26,27,27,27,28,28,28,29,29,29,29,29,30,31,32,33,33,34,34,null],"decoded":": : > > > (1): 117,886 - 140,244 4.5 — Lugano Motor Lodge 4.5* - : - , Expedia, Agoda, Asia Web Direct Boo","decoded_with_specials":": : > > > (1): 117,886 - 140,244 4.5 — Lugano Motor Lodge 4.5* - : - , Expedia, Agoda, Asia Web Direct Boo"} +{"kind":"sample","source":"fixtures/lang/rus_Cyrl.txt[:160]","text":"Покупая продукты в супермаркетах, сегодня уже мало кто верит в их качество. Как не сделать свое меню экстремальным и на что следует обращать внимание – читайте!","ids":[3,2,2044,25644,2,2533,2,3,2,10338,5814,3700,6652,6725,2,8724,12681,3700,2,17657,20447,8452,6652,1757,15517,2,6,3,22036,2,17238,7184,2,3,3700,2,1757,3,20447,21044,3,6652,9592,8724,17657,22682,8724,3,2795,2,3,12095,2,1757,24832,2044,5,3,2,2533,6652,3,14142,12681,19473,17148,6725,2,12681,19414,1757,3,28232,2,3,2,6652,10458,13400,20447,6588,2,7184,2,6469,3,2795,3,8194,3,2,9592,12681,19229,5814,3700,15042,3,2044,2,7948,2,22581,2,8724,14391,6469,27616,1757,3,104,3,2,2795,15517,2,14982,55,1],"ids_no_specials":[3,2,2044,25644,2,2533,2,3,2,10338,5814,3700,6652,6725,2,8724,12681,3700,2,17657,20447,8452,6652,1757,15517,2,6,3,22036,2,17238,7184,2,3,3700,2,1757,3,20447,21044,3,6652,9592,8724,17657,22682,8724,3,2795,2,3,12095,2,1757,24832,2044,5,3,2,2533,6652,3,14142,12681,19473,17148,6725,2,12681,19414,1757,3,28232,2,3,2,6652,10458,13400,20447,6588,2,7184,2,6469,3,2795,3,8194,3,2,9592,12681,19229,5814,3700,15042,3,2044,2,7948,2,22581,2,8724,14391,6469,27616,1757,3,104,3,2,2795,15517,2,14982,55],"tokens":["▁","П","о","ку","п","а","я","▁","п","ро","д","у","к","т","ы","▁в","▁с","у","п","ер","ма","р","к","е","та","х",",","▁","се","г","од","н","я","▁","у","ж","е","▁","ма","ло","▁","к","то","▁в","ер","ит","▁в","▁","и","х","▁","ка","ч","е","ств","о",".","▁","К","а","к","▁","не","▁с","де","ла","т","ь","▁с","во","е","▁","мен","ю","▁","э","к","ст","ре","ма","л","ь","н","ы","м","▁","и","▁","на","▁","ч","то","▁с","ле","д","у","ет","▁","о","б","ра","щ","ат","ь","▁в","ни","м","ани","е","▁","–","▁","ч","и","та","й","те","!",""],"offsets":[[0,1],[0,1],[1,2],[2,4],[4,5],[5,6],[6,7],[8,9],[8,9],[9,11],[11,12],[12,13],[13,14],[14,15],[15,16],[17,18],[19,20],[20,21],[21,22],[22,24],[24,26],[26,27],[27,28],[28,29],[29,31],[31,32],[32,33],[34,35],[34,36],[36,37],[37,39],[39,40],[40,41],[42,43],[42,43],[43,44],[44,45],[46,47],[46,48],[48,50],[51,52],[51,52],[52,54],[55,56],[56,58],[58,60],[61,62],[63,64],[63,64],[64,65],[66,67],[66,68],[68,69],[69,70],[70,73],[73,74],[74,75],[76,77],[76,77],[77,78],[78,79],[80,81],[80,82],[83,84],[84,86],[86,88],[88,89],[89,90],[91,92],[92,94],[94,95],[96,97],[96,99],[99,100],[101,102],[101,102],[102,103],[103,105],[105,107],[107,109],[109,110],[110,111],[111,112],[112,113],[113,114],[115,116],[115,116],[117,118],[117,119],[120,121],[120,121],[121,123],[124,125],[125,127],[127,128],[128,129],[129,131],[132,133],[132,133],[133,134],[134,136],[136,137],[137,139],[139,140],[141,142],[142,144],[144,145],[145,148],[148,149],[150,151],[150,151],[152,153],[152,153],[153,154],[154,156],[156,157],[157,159],[159,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,2,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,5,5,5,5,6,6,6,7,7,7,8,8,8,9,10,10,10,11,11,11,11,11,11,11,12,12,12,12,13,13,14,14,14,14,14,15,15,15,16,16,16,17,17,17,17,17,17,17,17,17,17,17,18,18,19,19,20,20,20,21,21,21,21,21,22,22,22,22,22,22,22,23,23,23,23,23,24,24,25,25,25,25,25,25,25,null],"decoded":"окуа родукт в суермаркета, сеодн уе мало кто верит в и каество. ак не сделат свое мен кстремалнм и на то следует ораат внимание – итате!","decoded_with_specials":"окуа родукт в суермаркета, сеодн уе мало кто верит в и каество. ак не сделат свое мен кстремалнм и на то следует ораат внимание – итате!"} +{"kind":"sample","source":"fixtures/lang/tam_Taml.txt[:160]","text":"‘ஹலோ கலெக்டர் சாரா… சரக்கு வேணும்… கடை எப்ப திறப்பீங்க?’\nஒரு படத்தில் ஒயின்ஷாப்புக்குள் திருடப் போன வடிவேலு நன்றாக சரக்கடித்து விட்டு, ஒயின்ஷாப் ஓனருக்கு போன் ப","ids":[458,2,3,2,3,2,233,3,2,3,2,233,3,2,3,2,3,2,58,22,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,6,3,2,3,2,3,2,3,2,1],"ids_no_specials":[458,2,3,2,3,2,233,3,2,3,2,233,3,2,3,2,3,2,58,22,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,6,3,2,3,2,3,2,3,2],"tokens":["▁‘","ஹலோ","▁","கலெக்டர்","▁","சாரா","...","▁","சரக்கு","▁","வேணும்","...","▁","கடை","▁","எப்ப","▁","திறப்பீங்க","?","’","▁","ஒரு","▁","படத்தில்","▁","ஒயின்ஷாப்புக்குள்","▁","திருடப்","▁","போன","▁","வடிவேலு","▁","நன்றாக","▁","சரக்கடித்து","▁","விட்டு",",","▁","ஒயின்ஷாப்","▁","ஓனருக்கு","▁","போன்","▁","ப",""],"offsets":[[0,1],[1,4],[5,6],[5,13],[14,15],[14,18],[18,19],[20,21],[20,26],[27,28],[27,33],[33,34],[35,36],[35,38],[39,40],[39,43],[44,45],[44,54],[54,55],[55,56],[57,58],[57,60],[61,62],[61,69],[70,71],[70,87],[88,89],[88,95],[96,97],[96,99],[100,101],[100,107],[108,109],[108,114],[115,116],[115,126],[127,128],[127,133],[133,134],[135,136],[135,144],[145,146],[145,153],[154,155],[154,158],[159,160],[159,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[0,0,1,1,2,2,2,3,3,4,4,4,5,5,6,6,7,7,7,7,8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,16,16,16,17,17,18,18,19,19,20,20,null],"decoded":"‘ ... ... ?’ , ","decoded_with_specials":"‘ ... ... ?’ , "} +{"kind":"sample","source":"fixtures/lang/tha_Thai.txt[:160]","text":"นับเจดีย์ดูนะครับว่าครบยี่สิบองค์รึเปล่า คำว่าซาว ทางเหนือแปลว่่ายี่สิบครับ ผมนับดูแล้วก็ครบนะ ลองดู เสร็จแล้วก็เข้าเยี่ยมชมพิพธภัณฑ์ เดินเข้าไปด้านในหน่อยก็จะม","ids":[3,2,3,2,3,2,3,2,3,2,3,2,3,2,1],"ids_no_specials":[3,2,3,2,3,2,3,2,3,2,3,2,3,2],"tokens":["▁","นับเจดีย์ดูนะครับว่าครบยี่สิบองค์รึเปล่า","▁","คําว่าซาว","▁","ทางเหนือแปลว่่ายี่สิบครับ","▁","ผมนับดูแล้วก็ครบนะ","▁","ลองดู","▁","เสร็จแล้วก็เข้าเยี่ยมชมพิพธภัณฑ์","▁","เดินเข้าไปด้านในหน่อยก็จะม",""],"offsets":[[0,1],[0,40],[41,42],[41,49],[50,51],[50,75],[76,77],[76,94],[95,96],[95,100],[101,102],[101,133],[134,135],[134,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[0,0,1,1,2,2,3,3,4,4,5,5,6,6,null],"decoded":" ","decoded_with_specials":" "} +{"kind":"sample","source":"fixtures/modalities/added_normalized_dense.txt[:160]","text":"FLIBBERJAST CRUNGLEDORF FLIBBERJAST SNORLAXIAN fast BLORPTRONIC ZORPTASTIC split WIDGETRON split stage SNORLAXIAN BLORPTRONIC BLORPTRONIC \n QUIBBLENAUT CRUNGLED","ids":[7212,196,279,12920,683,12510,3,4545,25158,17717,2990,371,7212,196,279,12920,683,12510,180,24833,4569,4,21758,1006,272,5017,6294,22749,18830,1027,2990,6383,12510,4666,5679,549,4309,20750,13044,5679,1726,180,24833,4569,4,21758,272,5017,6294,22749,18830,272,5017,6294,22749,18830,3,21672,7640,3765,5999,6675,3,4545,25158,17717,1],"ids_no_specials":[7212,196,279,12920,683,12510,3,4545,25158,17717,2990,371,7212,196,279,12920,683,12510,180,24833,4569,4,21758,1006,272,5017,6294,22749,18830,1027,2990,6383,12510,4666,5679,549,4309,20750,13044,5679,1726,180,24833,4569,4,21758,272,5017,6294,22749,18830,272,5017,6294,22749,18830,3,21672,7640,3765,5999,6675,3,4545,25158,17717],"tokens":["▁FL","I","B","BER","J","AST","▁","CR","UNG","LED","OR","F","▁FL","I","B","BER","J","AST","▁S","NOR","LA","X","IAN","▁fast","▁B","LO","RP","TRO","NIC","▁Z","OR","PT","AST","IC","▁split","▁W","ID","GET","RON","▁split","▁stage","▁S","NOR","LA","X","IAN","▁B","LO","RP","TRO","NIC","▁B","LO","RP","TRO","NIC","▁","QUI","BB","LE","NA","UT","▁","CR","UNG","LED",""],"offsets":[[0,2],[2,3],[3,4],[4,7],[7,8],[8,11],[12,13],[12,14],[14,17],[17,20],[20,22],[22,23],[24,26],[26,27],[27,28],[28,31],[31,32],[32,35],[36,37],[37,40],[40,42],[42,43],[43,46],[47,51],[52,53],[53,55],[55,57],[57,60],[60,63],[64,65],[65,67],[67,69],[69,72],[72,74],[75,80],[81,82],[82,84],[84,87],[87,90],[91,96],[97,102],[103,104],[104,107],[107,109],[109,110],[110,113],[114,115],[115,117],[117,119],[119,122],[122,125],[126,127],[127,129],[129,131],[131,134],[134,137],[140,141],[140,143],[143,145],[145,147],[147,149],[149,151],[152,153],[152,154],[154,157],[157,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[0,0,0,0,0,0,1,1,1,1,1,1,2,2,2,2,2,2,3,3,3,3,3,4,5,5,5,5,5,6,6,6,6,6,7,8,8,8,8,9,10,11,11,11,11,11,12,12,12,12,12,13,13,13,13,13,14,14,14,14,14,14,15,15,15,15,null],"decoded":"FLIBBERJAST CRUNGLEDORF FLIBBERJAST SNORLAXIAN fast BLORPTRONIC ZORPTASTIC split WIDGETRON split stage SNORLAXIAN BLORPTRONIC BLORPTRONIC QUIBBLENAUT CRUNGLED","decoded_with_specials":"FLIBBERJAST CRUNGLEDORF FLIBBERJAST SNORLAXIAN fast BLORPTRONIC ZORPTASTIC split WIDGETRON split stage SNORLAXIAN BLORPTRONIC BLORPTRONIC QUIBBLENAUT CRUNGLED"} +{"kind":"sample","source":"fixtures/modalities/added_normalized_sparse.txt[:160]","text":"ZORPTASTIC for we for decode CRUNGLEDORF here through ZORPTASTIC and the we WIDGETRON model \n normalize bytes and back flows text CRUNGLEDORF WUZZLEFANG chunk b","ids":[1027,2990,6383,12510,4666,21,62,21,20,4978,3,4545,25158,17717,2990,371,270,190,1027,2990,6383,12510,4666,11,8,62,549,4309,20750,13044,825,1389,1737,57,1422,11,223,14428,1499,3,4545,25158,17717,2990,371,549,1265,956,956,3765,371,19775,16749,3,115,1],"ids_no_specials":[1027,2990,6383,12510,4666,21,62,21,20,4978,3,4545,25158,17717,2990,371,270,190,1027,2990,6383,12510,4666,11,8,62,549,4309,20750,13044,825,1389,1737,57,1422,11,223,14428,1499,3,4545,25158,17717,2990,371,549,1265,956,956,3765,371,19775,16749,3,115],"tokens":["▁Z","OR","PT","AST","IC","▁for","▁we","▁for","▁de","code","▁","CR","UNG","LED","OR","F","▁here","▁through","▁Z","OR","PT","AST","IC","▁and","▁the","▁we","▁W","ID","GET","RON","▁model","▁normal","ize","▁by","tes","▁and","▁back","▁flows","▁text","▁","CR","UNG","LED","OR","F","▁W","U","Z","Z","LE","F","ANG","▁chunk","▁","b",""],"offsets":[[0,1],[1,3],[3,5],[5,8],[8,10],[11,14],[15,17],[18,21],[22,24],[24,28],[29,30],[29,31],[31,34],[34,37],[37,39],[39,40],[41,45],[46,53],[54,55],[55,57],[57,59],[59,62],[62,64],[65,68],[69,72],[73,75],[76,77],[77,79],[79,82],[82,85],[86,91],[94,100],[100,103],[104,106],[106,109],[110,113],[114,118],[119,124],[125,129],[130,131],[130,132],[132,135],[135,138],[138,140],[140,141],[142,143],[143,144],[144,145],[145,146],[146,148],[148,149],[149,152],[153,158],[159,160],[159,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[0,0,0,0,0,1,2,3,4,4,5,5,5,5,5,5,6,7,8,8,8,8,8,9,10,11,12,12,12,12,13,14,14,15,15,16,17,18,19,20,20,20,20,20,20,21,21,21,21,21,21,21,22,23,23,null],"decoded":"ZORPTASTIC for we for decode CRUNGLEDORF here through ZORPTASTIC and the we WIDGETRON model normalize bytes and back flows text CRUNGLEDORF WUZZLEFANG chunk b","decoded_with_specials":"ZORPTASTIC for we for decode CRUNGLEDORF here through ZORPTASTIC and the we WIDGETRON model normalize bytes and back flows text CRUNGLEDORF WUZZLEFANG chunk b"} +{"kind":"sample","source":"fixtures/modalities/added_special_dense.txt[:160]","text":"fast <|xs2|> chunk <|xs1|> <|xs4|> <|xs3|> <|xs4|> modality language <|xs0|> reads <|xs2|> and and \n <|xs1|> <|xs0|> <|xs1|> <|xs4|> <|xs4|> <|xs1|> back and re","ids":[1006,3,2,9175,226,7,357,9175,3155,16749,3,2,9175,226,7,536,9175,3155,3,2,9175,226,7,591,9175,3155,3,2,9175,226,7,519,9175,3155,3,2,9175,226,7,591,9175,3155,1794,10355,1612,3,2,9175,226,7,632,9175,3155,608,7,3,2,9175,226,7,357,9175,3155,11,11,3,2,9175,226,7,536,9175,3155,3,2,9175,226,7,632,9175,3155,3,2,9175,226,7,536,9175,3155,3,2,9175,226,7,591,9175,3155,3,2,9175,226,7,591,9175,3155,3,2,9175,226,7,536,9175,3155,223,11,3,60,1],"ids_no_specials":[1006,3,2,9175,226,7,357,9175,3155,16749,3,2,9175,226,7,536,9175,3155,3,2,9175,226,7,591,9175,3155,3,2,9175,226,7,519,9175,3155,3,2,9175,226,7,591,9175,3155,1794,10355,1612,3,2,9175,226,7,632,9175,3155,608,7,3,2,9175,226,7,357,9175,3155,11,11,3,2,9175,226,7,536,9175,3155,3,2,9175,226,7,632,9175,3155,3,2,9175,226,7,536,9175,3155,3,2,9175,226,7,591,9175,3155,3,2,9175,226,7,591,9175,3155,3,2,9175,226,7,536,9175,3155,223,11,3,60],"tokens":["▁fast","▁","<","|","x","s","2","|",">","▁chunk","▁","<","|","x","s","1","|",">","▁","<","|","x","s","4","|",">","▁","<","|","x","s","3","|",">","▁","<","|","x","s","4","|",">","▁mod","ality","▁language","▁","<","|","x","s","0","|",">","▁read","s","▁","<","|","x","s","2","|",">","▁and","▁and","▁","<","|","x","s","1","|",">","▁","<","|","x","s","0","|",">","▁","<","|","x","s","1","|",">","▁","<","|","x","s","4","|",">","▁","<","|","x","s","4","|",">","▁","<","|","x","s","1","|",">","▁back","▁and","▁","re",""],"offsets":[[0,4],[5,6],[5,6],[6,7],[7,8],[8,9],[9,10],[10,11],[11,12],[13,18],[19,20],[19,20],[20,21],[21,22],[22,23],[23,24],[24,25],[25,26],[27,28],[27,28],[28,29],[29,30],[30,31],[31,32],[32,33],[33,34],[35,36],[35,36],[36,37],[37,38],[38,39],[39,40],[40,41],[41,42],[43,44],[43,44],[44,45],[45,46],[46,47],[47,48],[48,49],[49,50],[51,54],[54,59],[60,68],[69,70],[69,70],[70,71],[71,72],[72,73],[73,74],[74,75],[75,76],[77,81],[81,82],[83,84],[83,84],[84,85],[85,86],[86,87],[87,88],[88,89],[89,90],[91,94],[95,98],[101,102],[101,102],[102,103],[103,104],[104,105],[105,106],[106,107],[107,108],[109,110],[109,110],[110,111],[111,112],[112,113],[113,114],[114,115],[115,116],[117,118],[117,118],[118,119],[119,120],[120,121],[121,122],[122,123],[123,124],[125,126],[125,126],[126,127],[127,128],[128,129],[129,130],[130,131],[131,132],[133,134],[133,134],[134,135],[135,136],[136,137],[137,138],[138,139],[139,140],[141,142],[141,142],[142,143],[143,144],[144,145],[145,146],[146,147],[147,148],[149,153],[154,157],[158,159],[158,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[0,1,1,1,1,1,1,1,1,2,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,7,7,8,9,9,9,9,9,9,9,9,10,10,11,11,11,11,11,11,11,11,12,13,14,14,14,14,14,14,14,14,15,15,15,15,15,15,15,15,16,16,16,16,16,16,16,16,17,17,17,17,17,17,17,17,18,18,18,18,18,18,18,18,19,19,19,19,19,19,19,19,20,21,22,22,null],"decoded":"fast |xs2|> chunk |xs1|> |xs4|> |xs3|> |xs4|> modality language |xs0|> reads |xs2|> and and |xs1|> |xs0|> |xs1|> |xs4|> |xs4|> |xs1|> back and re","decoded_with_specials":"fast |xs2|> chunk |xs1|> |xs4|> |xs3|> |xs4|> modality language |xs0|> reads |xs2|> and and |xs1|> |xs0|> |xs1|> |xs4|> |xs4|> |xs1|> back and re"} +{"kind":"sample","source":"fixtures/modalities/added_special_sparse.txt[:160]","text":"<|xs0|> every for encode <|xs0|> bytes the normalize decode text model <|xs4|> <|xs3|> and \n and <|xs3|> here <|xs1|> again model here text <|xs2|> <|xs2|> lang","ids":[3,2,9175,226,7,632,9175,3155,334,21,23734,3,2,9175,226,7,632,9175,3155,57,1422,8,1389,1737,20,4978,1499,825,3,2,9175,226,7,591,9175,3155,3,2,9175,226,7,519,9175,3155,11,11,3,2,9175,226,7,519,9175,3155,270,3,2,9175,226,7,536,9175,3155,541,825,270,1499,3,2,9175,226,7,357,9175,3155,3,2,9175,226,7,357,9175,3155,12142,1],"ids_no_specials":[3,2,9175,226,7,632,9175,3155,334,21,23734,3,2,9175,226,7,632,9175,3155,57,1422,8,1389,1737,20,4978,1499,825,3,2,9175,226,7,591,9175,3155,3,2,9175,226,7,519,9175,3155,11,11,3,2,9175,226,7,519,9175,3155,270,3,2,9175,226,7,536,9175,3155,541,825,270,1499,3,2,9175,226,7,357,9175,3155,3,2,9175,226,7,357,9175,3155,12142],"tokens":["▁","<","|","x","s","0","|",">","▁every","▁for","▁encode","▁","<","|","x","s","0","|",">","▁by","tes","▁the","▁normal","ize","▁de","code","▁text","▁model","▁","<","|","x","s","4","|",">","▁","<","|","x","s","3","|",">","▁and","▁and","▁","<","|","x","s","3","|",">","▁here","▁","<","|","x","s","1","|",">","▁again","▁model","▁here","▁text","▁","<","|","x","s","2","|",">","▁","<","|","x","s","2","|",">","▁lang",""],"offsets":[[0,1],[0,1],[1,2],[2,3],[3,4],[4,5],[5,6],[6,7],[8,13],[14,17],[18,24],[25,26],[25,26],[26,27],[27,28],[28,29],[29,30],[30,31],[31,32],[33,35],[35,38],[39,42],[43,49],[49,52],[53,55],[55,59],[60,64],[65,70],[71,72],[71,72],[72,73],[73,74],[74,75],[75,76],[76,77],[77,78],[79,80],[79,80],[80,81],[81,82],[82,83],[83,84],[84,85],[85,86],[87,90],[93,96],[97,98],[97,98],[98,99],[99,100],[100,101],[101,102],[102,103],[103,104],[105,109],[110,111],[110,111],[111,112],[112,113],[113,114],[114,115],[115,116],[116,117],[118,123],[124,129],[130,134],[135,139],[140,141],[140,141],[141,142],[142,143],[143,144],[144,145],[145,146],[146,147],[148,149],[148,149],[149,150],[150,151],[151,152],[152,153],[153,154],[154,155],[156,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[0,0,0,0,0,0,0,0,1,2,3,4,4,4,4,4,4,4,4,5,5,6,7,7,8,8,9,10,11,11,11,11,11,11,11,11,12,12,12,12,12,12,12,12,13,14,15,15,15,15,15,15,15,15,16,17,17,17,17,17,17,17,17,18,19,20,21,22,22,22,22,22,22,22,22,23,23,23,23,23,23,23,23,24,null],"decoded":"|xs0|> every for encode |xs0|> bytes the normalize decode text model |xs4|> |xs3|> and and |xs3|> here |xs1|> again model here text |xs2|> |xs2|> lang","decoded_with_specials":"|xs0|> every for encode |xs0|> bytes the normalize decode text model |xs4|> |xs3|> and and |xs3|> here |xs1|> again model here text |xs2|> |xs2|> lang"} +{"kind":"sample","source":"fixtures/modalities/agentic-traces.txt[:160]","text":"Analyze this codebase and summarize what it is and how it works.\nI need to explore the codebase structure by reading the main entry points and configuration fil","ids":[5331,120,776,48,1081,10925,11,21603,125,34,19,11,149,34,930,5,27,174,12,2075,8,1081,10925,1809,57,1183,8,711,1764,979,11,5298,5375,1],"ids_no_specials":[5331,120,776,48,1081,10925,11,21603,125,34,19,11,149,34,930,5,27,174,12,2075,8,1081,10925,1809,57,1183,8,711,1764,979,11,5298,5375],"tokens":["▁Ana","ly","ze","▁this","▁code","base","▁and","▁summarize","▁what","▁it","▁is","▁and","▁how","▁it","▁works",".","▁I","▁need","▁to","▁explore","▁the","▁code","base","▁structure","▁by","▁reading","▁the","▁main","▁entry","▁points","▁and","▁configuration","▁fil",""],"offsets":[[0,3],[3,5],[5,7],[8,12],[13,17],[17,21],[22,25],[26,35],[36,40],[41,43],[44,46],[47,50],[51,54],[55,57],[58,63],[63,64],[65,66],[67,71],[72,74],[75,82],[83,86],[87,91],[91,95],[96,105],[106,108],[109,116],[117,120],[121,125],[126,131],[132,138],[139,142],[143,156],[157,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[0,0,0,1,2,2,3,4,5,6,7,8,9,10,11,11,12,13,14,15,16,17,17,18,19,20,21,22,23,24,25,26,27,null],"decoded":"Analyze this codebase and summarize what it is and how it works. I need to explore the codebase structure by reading the main entry points and configuration fil","decoded_with_specials":"Analyze this codebase and summarize what it is and how it works. I need to explore the codebase structure by reading the main entry points and configuration fil"} +{"kind":"sample","source":"fixtures/modalities/agentic_swe.txt[:160]","text":"[system]\nYou are a helpful assistant that can interact with a computer to solve tasks.\n\n[user]\n\n/testbed\n\nI've uploaded a pytho","ids":[784,3734,908,148,33,3,9,2690,6165,24,54,6815,28,3,9,1218,12,4602,4145,5,784,10041,908,3,2,413,19496,834,11966,7,3155,3,87,4377,4143,3,2,87,413,19496,834,11966,7,3155,27,31,162,14686,3,9,3,102,63,189,32,1],"ids_no_specials":[784,3734,908,148,33,3,9,2690,6165,24,54,6815,28,3,9,1218,12,4602,4145,5,784,10041,908,3,2,413,19496,834,11966,7,3155,3,87,4377,4143,3,2,87,413,19496,834,11966,7,3155,27,31,162,14686,3,9,3,102,63,189,32],"tokens":["▁[","system","]","▁You","▁are","▁","a","▁helpful","▁assistant","▁that","▁can","▁interact","▁with","▁","a","▁computer","▁to","▁solve","▁tasks",".","▁[","user","]","▁","<","up","loaded","_","file","s",">","▁","/","test","bed","▁","<","/","up","loaded","_","file","s",">","▁I","'","ve","▁uploaded","▁","a","▁","p","y","th","o",""],"offsets":[[0,1],[1,7],[7,8],[9,12],[13,16],[17,18],[17,18],[19,26],[27,36],[37,41],[42,45],[46,54],[55,59],[60,61],[60,61],[62,70],[71,73],[74,79],[80,85],[85,86],[88,89],[89,93],[93,94],[95,96],[95,96],[96,98],[98,104],[104,105],[105,109],[109,110],[110,111],[112,113],[112,113],[113,117],[117,120],[121,122],[121,122],[122,123],[123,125],[125,131],[131,132],[132,136],[136,137],[137,138],[139,140],[140,141],[141,143],[144,152],[153,154],[153,154],[155,156],[155,156],[156,157],[157,159],[159,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[0,0,0,1,2,3,3,4,5,6,7,8,9,10,10,11,12,13,14,14,15,15,15,16,16,16,16,16,16,16,16,17,17,17,17,18,18,18,18,18,18,18,18,18,19,19,19,20,21,21,22,22,22,22,22,null],"decoded":"[system] You are a helpful assistant that can interact with a computer to solve tasks. [user] uploaded_files> /testbed /uploaded_files> I've uploaded a pytho","decoded_with_specials":"[system] You are a helpful assistant that can interact with a computer to solve tasks. [user] uploaded_files> /testbed /uploaded_files> I've uploaded a pytho"} +{"kind":"sample","source":"fixtures/modalities/code_mixed.txt[:160]","text":"// django-f3f9601cff03b38942e70ce8042bdfdec6002449/django/__init__.py\nfrom django.utils.version import get_version\n\nVERSION = (6, 2, 0, \"alpha\", 0)\n\n__version__","ids":[13751,3,26,7066,839,18,89,519,89,4314,4542,75,89,89,4928,115,3747,4240,357,15,2518,565,2079,4165,115,26,89,221,75,6007,2266,3647,87,26,7066,839,87,834,834,77,155,834,834,5,102,63,45,3,26,7066,839,5,13780,7,5,8674,4830,129,834,8674,3,26794,9215,3274,11372,6,3547,8014,96,138,6977,1686,3,632,61,3,834,834,8674,834,834,1],"ids_no_specials":[13751,3,26,7066,839,18,89,519,89,4314,4542,75,89,89,4928,115,3747,4240,357,15,2518,565,2079,4165,115,26,89,221,75,6007,2266,3647,87,26,7066,839,87,834,834,77,155,834,834,5,102,63,45,3,26,7066,839,5,13780,7,5,8674,4830,129,834,8674,3,26794,9215,3274,11372,6,3547,8014,96,138,6977,1686,3,632,61,3,834,834,8674,834,834],"tokens":["▁//","▁","d","jan","go","-","f","3","f","96","01","c","f","f","03","b","38","94","2","e","70","ce","80","42","b","d","f","de","c","600","24","49","/","d","jan","go","/","_","_","in","it","_","_",".","p","y","▁from","▁","d","jan","go",".","util","s",".","version","▁import","▁get","_","version","▁","VERS","ION","▁=","▁(6",",","▁2,","▁0,","▁\"","al","pha","\",","▁","0",")","▁","_","_","version","_","_",""],"offsets":[[0,2],[3,4],[3,4],[4,7],[7,9],[9,10],[10,11],[11,12],[12,13],[13,15],[15,17],[17,18],[18,19],[19,20],[20,22],[22,23],[23,25],[25,27],[27,28],[28,29],[29,31],[31,33],[33,35],[35,37],[37,38],[38,39],[39,40],[40,42],[42,43],[43,46],[46,48],[48,50],[50,51],[51,52],[52,55],[55,57],[57,58],[58,59],[59,60],[60,62],[62,64],[64,65],[65,66],[66,67],[67,68],[68,69],[70,74],[75,76],[75,76],[76,79],[79,81],[81,82],[82,86],[86,87],[87,88],[88,95],[96,102],[103,106],[106,107],[107,114],[116,117],[116,120],[120,123],[124,125],[126,128],[128,129],[130,132],[133,135],[136,137],[137,139],[139,142],[142,144],[145,146],[145,146],[146,147],[149,150],[149,150],[150,151],[151,158],[158,159],[159,160],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,3,3,3,3,3,3,3,3,3,4,5,5,5,6,6,6,7,8,8,9,10,11,11,11,11,12,12,12,13,13,13,13,13,13,null],"decoded":"// django-f3f9601cff03b38942e70ce8042bdfdec6002449/django/__init__.py from django.utils.version import get_version VERSION = (6, 2, 0, \"alpha\", 0) __version__","decoded_with_specials":"// django-f3f9601cff03b38942e70ce8042bdfdec6002449/django/__init__.py from django.utils.version import get_version VERSION = (6, 2, 0, \"alpha\", 0) __version__"} +{"kind":"sample","source":"fixtures/modalities/math_latex.txt[:160]","text":"Bayes and his Theorem\n\nMy earlier post on Bayesian probability seems to have generated quite a lot of readers, so this lunchtime I thought I’d add a little bit ","ids":[2474,15,7,11,112,37,127,15,51,499,2283,442,30,2474,15,10488,15834,1330,12,43,6126,882,3,9,418,13,3962,6,78,48,3074,715,27,816,27,22,26,617,3,9,385,720,1],"ids_no_specials":[2474,15,7,11,112,37,127,15,51,499,2283,442,30,2474,15,10488,15834,1330,12,43,6126,882,3,9,418,13,3962,6,78,48,3074,715,27,816,27,22,26,617,3,9,385,720],"tokens":["▁Bay","e","s","▁and","▁his","▁The","or","e","m","▁My","▁earlier","▁post","▁on","▁Bay","e","sian","▁probability","▁seems","▁to","▁have","▁generated","▁quite","▁","a","▁lot","▁of","▁readers",",","▁so","▁this","▁lunch","time","▁I","▁thought","▁I","’","d","▁add","▁","a","▁little","▁bit",""],"offsets":[[0,3],[3,4],[4,5],[6,9],[10,13],[14,17],[17,19],[19,20],[20,21],[23,25],[26,33],[34,38],[39,41],[42,45],[45,46],[46,50],[51,62],[63,68],[69,71],[72,76],[77,86],[87,92],[93,94],[93,94],[95,98],[99,101],[102,109],[109,110],[111,113],[114,118],[119,124],[124,128],[129,130],[131,138],[139,140],[140,141],[141,142],[143,146],[147,148],[147,148],[149,155],[156,159],[0,0]],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"special_tokens_mask":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"word_ids":[0,0,0,1,2,3,3,3,3,4,5,6,7,8,8,8,9,10,11,12,13,14,15,15,16,17,18,18,19,20,21,21,22,23,24,24,24,25,26,26,27,28,null],"decoded":"Bayes and his Theorem My earlier post on Bayesian probability seems to have generated quite a lot of readers, so this lunchtime I thought I’d add a little bit","decoded_with_specials":"Bayes and his Theorem My earlier post on Bayesian probability seems to have generated quite a lot of readers, so this lunchtime I thought I’d add a little bit"} +{"kind":"pair","text":"What is the capital of France?","pair":"Paris is the capital of France.","ids":[363,19,8,1784,13,1410,58,1,1919,19,8,1784,13,1410,5,1],"tokens":["▁What","▁is","▁the","▁capital","▁of","▁France","?","","▁Paris","▁is","▁the","▁capital","▁of","▁France",".",""],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"sequence_ids":[0,0,0,0,0,0,0,null,1,1,1,1,1,1,1,null],"special_tokens_mask":[0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1],"offsets":[[0,4],[5,7],[8,11],[12,19],[20,22],[23,29],[29,30],[0,0],[0,5],[6,8],[9,12],[13,20],[21,23],[24,30],[30,31],[0,0]]} +{"kind":"pair","text":"Question in English?","pair":"Ответ на русском языке, с цифрами 123.","ids":[11860,16,1566,58,1,3,2,6725,6609,15042,3,8194,3,23912,5345,29577,6469,3,2,6652,1757,6,12681,3,2,2795,2,7948,21325,3,14574,5,1],"tokens":["▁Question","▁in","▁English","?","","▁","О","т","в","ет","▁","на","▁","ру","с","ско","м","▁","язы","к","е",",","▁с","▁","ц","и","ф","ра","ми","▁","123",".",""],"type_ids":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"sequence_ids":[0,0,0,0,null,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,null],"special_tokens_mask":[0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1],"offsets":[[0,8],[9,11],[12,19],[19,20],[0,0],[0,1],[0,1],[1,2],[2,3],[3,5],[6,7],[6,8],[9,10],[9,11],[11,12],[12,15],[15,16],[17,18],[17,20],[20,21],[21,22],[22,23],[24,25],[26,27],[26,27],[27,28],[28,29],[29,31],[31,33],[34,35],[34,37],[37,38],[0,0]]} +{"kind":"digest","file":"fixtures/lang/amh_Ethi.txt","cap_chars":200000,"text_sha256":"e353d32f45b449b76fb65ee48f1522086c02a6bffa607e73ed8c875c03c84908","n_tokens":82493,"ids_sha256":"d13873d1cd0501623a44c03d887a31e7fcbbb555bd442366a10406c9964e13b0"} +{"kind":"digest","file":"fixtures/lang/arb_Arab.txt","cap_chars":200000,"text_sha256":"529587117db0a7abaa2c8f31ab05d7b2f77e84a11068b46e26b2b37774d836c7","n_tokens":69793,"ids_sha256":"afc57e014c9a888289a5eb27ce821ed4d2063cff69565ed6749c2ad61378996a"} +{"kind":"digest","file":"fixtures/lang/ben_Beng.txt","cap_chars":200000,"text_sha256":"915afd7dbbea8256bdb6112b16424c83aa47dae63c5cca0ebadfc6c227d2b89f","n_tokens":63952,"ids_sha256":"6516377138b9cebdbcdc9f116a7390250ec5d64c2724db630c5d3e3e0c730355"} +{"kind":"digest","file":"fixtures/lang/cmn_Hani.txt","cap_chars":200000,"text_sha256":"1273c0c093a00739be9efea7804b10a295f902359d11663306d097211934c6df","n_tokens":44433,"ids_sha256":"6962c2fc925beeb0ef444dc34bc7e54105e430fa4fe5874cbb9d34c53fe6ef90"} +{"kind":"digest","file":"fixtures/lang/ell_Grek.txt","cap_chars":200000,"text_sha256":"158c8d3e4933091099bf12a1e21f57c7b7ff3f5381d4b25901eb1fa7ca0926f4","n_tokens":66033,"ids_sha256":"a816598af39c8d71b3a23ffce4d6dd2d88f55c1cd0e6283368a535de882516c9"} +{"kind":"digest","file":"fixtures/lang/eng_Latn.txt","cap_chars":200000,"text_sha256":"d998cb3153e3a56c2ecdf49293062e8bc6ca43365b6c91cfed147d73b84c19fb","n_tokens":49203,"ids_sha256":"cddc56c0b082ffc01ee6ab245e69f7129de4a12e690fd58192768dda633632b8"} +{"kind":"digest","file":"fixtures/lang/heb_Hebr.txt","cap_chars":200000,"text_sha256":"386da8c0e0d416bffbe84b51baf895c67dbce6e0fd153be597933261794300fd","n_tokens":77255,"ids_sha256":"4f42d654518fca9de70d3f668be30de53b7bb0ab78a01a5e00af04461f506607"} +{"kind":"digest","file":"fixtures/lang/hin_Deva.txt","cap_chars":200000,"text_sha256":"2d114148c81035b1b63c94c7e9e805933c215d8aa9500d916923aec0a124ddde","n_tokens":80569,"ids_sha256":"aa228f10676ce3106995f9c2054d10a55977638162bf3f02004f1cc3177b7b4b"} +{"kind":"digest","file":"fixtures/lang/jpn_Jpan.txt","cap_chars":200000,"text_sha256":"2e55c036f500d7720d020be2db45c87a49c17d3b7e63b30bada9cef4cb6aa158","n_tokens":24775,"ids_sha256":"4971a66e5d2fdc4e90a3a530b9c75678c307930070cf4f1ae43f60f6c2f41eea"} +{"kind":"digest","file":"fixtures/lang/kat_Geor.txt","cap_chars":200000,"text_sha256":"c69a1aa8fb80b41459c98c15c756eb8f39a9151468b1d5b3f9bbffe4d82eb87a","n_tokens":54826,"ids_sha256":"f966ea789e3806e252bc5139243200de114114f4237bd32476b8f19c38be9069"} +{"kind":"digest","file":"fixtures/lang/kor_Hang.txt","cap_chars":200000,"text_sha256":"68659781c288136c1caffcb10eec9130406ae8a7b003c1e327b7e3601eba041d","n_tokens":99251,"ids_sha256":"1751166eb8b4eb1cb8ff326725b91f8368f8e6ac2d9aa32e63a98e0dceb74192"} +{"kind":"digest","file":"fixtures/lang/rus_Cyrl.txt","cap_chars":200000,"text_sha256":"5f967a82627afbd72c2f21c6e56183dfa5efc986ba34350c1cf9eea6c27a6132","n_tokens":143499,"ids_sha256":"417b79ec0778adb7c0a8dcbb852d433795acb0f398f09677d23fac4fd4e8bfa4"} +{"kind":"digest","file":"fixtures/lang/tam_Taml.txt","cap_chars":200000,"text_sha256":"bf09e1bca55aeab34162d5dc61afec7ac976b4e66eb5833ac5a037305911d165","n_tokens":48385,"ids_sha256":"2e52d0e09d08a4a3cb2482e98b1fb87dffa126e1a401bf4d948e9e6ca34e54f5"} +{"kind":"digest","file":"fixtures/lang/tha_Thai.txt","cap_chars":200000,"text_sha256":"25209f4750b3d3f04440f98aeff50316a118cb982462a4763106e8747172b79c","n_tokens":19104,"ids_sha256":"5c56e51e1cc2b846bd9cd996bed90ac07b4bcc7deb2ed92d59d0518078b04e77"} +{"kind":"digest","file":"fixtures/modalities/added_normalized_dense.txt","cap_chars":200000,"text_sha256":"9d49b3cacf40962c051a09f1350a0f5dc2fc210556889ed0f341cfe0cf0af4f9","n_tokens":35615,"ids_sha256":"d56a7ce88d7c52d3b2f081e3491c9c895f72adc22609f7118c51a6d877176414"} +{"kind":"digest","file":"fixtures/modalities/added_normalized_sparse.txt","cap_chars":200000,"text_sha256":"55383f52b02a5d9686f9e4347729e5c08e61b9cff77913bd94ceb65c2fdf7fe4","n_tokens":26035,"ids_sha256":"bed3e3f9f3767e0fae549258e6ea1897e5c404b8666a122586efe9d23d778c52"} +{"kind":"digest","file":"fixtures/modalities/added_special_dense.txt","cap_chars":200000,"text_sha256":"472f90f6f2f5803626df5d7088f8ef12984de6e88fedf67723887511d35b64b2","n_tokens":67672,"ids_sha256":"2badf79626fca1ceaf5198fe2410ba6d04f5b057b3af9870923399c9a1216657"} +{"kind":"digest","file":"fixtures/modalities/added_special_sparse.txt","cap_chars":200000,"text_sha256":"9c52602f7c8d009b11377743f95e5bfe9053694fc6b48c3dd6c1c6dc0681ab5a","n_tokens":37360,"ids_sha256":"e9875f6e9dc3635fa1a5e9cca99a77f1557e871336164578aac03f88eae777c1"} +{"kind":"digest","file":"fixtures/modalities/agentic-traces.txt","cap_chars":200000,"text_sha256":"ae9d63c1adc49f572d9af635ac1b0421461e4d1b92c5f35ea572980ff1e2480c","n_tokens":69510,"ids_sha256":"d8949740ca398dab7082142b642660cf6e6671382151e0577c03d399004ddf1e"} +{"kind":"digest","file":"fixtures/modalities/agentic_swe.txt","cap_chars":200000,"text_sha256":"b2d31e91ff91bb6c9a3aec3832d25acbecf9b58d5c0674e364d3b287182e7253","n_tokens":83279,"ids_sha256":"7af3ea6a443545176c4ed722971ff7b609385aeaa9b6939a6f102d1356831c06"} +{"kind":"digest","file":"fixtures/modalities/code_mixed.txt","cap_chars":200000,"text_sha256":"0fa7314727726db056433d36bc1bda021803be7f1d150cae83e362a3ff41821d","n_tokens":86720,"ids_sha256":"f5e352623dad5d433f1fa6f74e158f00542d077650668b81ff8cee846c215f84"} +{"kind":"digest","file":"fixtures/modalities/math_latex.txt","cap_chars":200000,"text_sha256":"2d17be8704f1f7b4f2174a054c0b85d6d22039e00be8e4a487927d27f3852e90","n_tokens":58276,"ids_sha256":"fac7e13ff194a494f764069f1aa26e893bf19f5b8d1441f507506b130ae4e03e"} diff --git a/bindings/python/tests/golden/test_golden.py b/bindings/python/tests/golden/test_golden.py new file mode 100644 index 000000000..adccf9864 --- /dev/null +++ b/bindings/python/tests/golden/test_golden.py @@ -0,0 +1,133 @@ +"""Replay the golden inputs on the current build and diff every output. + +One test per output domain, parametrized per model, so the failure report +reads as a conformance matrix: which field diverges (or raises) on which +tokenizer archetype. A mismatch means this build disagrees with the released +wheel the goldens were generated from — never edit a golden to make it pass; +regenerate them from the release (`make golden-regen`) or fix the build. +""" + +import json +from functools import cache + +import pytest +from tokenizers import Tokenizer + +from .generate import DATA, GOLDENS, REPO, ids_digest, text_digest + +FETCH_HINT = f"run `make -C {REPO / 'tokenizers'} bench-models fixtures`" + + +@cache +def records(name: str) -> list[dict]: + return [json.loads(line) for line in (GOLDENS / f"{name}.jsonl").read_text().splitlines()] + + +@cache +def tokenizer(name: str) -> Tokenizer: + meta = records(name)[0] + path = DATA / meta["tokenizer_file"] + if not path.is_file(): + pytest.skip(f"{path} missing — {FETCH_HINT}") + return Tokenizer.from_file(str(path)) + + +def golden_models() -> list[str]: + names = sorted(path.stem for path in GOLDENS.glob("*.jsonl")) + assert names, f"no golden files in {GOLDENS} — run `make golden-regen` and commit the result" + return names + + +@pytest.fixture(params=golden_models()) +def model(request): + return request.param + + +def samples(name: str) -> list[dict]: + return [r for r in records(name) if r["kind"] == "sample"] + + +def test_ids(model): + tok = tokenizer(model) + for s in samples(model): + assert tok.encode(s["text"]).ids == s["ids"], s["source"] + assert tok.encode(s["text"], add_special_tokens=False).ids == s["ids_no_specials"], ( + f"{s['source']} (no specials)" + ) + + +def test_tokens(model): + tok = tokenizer(model) + for s in samples(model): + assert tok.encode(s["text"]).tokens == s["tokens"], s["source"] + + +def test_offsets(model): + tok = tokenizer(model) + for s in samples(model): + assert [list(span) for span in tok.encode(s["text"]).offsets] == s["offsets"], s["source"] + + +def test_word_ids(model): + tok = tokenizer(model) + for s in samples(model): + assert tok.encode(s["text"]).word_ids == s["word_ids"], s["source"] + + +def test_type_ids(model): + tok = tokenizer(model) + for s in samples(model): + assert tok.encode(s["text"]).type_ids == s["type_ids"], s["source"] + + +def test_special_tokens_mask(model): + tok = tokenizer(model) + for s in samples(model): + assert tok.encode(s["text"]).special_tokens_mask == s["special_tokens_mask"], s["source"] + + +def test_attention_mask(model): + # Not recorded in the goldens: for a single unpadded sequence it is all + # ones by definition. + tok = tokenizer(model) + for s in samples(model): + enc = tok.encode(s["text"]) + assert enc.attention_mask == [1] * len(enc.ids), s["source"] + + +def test_decode(model): + tok = tokenizer(model) + for s in samples(model): + assert tok.decode(s["ids"], skip_special_tokens=True) == s["decoded"], s["source"] + assert tok.decode(s["ids"], skip_special_tokens=False) == s["decoded_with_specials"], ( + f"{s['source']} (specials)" + ) + + +def test_pairs(model): + tok = tokenizer(model) + for p in (r for r in records(model) if r["kind"] == "pair"): + enc = tok.encode(p["text"], p["pair"]) + assert enc.ids == p["ids"], p["text"] + assert enc.tokens == p["tokens"], p["text"] + assert enc.type_ids == p["type_ids"], p["text"] + assert enc.sequence_ids == p["sequence_ids"], p["text"] + assert enc.special_tokens_mask == p["special_tokens_mask"], p["text"] + assert [list(span) for span in enc.offsets] == p["offsets"], p["text"] + + +def test_fixture_digests(model): + # Breadth: the full (capped) fixture corpora, compared as an ids digest. + # A mismatch says the encoder diverges somewhere in that file; re-run + # generate.py side by side to find where. + tok = tokenizer(model) + for d in (r for r in records(model) if r["kind"] == "digest"): + path = DATA / d["file"] + if not path.is_file(): + pytest.skip(f"{path} missing — {FETCH_HINT}") + text = path.read_text()[: d["cap_chars"]] + if text_digest(text) != d["text_sha256"]: + pytest.skip(f"{d['file']} differs from the copy the goldens were generated from") + ids = tok.encode(text).ids + assert len(ids) == d["n_tokens"], d["file"] + assert ids_digest(ids) == d["ids_sha256"], d["file"] diff --git a/bindings/python/tests/test_async.py b/bindings/python/tests/test_async.py index f9a1d9445..88bb29fae 100644 --- a/bindings/python/tests/test_async.py +++ b/bindings/python/tests/test_async.py @@ -2,7 +2,6 @@ import numpy as np import pytest - from tokenizers import Encoding, EncodingBatch from .conftest import SENTENCES, train_word_tokenizer diff --git a/bindings/python/tests/test_components.py b/bindings/python/tests/test_components.py index aa723c2d7..ede9db378 100644 --- a/bindings/python/tests/test_components.py +++ b/bindings/python/tests/test_components.py @@ -1,6 +1,5 @@ import numpy as np import pytest - from tokenizers import Tokenizer, models, normalizers, pre_tokenizers, trainers from .conftest import train_word_tokenizer @@ -93,5 +92,5 @@ def test_component_assignment_invalidates_pipeline(): def test_split_rejects_bad_behavior(): - with pytest.raises(Exception): + with pytest.raises(ValueError): pre_tokenizers.Split(" ", behavior="not-a-behavior") diff --git a/bindings/python/tests/test_encoding.py b/bindings/python/tests/test_encoding.py index 3a5355db2..8f9dfcfae 100644 --- a/bindings/python/tests/test_encoding.py +++ b/bindings/python/tests/test_encoding.py @@ -1,6 +1,5 @@ import numpy as np import pytest - from tokenizers import Encoding, EncodingBatch, Tokenizer, models, pre_tokenizers, trainers from .conftest import SENTENCES, train_word_tokenizer diff --git a/bindings/python/tests/test_parity_trainer.py b/bindings/python/tests/test_parity_trainer.py index ba612b741..bad193e53 100644 --- a/bindings/python/tests/test_parity_trainer.py +++ b/bindings/python/tests/test_parity_trainer.py @@ -1,5 +1,4 @@ import pytest - from tokenizers import Tokenizer, models, pre_tokenizers, trainers EN = ["the cat sat on the mat", "the dog ate the food", "a cat and a dog"] * 6 diff --git a/bindings/python/tests/test_pretrained.py b/bindings/python/tests/test_pretrained.py index 73dc024bb..a9b43bc83 100644 --- a/bindings/python/tests/test_pretrained.py +++ b/bindings/python/tests/test_pretrained.py @@ -1,6 +1,5 @@ import numpy as np import pytest - from tokenizers import Tokenizer, TokenizersError diff --git a/bindings/python/tests/test_tokenizer.py b/bindings/python/tests/test_tokenizer.py index 0b2f9586e..54e76fcba 100644 --- a/bindings/python/tests/test_tokenizer.py +++ b/bindings/python/tests/test_tokenizer.py @@ -2,7 +2,6 @@ import numpy as np import pytest - from tokenizers import AddedToken, Tokenizer, models from .conftest import SENTENCES, train_word_tokenizer diff --git a/bindings/python/tests/test_trainers.py b/bindings/python/tests/test_trainers.py index 2c123cfb7..894373479 100644 --- a/bindings/python/tests/test_trainers.py +++ b/bindings/python/tests/test_trainers.py @@ -1,5 +1,4 @@ import pytest - from tokenizers import Tokenizer, models, pre_tokenizers, trainers from .conftest import SENTENCES diff --git a/tokenizers/tk-encode/src/tokenizer/pipeline.rs b/tokenizers/tk-encode/src/tokenizer/pipeline.rs index 42a6e7dbc..f0b8bc52e 100644 --- a/tokenizers/tk-encode/src/tokenizer/pipeline.rs +++ b/tokenizers/tk-encode/src/tokenizer/pipeline.rs @@ -158,15 +158,16 @@ impl TryFrom for PipelinePreTokenizer { /// Processors that don't reduce to such a frame are rejected at conversion. /// /// Example: -/// PipelinePostProcessor { -/// prefix: vec![100].into_boxed_slice(), -/// suffix: vec![101, 102].into_boxed_slice() -/// }; -/// -/// [CLS] The quick Brown fox [SEP] -/// <100>| <3> <4> <19> <67> | <101> <102> -/// prefix | sequence encoding | suffix +/// ```text +/// PipelinePostProcessor { +/// prefix: vec![100].into_boxed_slice(), +/// suffix: vec![101, 102].into_boxed_slice() +/// }; /// +/// [CLS] The quick Brown fox [SEP] +/// <100>| <3> <4> <19> <67> | <101> <102> +/// prefix | sequence encoding | suffix +/// ``` #[derive(Debug, Default)] pub struct PipelinePostProcessor { prefix: Box<[PipelineToken]>,