From 930ed776f00bb4695e9899b213ddeb30eb22c0eb Mon Sep 17 00:00:00 2001
From: SBrandeis <33657802+SBrandeis@users.noreply.github.com>
Date: Mon, 20 Jul 2026 17:17:13 +0200
Subject: [PATCH 1/3] bench(pipeline): add sebpop/upstream as a second
 reference series
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Bench the tokenizers crate from sebpop's performance branch
(github.com/sebpop/tokenizers#upstream) alongside the released 0.23.1
in the pipeline benchmark: single-thread throughput per fixture, the
multi-thread sweep, per-implementation memory footprint, minimal-binary
size, and a report-only id-diff flag. The charts keep a single vs-release
axis — sebpop renders as a second (green) bar next to the pipeline's.

The git dep uses sebpop's default features minus mimalloc: its override
feature would swap the global allocator for the whole bench process and
skew every series. Cargo.lock pins the resolved commit;
`cargo update tokenizers@0.22.3-dev.0` moves it to the branch tip.

Older cached bench JSONs (base-branch baselines) without the sebpop keys
still render unchanged.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 .github/scripts/render_pipeline_bench.py      | 311 +++++++++++++-----
 .github/workflows/pipeline-bench.yml          |  23 +-
 tokenizers/Cargo.lock                         |  34 ++
 tokenizers/tk-encode/Cargo.toml               |  12 +-
 .../tk-encode/examples/binsize_sebpop.rs      |  17 +
 .../tk-encode/examples/fixture_bench.rs       | 127 +++++--
 6 files changed, 400 insertions(+), 124 deletions(-)
 create mode 100644 tokenizers/tk-encode/examples/binsize_sebpop.rs

diff --git a/.github/scripts/render_pipeline_bench.py b/.github/scripts/render_pipeline_bench.py
index be91dd907..23b91ebb1 100644
--- a/.github/scripts/render_pipeline_bench.py
+++ b/.github/scripts/render_pipeline_bench.py
@@ -6,28 +6,33 @@
 tk-encode/bench-baseline --example fixture_bench`:
 
     {baseline: {crate, version},
+     sebpop: {crate, ref},
      models: [{model, shape, desc, [reason],
-               memory: {baseline|pipeline:
+               memory: {baseline|sebpop|pipeline:
                         {load_bytes, encode_bytes, peak_bytes} | null} | null,
                threads: {counts: [1,2,4,8,max],
-                         pipeline_mbps: [..], baseline_mbps: [..|null]},
+                         pipeline_mbps: [..], baseline_mbps: [..|null],
+                         sebpop_mbps: [..|null]},
                results: [{fixture, group, bytes, chunks,
-                          mbps: {baseline, pipeline},
-                          ids_match, ids_match_baseline,
+                          mbps: {baseline, sebpop, pipeline},
+                          ids_match, ids_match_baseline, ids_match_sebpop,
                           stage_ns_per_byte: {added_split, normalize,
                                               pre_tokenize, model, total},
                           pretok_vs_regex: {cls_simd, cls_scalar,
                                             onig|null, fancy|null,
                                             pcre2|null, logos|null}}]}]}
 
-Two series: `baseline` — the latest released tokenizers crate, the bar to beat
-(the in-tree Tokenizer is on its way out, so it isn't benched; it only serves
-as the id oracle behind `ids_match`) — drawn gray as context, and `pipeline` —
-the experimental PipelineTokenizer, blue. The report leads with three
+Three series: `baseline` — the latest released tokenizers crate, the bar to
+beat (the in-tree Tokenizer is on its way out, so it isn't benched; it only
+serves as the id oracle behind `ids_match`) — drawn gray as context; `sebpop`
+— the tokenizers crate from sebpop's performance branch, green; and `pipeline`
+— the experimental PipelineTokenizer, blue. The `sebpop` keys are optional
+everywhere so older cached JSONs still render. The report leads with
 always-visible charts:
 
   1. throughput overview — per model, geomean ×speedup of the pipeline against
-     the release (×1.0 = release), with a min–max whisker across fixtures (no
+     the release (×1.0 = release), with a min–max whisker across fixtures, plus
+     a second (green) bar for sebpop's branch on the same vs-release axis (no
      cross-model aggregate: the models exercise different execution modes, so
      averaging them means nothing);
   2. memory overview — per model, resident-set delta after load plus the encode
@@ -68,10 +73,11 @@
 }
 # Series identity: the release baseline is context-gray on purpose — in the
 # speedup charts it *is* the ×1.0 axis, in memory/binary-size it's the
-# reference bar the pipeline is read against.
+# reference bar the pipeline is read against. sebpop's branch is the second
+# reference, green.
 SERIES_INK = {
-    "light": {"baseline": "#898781", "pipeline": "#2a78d6"},
-    "dark": {"baseline": "#898781", "pipeline": "#3987e5"},
+    "light": {"baseline": "#898781", "sebpop": "#2f9e44", "pipeline": "#2a78d6"},
+    "dark": {"baseline": "#898781", "sebpop": "#57c464", "pipeline": "#3987e5"},
 }
 FONT = "-apple-system,'Segoe UI',Helvetica,Arial,sans-serif"
 GUTTER, PLOT_W, PAD_R, COL_W, ROW_H, BAR_H = 190, 540, 110, 150, 26, 16
@@ -144,6 +150,27 @@ def model_speedups(model):
     return [v for v in (speedup(r) for r in model["results"]) if v]
 
 
+def sebpop_speedup(row):
+    """Pipeline throughput ÷ sebpop-branch throughput for the same fixture."""
+    s, p = row["mbps"].get("sebpop"), row["mbps"]["pipeline"]
+    return p / s if s and p else None
+
+
+def sebpop_model_speedups(model):
+    return [v for v in (sebpop_speedup(r) for r in model["results"]) if v]
+
+
+def sebpop_rel_release(row):
+    """sebpop-branch throughput ÷ release throughput — sebpop's own speedup on
+    the same vs-release axis the pipeline bars are drawn on."""
+    b, s = row["mbps"]["baseline"], row["mbps"].get("sebpop")
+    return s / b if b and s else None
+
+
+def sebpop_rel_model_speedups(model):
+    return [v for v in (sebpop_rel_release(r) for r in model["results"]) if v]
+
+
 def base_speedup(row, base_lookup, model_name):
     """This PR's pipeline throughput ÷ the base branch's pipeline throughput for the
     same (model, fixture) — the "did this PR help vs the base branch" ratio. `None`
@@ -242,7 +269,8 @@ def speedup_axis(ink, x, ticks, top, bottom):
 
 def overview_svg(models, mode, subtitle_base, meta, lo, hi, baseline_label,
                  speedups_of=model_speedups, title=None, ref_label=None,
-                 mark_regressions=False, no_cmp_msg=None):
+                 mark_regressions=False, no_cmp_msg=None,
+                 speedups2_of=None, series2_label=None):
     """The headline chart: a row per manifest model — name + workload desc,
     geomean ×speedup of the pipeline vs the reference (×1.0) with a min–max
     whisker across fixtures. No cross-model aggregate on purpose: the models
@@ -251,7 +279,9 @@ def overview_svg(models, mode, subtitle_base, meta, lo, hi, baseline_label,
 
     Defaults draw "vs latest release". Pass `speedups_of=base_model_speedups`-style
     with `mark_regressions=True` for the "vs base branch" twin: bars for models that
-    got slower than the base branch turn red."""
+    got slower than the base branch turn red. Pass `speedups2_of`/`series2_label`
+    to add a second series (sebpop's branch, green) as a second bar per model on
+    the same vs-reference axis."""
     ink, sink = INK[mode], SERIES_INK[mode]
     title = title or "PipelineTokenizer vs latest release — encode throughput"
     ref_label = ref_label or baseline_label
@@ -261,6 +291,24 @@ def x(v):
 
     ticks = thin_ticks([t for t in TICKS if lo <= t <= hi], x, min_px=34, keep=1.0)
 
+    def series_bar(vals, cy, bar_h, color, label_fill, label_size, label_weight):
+        """One geomean bar + min–max whisker + ×label, centered on `cy`."""
+        g, mn, mx = geomean(vals), min(vals), max(vals)
+        parts = [hbar(x(1.0), x(g), cy - bar_h / 2, bar_h, color),
+                 f'<line x1="{x(mn):.1f}" y1="{cy:.1f}" x2="{x(mx):.1f}" y2="{cy:.1f}" '
+                 f'stroke="{ink["secondary"]}" stroke-width="1.5"/>']
+        for v in (mn, mx):
+            parts.append(f'<line x1="{x(v):.1f}" y1="{cy - 4:.1f}" x2="{x(v):.1f}" y2="{cy + 4:.1f}" '
+                         f'stroke="{ink["secondary"]}" stroke-width="1.5"/>')
+        anchor, lx = (("start", max(x(mx), x(1.0)) + 8) if g >= 1
+                      else ("end", min(x(mn), x(1.0)) - 8))
+        if anchor == "end" and lx - 40 < OV_GUTTER + 4:
+            anchor, lx = "start", max(x(mx), x(1.0)) + 8
+        parts.append(f'<text x="{lx:.1f}" y="{cy + 4:.1f}" fill="{label_fill}" '
+                     f'font-size="{label_size}" font-weight="{label_weight}" text-anchor="{anchor}" '
+                     f'style="font-variant-numeric:tabular-nums">×{g:.2f}</text>')
+        return parts
+
     top, row_h = 74, 40
     col_x = CHART_W - 16
     body = [f'<text x="{col_x}" y="{top - 14}" fill="{ink["muted"]}" font-size="11" '
@@ -274,22 +322,16 @@ def x(v):
         body.append(f'<text x="{OV_GUTTER - 14}" y="{cy + 11:.1f}" fill="{ink["muted"]}" '
                     f'font-size="10.5" text-anchor="end">{escape(desc)}</text>')
         vals = speedups_of(m)
+        vals2 = speedups2_of(m) if speedups2_of else None
         if vals:
-            g, mn, mx = geomean(vals), min(vals), max(vals)
+            g = geomean(vals)
             bar_color = ink["critical"] if (mark_regressions and g < 1) else sink["pipeline"]
-            body.append(hbar(x(1.0), x(g), cy - 7, 14, bar_color))
-            body.append(f'<line x1="{x(mn):.1f}" y1="{cy:.1f}" x2="{x(mx):.1f}" y2="{cy:.1f}" '
-                        f'stroke="{ink["secondary"]}" stroke-width="1.5"/>')
-            for v in (mn, mx):
-                body.append(f'<line x1="{x(v):.1f}" y1="{cy - 4:.1f}" x2="{x(v):.1f}" y2="{cy + 4:.1f}" '
-                            f'stroke="{ink["secondary"]}" stroke-width="1.5"/>')
-            anchor, lx = (("start", max(x(mx), x(1.0)) + 8) if g >= 1
-                          else ("end", min(x(mn), x(1.0)) - 8))
-            if anchor == "end" and lx - 40 < OV_GUTTER + 4:
-                anchor, lx = "start", max(x(mx), x(1.0)) + 8
-            body.append(f'<text x="{lx:.1f}" y="{cy + 4:.1f}" fill="{ink["primary"]}" font-size="12" '
-                        f'font-weight="600" text-anchor="{anchor}" '
-                        f'style="font-variant-numeric:tabular-nums">×{g:.2f}</text>')
+            if vals2:
+                # two bars on the same axis: pipeline on top, the second series below
+                body += series_bar(vals, cy - 7, 11, bar_color, ink["primary"], 11, 600)
+                body += series_bar(vals2, cy + 7, 11, sink["sebpop"], ink["secondary"], 10.5, 400)
+            else:
+                body += series_bar(vals, cy, 14, bar_color, ink["primary"], 12, 600)
             bad = sum(1 for r in m["results"] if r["ids_match"] is False)
             right, fill = ((f"⚠ {bad} differ", ink["critical"]) if bad
                            else (f'{len(m["results"])} · ids ok', ink["secondary"]))
@@ -310,21 +352,23 @@ def x(v):
 
     axis = speedup_axis(ink, x, ticks, top, y + 4)
     y += 30
-    legend = legend_row(ink, sink, y, [
-        ("swatch", "pipeline", "PipelineTokenizer"),
-        ("tick", ink["baseline"], f"×1.0 = {ref_label}"),
-    ])
+    entries = [("swatch", "pipeline", "PipelineTokenizer")]
+    if speedups2_of:
+        entries.append(("swatch", "sebpop", series2_label))
+    entries.append(("tick", ink["baseline"], f"×1.0 = {ref_label}"))
+    legend = legend_row(ink, sink, y, entries)
     height = y + 34
     subtitle = (f"geomean ×speedup per model vs {ref_label} · "
                 f"whisker: min–max across fixtures · {subtitle_base}")
     return svg_doc(ink, height, title, subtitle, axis + "".join(body) + legend, meta)
 
 
-def memory_svg(models, mode, meta, baseline_label):
+def memory_svg(models, mode, meta, baseline_label, sebpop_label=None):
     """Per model: resident-set delta of each implementation — load footprint plus
     the encode-pass delta as stacked segments, peak RSS as a tick."""
     ink, sink = INK[mode], SERIES_INK[mode]
     models = [m for m in models if isinstance(m.get("memory"), dict)]
+    impls = ("baseline", "sebpop", "pipeline") if sebpop_label else ("baseline", "pipeline")
 
     def mem(m, impl):
         d = m["memory"].get(impl)
@@ -335,7 +379,7 @@ def mem(m, impl):
 
     vals = []
     for m in models:
-        for impl in ("baseline", "pipeline"):
+        for impl in impls:
             d = mem(m, impl)
             if d:
                 vals.append((d["load_bytes"] or 0) + (d["encode_bytes"] or 0))
@@ -349,10 +393,14 @@ def mem(m, impl):
     def x(v):
         return OV_GUTTER + v / max_mb * plot_w
 
-    top, bar_h, row_h = 78, 12, 2 * (12 + 3) + 16
+    top, bar_h = 78, 12
+    row_h = len(impls) * (bar_h + 3) + 16
     col_x = CHART_W - 16
+    col_head = "MB: " + " → ".join(
+        {"baseline": baseline_label, "sebpop": sebpop_label, "pipeline": "Pipeline"}[i]
+        for i in impls)
     body = [f'<text x="{col_x}" y="{top - 14}" fill="{ink["muted"]}" font-size="11" '
-            f'text-anchor="end">MB: {escape(baseline_label)} → Pipeline</text>',
+            f'text-anchor="end">{escape(col_head)}</text>',
             f'<text x="{OV_GUTTER}" y="{top - 14}" fill="{ink["muted"]}" font-size="11">'
             f'smaller is better · solid: after load · translucent: encode-pass delta</text>']
     y = top
@@ -362,7 +410,7 @@ def x(v):
                     f'font-size="12.5" font-weight="600" text-anchor="end">{escape(m["model"])}</text>')
         totals = []
         by = y + 8
-        for impl in ("baseline", "pipeline"):
+        for impl in impls:
             d = mem(m, impl)
             if not d:
                 totals.append(None)
@@ -395,20 +443,23 @@ def x(v):
         grid.append(f'<text x="{x(tv):.1f}" y="{y + 10}" fill="{ink["muted"]}" font-size="11" '
                     f'text-anchor="middle" style="font-variant-numeric:tabular-nums">{tv:g}{unit}</text>')
     y += 30
-    legend = legend_row(ink, sink, y, [
-        ("swatch", "baseline", baseline_label),
-        ("swatch", "pipeline", "PipelineTokenizer"),
-        ("tick", ink["primary"], "peak RSS (VmHWM)"),
-    ])
+    entries = [("swatch", "baseline", baseline_label)]
+    if sebpop_label:
+        entries.append(("swatch", "sebpop", sebpop_label))
+    entries += [("swatch", "pipeline", "PipelineTokenizer"),
+                ("tick", ink["primary"], "peak RSS (VmHWM)")]
+    legend = legend_row(ink, sink, y, entries)
     height = y + 34
     subtitle = "resident-set delta per implementation, one process each · load + encode pass"
     return svg_doc(ink, height, "Memory footprint",
                    subtitle, "".join(grid) + "".join(body) + legend, meta)
 
 
-def chart_svg(model, mode, subtitle_base, meta, lo, hi, baseline_label):
-    """Full-size per-fixture chart: the pipeline's ×speedup vs the release, with
-    the `MB/s: release → Pipeline` throughput column."""
+def chart_svg(model, mode, subtitle_base, meta, lo, hi, baseline_label,
+              sebpop_label=None):
+    """Full-size per-fixture chart: the pipeline's ×speedup vs the release —
+    with sebpop's branch as a second bar on the same vs-release axis — and the
+    `MB/s: release → sebpop → Pipeline` throughput column."""
     ink, sink = INK[mode], SERIES_INK[mode]
     rows = model["results"]
 
@@ -419,10 +470,13 @@ def x(v):
 
     top = 74
     col_x = GUTTER + PLOT_W + PAD_R + COL_W - 16
+    col_head = (f"MB/s: {baseline_label} → {sebpop_label} → Pipeline" if sebpop_label
+                else f"MB/s: {baseline_label} → Pipeline")
     body = [f'<text x="{col_x}" y="{top - 14}" fill="{ink["muted"]}" font-size="11" '
-            f'text-anchor="end">MB/s: {escape(baseline_label)} → Pipeline</text>']
+            f'text-anchor="end">{escape(col_head)}</text>']
     y = top
     baseline_id_note = False
+    sebpop_id_note = False
     for key, title in GROUPS:
         # stable order (alphabetical by fixture) so a fixture keeps its row across
         # runs and lines up with the stage-decomposition chart — not sorted by the
@@ -439,38 +493,52 @@ def x(v):
             if r.get("ids_match_baseline") is False:
                 label += " †"
                 baseline_id_note = True
+            if r.get("ids_match_sebpop") is False:
+                label += " ‡"
+                sebpop_id_note = True
             body.append(f'<text x="{GUTTER - 10}" y="{y + ROW_H / 2 + 4}" fill="{ink["secondary"]}" '
                         f'font-size="12.5" text-anchor="end">{escape(label)}</text>')
             v = speedup(r)
-            by = y + (ROW_H - BAR_H) / 2
-            if v:
-                body.append(hbar(x(1.0), x(v), by, BAR_H, sink["pipeline"]))
-                txt = f"×{v:.2f}"
-                fill = ink["primary"]
-                if r["ids_match"] is False:
+            sv = sebpop_rel_release(r) if sebpop_label else None
+            # one full-height pipeline bar, or — when sebpop benched this fixture —
+            # two half-height bars on the same vs-release axis (pipeline on top)
+            series = ([("pipeline", v, y + (ROW_H - BAR_H) / 2, BAR_H, 12)] if not sv else
+                      [("pipeline", v, y + (ROW_H - 20) / 2, 9, 10.5),
+                       ("sebpop", sv, y + (ROW_H - 20) / 2 + 11, 9, 10.5)])
+            for skey, val, by, bh, fs in series:
+                if not val:
+                    continue
+                body.append(hbar(x(1.0), x(val), by, bh, sink[skey]))
+                txt = f"×{val:.2f}"
+                fill = ink["primary"] if skey == "pipeline" else ink["secondary"]
+                if skey == "pipeline" and r["ids_match"] is False:
                     txt += "  ⚠ ids differ"
                     fill = ink["critical"]
-                anchor, lx = ("start", max(x(1.0), x(v)) + 6) if v >= 1 else ("end", min(x(1.0), x(v)) - 6)
+                anchor, lx = (("start", max(x(1.0), x(val)) + 6) if val >= 1
+                              else ("end", min(x(1.0), x(val)) - 6))
                 # a long label left of a slow bar would run into the fixture
                 # names — flip it to the empty space right of the ×1.0 axis
                 if anchor == "end" and lx - len(txt) * 6.7 < GUTTER + 4:
                     anchor, lx = "start", x(1.0) + 6
-                body.append(f'<text x="{lx:.1f}" y="{y + ROW_H / 2 + 4}" fill="{fill}" '
-                            f'font-size="12" font-weight="600" text-anchor="{anchor}" '
+                body.append(f'<text x="{lx:.1f}" y="{by + bh / 2 + 4:.1f}" fill="{fill}" '
+                            f'font-size="{fs}" font-weight="600" text-anchor="{anchor}" '
                             f'style="font-variant-numeric:tabular-nums">{txt}</text>')
             mb = r["mbps"]
+            col_vals = ([mb.get("baseline"), mb.get("sebpop"), mb.get("pipeline")] if sebpop_label
+                        else [mb.get("baseline"), mb.get("pipeline")])
             body.append(f'<text x="{col_x}" y="{y + ROW_H / 2 + 4}" fill="{ink["secondary"]}" '
                         f'font-size="12" text-anchor="end" style="font-variant-numeric:tabular-nums">'
-                        f'{chain([mb.get("baseline"), mb.get("pipeline")])}</text>')
+                        f'{chain(col_vals)}</text>')
             y += ROW_H
         y += 10
 
     axis = speedup_axis(ink, x, ticks, top, y)
     y += 26
-    legend = legend_row(ink, sink, y, [
-        ("swatch", "pipeline", "PipelineTokenizer"),
-        ("tick", ink["baseline"], f"×1.0 = {baseline_label}"),
-    ])
+    entries = [("swatch", "pipeline", "PipelineTokenizer")]
+    if sebpop_label:
+        entries.append(("swatch", "sebpop", sebpop_label))
+    entries.append(("tick", ink["baseline"], f"×1.0 = {baseline_label}"))
+    legend = legend_row(ink, sink, y, entries)
     height = y + 44
 
     parts = [model["shape"]]
@@ -479,8 +547,13 @@ def x(v):
         parts.append(f"geomean ×{geomean(vals):.2f} vs {baseline_label}")
     else:
         parts.append(f"{baseline_label} can’t load this model — no comparison")
+    svals = sebpop_model_speedups(model)
+    if svals:
+        parts.append(f"×{geomean(svals):.2f} vs {sebpop_label}")
     if baseline_id_note:
         parts.append(f"† ids differ from {baseline_label}")
+    if sebpop_id_note:
+        parts.append(f"‡ ids differ from {sebpop_label}")
     return svg_doc(ink, height, f'{model["model"]} — PipelineTokenizer encode throughput',
                    " · ".join(parts), axis + "".join(body) + legend, meta, subtitle_base)
 
@@ -650,7 +723,7 @@ def picture(base, run_id, slug, alt, width):
         "</picture>"])
 
 
-def mem_line(model, baseline_label):
+def mem_line(model, baseline_label, sebpop_label=None):
     mem = model["memory"]
     def part(impl, label):
         d = mem.get(impl)
@@ -658,17 +731,23 @@ def part(impl, label):
             return f"{label} —"
         cell = lambda k: ("—" if d.get(k) is None else f"{max(0, d[k]) / 1e6:.0f}")
         return f"{label} {cell('load_bytes')}+{cell('encode_bytes')} (peak {cell('peak_bytes')})"
+    impls = [("baseline", baseline_label)]
+    if sebpop_label:
+        impls.append(("sebpop", sebpop_label))
+    impls.append(("pipeline", "Pipeline"))
     return ("**Memory** (RSS MB, load+encode): "
-            + " · ".join(part(i, l) for i, l in
-                         (("baseline", baseline_label), ("pipeline", "Pipeline"))))
+            + " · ".join(part(i, l) for i, l in impls))
 
 
-def binsize_svg(sizes, mode, meta, baseline_label):
+def binsize_svg(sizes, mode, meta, baseline_label, sebpop_label=None):
     """Stripped size of a minimal release-built encode program (load a
     tokenizer.json, encode one string) linking each implementation — what the
     library adds to a shipped binary. Bars are 0-anchored on a linear MB axis."""
     ink, sink = INK[mode], SERIES_INK[mode]
-    rows = [("baseline", baseline_label), ("pipeline", "PipelineTokenizer")]
+    rows = [("baseline", baseline_label)]
+    if sebpop_label and "sebpop" in sizes:
+        rows.append(("sebpop", sebpop_label))
+    rows.append(("pipeline", "PipelineTokenizer"))
     max_mb = max(sizes.values()) / 1e6 * 1.15
     plot_w = CHART_W - OV_GUTTER - PAD_R - COL_W
 
@@ -713,17 +792,21 @@ def has_threads(m):
     return isinstance(t, dict) and bool(t.get("counts"))
 
 
-def threads_svg(model, mode, meta, baseline_label):
-    """Per model: encode throughput (MB/s) at 1/2/4/8/device-max threads — pipeline vs the release —
-    with a per-row *ideal linear* tick (single-thread throughput × N) on the pipeline bar. So whether the
-    pipeline scales linearly (bar reaches the tick) or sub-linearly (bar falls short of it) is visible at a
-    glance, alongside the pipeline↔release gap; the right column carries the self-scaling % of linear."""
+def threads_svg(model, mode, meta, baseline_label, sebpop_label=None):
+    """Per model: encode throughput (MB/s) at 1/2/4/8/device-max threads — pipeline vs the release (and
+    sebpop's branch when present) — with a per-row *ideal linear* tick (single-thread throughput × N) on
+    the pipeline bar. So whether the pipeline scales linearly (bar reaches the tick) or sub-linearly (bar
+    falls short of it) is visible at a glance, alongside the pipeline↔reference gaps; the right column
+    carries the self-scaling % of linear."""
     ink, sink = INK[mode], SERIES_INK[mode]
     t = model["threads"]
     counts, pipe, base = t["counts"], t["pipeline_mbps"], t["baseline_mbps"]
+    seb = t.get("sebpop_mbps") or []
+    has_seb = bool(sebpop_label) and any(v for v in seb)
     p1 = pipe[0] if pipe and pipe[0] else None  # single-thread anchor for the linear reference
     ideal = [p1 * n for n in counts] if p1 else []
-    vals = [v for v in pipe if v] + [v for v in base if v] + ideal
+    vals = ([v for v in pipe if v] + [v for v in base if v]
+            + [v for v in seb if v] + ideal)
     max_mb = (max(vals) if vals else 1.0) * 1.08
     plot_w = CHART_W - OV_GUTTER - PAD_R - COL_W
 
@@ -731,22 +814,30 @@ def x(v):
         return OV_GUTTER + v / max_mb * plot_w
 
     top, bar_h, gap = 78, 11, 3
-    row_h = 2 * (bar_h + gap) + 16
+    nbars = 3 if has_seb else 2
+    row_h = nbars * (bar_h + gap) + 16
     col_x = CHART_W - 16
+    order = (f"bars: {baseline_label} → {sebpop_label} → Pipeline" if has_seb
+             else f"top bar {baseline_label}, bottom Pipeline")
     body = [f'<text x="{col_x}" y="{top - 14}" fill="{ink["muted"]}" font-size="11" '
             f'text-anchor="end">Pipeline MB/s · self-scaling (% of linear)</text>',
             f'<text x="{OV_GUTTER}" y="{top - 14}" fill="{ink["muted"]}" font-size="11">'
-            f'higher is better · top bar {escape(baseline_label)}, bottom Pipeline · tick = ideal linear</text>']
+            f'higher is better · {escape(order)} · tick = ideal linear</text>']
     y = top
     for i, n in enumerate(counts):
         cy = y + row_h / 2
         label = f"{n} thread" + ("" if n == 1 else "s")
         body.append(f'<text x="{OV_GUTTER - 14}" y="{cy + 4:.1f}" fill="{ink["primary"]}" '
                     f'font-size="12.5" font-weight="600" text-anchor="end">{label}</text>')
-        base_y, pipe_y = y + 8, y + 8 + bar_h + gap
+        base_y = y + 8
+        pipe_y = base_y + (nbars - 1) * (bar_h + gap)
         b = base[i] if i < len(base) else None
         if b is not None:
             body.append(hbar(x(0), x(b), base_y, bar_h, sink["baseline"]))
+        if has_seb:
+            s = seb[i] if i < len(seb) else None
+            if s is not None:
+                body.append(hbar(x(0), x(s), base_y + bar_h + gap, bar_h, sink["sebpop"]))
         p = pipe[i] if i < len(pipe) else None
         if p is not None:
             body.append(hbar(x(0), x(p), pipe_y, bar_h, sink["pipeline"]))
@@ -774,17 +865,19 @@ def x(v):
         grid.append(f'<text x="{gx:.1f}" y="{y + 10}" fill="{ink["muted"]}" font-size="11" '
                     f'text-anchor="middle" style="font-variant-numeric:tabular-nums">{tv:g}{unit}</text>')
     y += 30
-    legend = legend_row(ink, sink, y, [
-        ("swatch", "baseline", baseline_label),
-        ("swatch", "pipeline", "PipelineTokenizer"),
-        ("tick", ink["primary"], "ideal linear (T₁ × N)"),
-    ])
+    entries = [("swatch", "baseline", baseline_label)]
+    if has_seb:
+        entries.append(("swatch", "sebpop", sebpop_label))
+    entries += [("swatch", "pipeline", "PipelineTokenizer"),
+                ("tick", ink["primary"], "ideal linear (T₁ × N)")]
+    legend = legend_row(ink, sink, y, entries)
     height = y + 34
     scaling = ""
     if p1 and len(pipe) >= 2 and pipe[-1]:
         sc = pipe[-1] / p1
         scaling = f" · pipeline {sc:.1f}× on {counts[-1]} threads ({sc / counts[-1] * 100:.0f}% of linear)"
-    subtitle = f"throughput at N threads vs {baseline_label}; tick = perfect linear scaling{scaling}"
+    refs = f"{baseline_label} + {sebpop_label}" if has_seb else baseline_label
+    subtitle = f"throughput at N threads vs {refs}; tick = perfect linear scaling{scaling}"
     return svg_doc(ink, height, "Thread scaling",
                    subtitle, "".join(grid) + "".join(body) + legend, meta)
 
@@ -836,16 +929,22 @@ def render_markdown(data, subtitle_base, meta, base, run_id, sizes,
     different execution modes, so only per-model geomeans are meaningful."""
     models = data["models"]
     baseline_label = f'v{data["baseline"]["version"]}'
+    sebpop_label = data.get("sebpop", {}).get("ref")
     benched = [m for m in models if m["results"]]
     unsupported = [m for m in models if not m["results"]]
     mismatches = [f"{m['model']}/{r['fixture']}"
                   for m in benched for r in m["results"] if r["ids_match"] is False]
     base_mismatch = sorted({m["model"] for m in benched
                             for r in m["results"] if r.get("ids_match_baseline") is False})
+    seb_mismatch = sorted({m["model"] for m in benched
+                           for r in m["results"] if r.get("ids_match_sebpop") is False})
 
+    refs = f"`tokenizers` {baseline_label} (latest release)"
+    if sebpop_label:
+        refs += f" and `{sebpop_label}`"
     md = ["## PipelineTokenizer benchmark", "",
           f"**{len(benched)} / {len(models)} models supported** — PipelineTokenizer vs "
-          f"`tokenizers` {baseline_label} (latest release) · {subtitle_base}", "",
+          f"{refs} · {subtitle_base}", "",
           f"`{meta[0]}` · {meta[1]}", "",
           picture(base, run_id, "overview", "Per-model encode throughput vs latest release", 860), ""]
     if base_lookup:
@@ -863,6 +962,9 @@ def render_markdown(data, subtitle_base, meta, base, run_id, sizes,
         md += [f"> ℹ️ Token ids differ from {baseline_label} on: {', '.join(base_mismatch)} "
                f"(† in the per-model charts) — expected when this branch fixes encode bugs, "
                f"but worth a look.", ""]
+    if seb_mismatch:
+        md += [f"> ℹ️ Token ids differ from {sebpop_label} on: {', '.join(seb_mismatch)} "
+               f"(‡ in the per-model charts).", ""]
 
     for m in benched:
         slug = slugify(m["model"])
@@ -870,6 +972,9 @@ def render_markdown(data, subtitle_base, meta, base, run_id, sizes,
         vals = model_speedups(m)
         summary = (f"×{geomean(vals):.2f} vs {baseline_label}" if vals
                    else f"{baseline_label} can't load — no comparison")
+        svals = sebpop_model_speedups(m)
+        if svals:
+            summary += f" · ×{geomean(svals):.2f} vs {sebpop_label}"
         if base_lookup:
             bvals = base_model_speedups(m, base_lookup)
             if bvals:
@@ -884,15 +989,19 @@ def render_markdown(data, subtitle_base, meta, base, run_id, sizes,
         if has_threads(m):
             md += [picture(base, run_id, f"{slug}-threads",
                            f"{m['model']} thread scaling", 860), ""]
-        md += [mem_line(m, baseline_label), ""]
+        md += [mem_line(m, baseline_label, sebpop_label), ""]
         # Per-stage columns show each split's share of the pipeline's own encode time
         # with the ns/byte alongside — `share% (ns/B)` — so the split cost is readable
         # as text regardless of how slow the release baseline is.
+        seb_mb_col = f" {sebpop_label} MB/s |" if sebpop_label else ""
+        seb_vs_col = f" vs {sebpop_label} |" if sebpop_label else ""
+        seb_sep = "---:|" if sebpop_label else ""
         base_col = " Δ base |" if base_lookup else ""
         base_sep = "---:|" if base_lookup else ""
-        md += [f"| Fixture | Group | {baseline_label} MB/s | Pipeline MB/s | Speedup |{base_col} "
+        md += [f"| Fixture | Group | {baseline_label} MB/s |{seb_mb_col} Pipeline MB/s "
+               f"| vs {baseline_label} |{seb_vs_col}{base_col} "
                "added-token | normalize | pre-tokenize | model | Ids |",
-               f"|---|---|---:|---:|---:|{base_sep}---:|---:|---:|---:|:--|"]
+               f"|---|---|---:|{seb_sep}---:|---:|{seb_sep}{base_sep}---:|---:|---:|---:|:--|"]
         for r in sorted(m["results"], key=lambda r: (r["group"], r["fixture"])):
             mb = r["mbps"]
             flags = []
@@ -900,17 +1009,22 @@ def render_markdown(data, subtitle_base, meta, base, run_id, sizes,
                 flags.append("⚠️ ≠ tree")
             if r.get("ids_match_baseline") is False:
                 flags.append(f"≠ {baseline_label}")
+            if r.get("ids_match_sebpop") is False:
+                flags.append(f"≠ {sebpop_label}")
             ids = " · ".join(flags) if flags else "match"
             s = r.get("stage_ns_per_byte")
             stages = " ".join(f"| {stage_cell(s, k)}"
                               for k in ("added_split", "normalize", "pre_tokenize", "model"))
+            seb_mb_cell = f"| {fnum(mb.get('sebpop'))} " if sebpop_label else ""
+            seb_vs_cell = (f"| {fnum(sebpop_speedup(r), '×{:.2f}')} "
+                           if sebpop_label else "")
             base_cell = (f"| {fnum(base_speedup(r, base_lookup, m['model']), '×{:.2f}')} "
                          if base_lookup else "")
             md.append(
                 f"| {r['fixture']} | {r['group']} "
-                f"| {fnum(mb.get('baseline'))} | {fnum(mb.get('pipeline'))} "
+                f"| {fnum(mb.get('baseline'))} {seb_mb_cell}| {fnum(mb.get('pipeline'))} "
                 f"| {fnum(speedup(r), '×{:.2f}')} "
-                f"{base_cell}{stages} | {ids} |")
+                f"{seb_vs_cell}{base_cell}{stages} | {ids} |")
         md += pretok_compare_md(m)
         md += ["", "</details>", ""]
 
@@ -959,9 +1073,15 @@ def main():
     data = json.loads(Path(args.results).read_text())
     models = data["models"]
     baseline_label = f'v{data["baseline"]["version"]}'
+    sebpop_label = data.get("sebpop", {}).get("ref")
     sizes = json.loads(Path(args.binary_sizes).read_text()) if args.binary_sizes else None
     benched = [m for m in models if m["results"]]
     lo, hi = scale(benched)
+    if sebpop_label:
+        # sebpop's bars share the vs-release axis with the pipeline's — widen
+        # the range so neither series falls off the plot
+        slo, shi = scale(benched, sebpop_rel_model_speedups)
+        lo, hi = min(lo, slo), max(hi, shi)
 
     # "vs base branch": join the PR results against the base branch's cached run by
     # (model, fixture). base_lookup[(model, fixture)] = base pipeline MB/s.
@@ -979,7 +1099,9 @@ def main():
     out = Path(args.out_dir)
     for mode in ("light", "dark"):
         (out / f"pipeline_bench_overview_{mode}.svg").write_text(
-            overview_svg(models, mode, args.subtitle, meta, lo, hi, baseline_label))
+            overview_svg(models, mode, args.subtitle, meta, lo, hi, baseline_label,
+                         speedups2_of=(sebpop_rel_model_speedups if sebpop_label else None),
+                         series2_label=sebpop_label))
         if base_lookup:
             (out / f"pipeline_bench_base-overview_{mode}.svg").write_text(
                 overview_svg(models, mode, args.subtitle, meta, blo, bhi, baseline_label,
@@ -988,14 +1110,15 @@ def main():
                              ref_label=(args.base_ref or "base branch"), mark_regressions=True,
                              no_cmp_msg="not benched on the base branch"))
         (out / f"pipeline_bench_memory_{mode}.svg").write_text(
-            memory_svg(models, mode, meta, baseline_label))
+            memory_svg(models, mode, meta, baseline_label, sebpop_label))
         if sizes:
             (out / f"pipeline_bench_binsize_{mode}.svg").write_text(
-                binsize_svg(sizes, mode, meta, baseline_label))
+                binsize_svg(sizes, mode, meta, baseline_label, sebpop_label))
     for m in models:
         slug = slugify(m["model"])
         for mode in ("light", "dark"):
-            svg = (chart_svg(m, mode, args.subtitle, meta, lo, hi, baseline_label)
+            svg = (chart_svg(m, mode, args.subtitle, meta, lo, hi, baseline_label,
+                             sebpop_label)
                    if m["results"] else card_svg(m, mode))
             (out / f"pipeline_bench_{slug}_{mode}.svg").write_text(svg)
             if has_stages(m):
@@ -1003,7 +1126,7 @@ def main():
                     stage_chart_svg(m, mode, args.subtitle, meta, baseline_label))
             if has_threads(m):
                 (out / f"pipeline_bench_{slug}-threads_{mode}.svg").write_text(
-                    threads_svg(m, mode, meta, baseline_label))
+                    threads_svg(m, mode, meta, baseline_label, sebpop_label))
 
     (out / "pipeline_bench.md").write_text(
         render_markdown(data, args.subtitle, meta, args.img_base, args.run_id, sizes,
@@ -1016,6 +1139,12 @@ def main():
         for m in benched if model_speedups(m))
     print(f"{len(benched)}/{len(models)} supported"
           + (f" · vs {baseline_label}: {per_model}" if per_model else ""))
+    if sebpop_label:
+        vs_seb = " · ".join(
+            f"{m['model']} x{geomean(sebpop_model_speedups(m)):.3f}"
+            for m in benched if sebpop_model_speedups(m))
+        print(f"vs {sebpop_label}: {vs_seb}" if vs_seb
+              else f"vs {sebpop_label}: no comparable fixtures")
     if base_lookup:
         vs_base = " · ".join(
             f"{m['model']} x{geomean(base_speedups_of(m)):.3f}"
diff --git a/.github/workflows/pipeline-bench.yml b/.github/workflows/pipeline-bench.yml
index e62718544..8480e88ad 100644
--- a/.github/workflows/pipeline-bench.yml
+++ b/.github/workflows/pipeline-bench.yml
@@ -1,14 +1,15 @@
 name: Pipeline Benchmark
 
-# Comparative benchmark: the experimental `PipelineTokenizer` vs the latest
-# *released* tokenizers crate (the baseline to beat — the in-tree legacy
-# `Tokenizer` is being phased out, so it is only the id-correctness oracle,
-# never a benched series), for every model in
+# Comparative benchmark: the experimental `PipelineTokenizer` vs two references
+# — the latest *released* tokenizers crate (the baseline to beat — the in-tree
+# legacy `Tokenizer` is being phased out, so it is only the id-correctness
+# oracle, never a benched series) and sebpop's performance branch
+# (github.com/sebpop/tokenizers#upstream) — for every model in
 # tk-encode/examples/bench_models.json across every corpus in data/fixtures/.
 # Measures single- and multi-thread throughput (1/2/4/8/device-max), per-
-# implementation memory footprint (RSS), and stripped minimal-binary size. The
-# release dep is behind tk-encode's `bench-baseline` feature, so production
-# builds never pull it.
+# implementation memory footprint (RSS), and stripped minimal-binary size. Both
+# reference deps are behind tk-encode's `bench-baseline` feature, so production
+# builds never pull them.
 #
 # The work is fanned out across a matrix: each `bench` shard benches a slice of
 # the models on its own isolated 8-vCPU runner (so the multi-thread sweep is
@@ -180,7 +181,9 @@ jobs:
           for f in parts:
               d = json.load(open(f))
               if merged is None:
-                  merged = {"baseline": d["baseline"], "models": []}
+                  # carry every metadata key (baseline, sebpop, …) from the first shard
+                  merged = {k: v for k, v in d.items() if k != "models"}
+                  merged["models"] = []
               merged["models"].extend(d["models"])
           if merged is None:
               raise SystemExit("no shard partials found")
@@ -194,10 +197,10 @@ jobs:
       - name: Measure binary sizes
         run: |
           cargo build --release -p tk-encode --features tk-encode/bench-baseline \
-            --example binsize_baseline --example binsize_pipeline
+            --example binsize_baseline --example binsize_sebpop --example binsize_pipeline
           printf '{' > binary_sizes.json
           sep=''
-          for key in baseline pipeline; do
+          for key in baseline sebpop pipeline; do
             bin="target/release/examples/binsize_$key"
             "$bin" data/gpt2.json "The quick brown fox jumps 123."   # binary actually works
             strip -o "$bin.stripped" "$bin"
diff --git a/tokenizers/Cargo.lock b/tokenizers/Cargo.lock
index 4e5a39e84..a4b582ad2 100644
--- a/tokenizers/Cargo.lock
+++ b/tokenizers/Cargo.lock
@@ -2300,6 +2300,7 @@ dependencies = [
  "spm_precompiled",
  "tempfile",
  "thiserror",
+ "tokenizers 0.22.3-dev.0",
  "tokenizers 0.23.1",
  "tracing",
  "tracing-subscriber",
@@ -2331,6 +2332,39 @@ dependencies = [
  "tk-encode",
 ]
 
+[[package]]
+name = "tokenizers"
+version = "0.22.3-dev.0"
+source = "git+https://github.com/sebpop/tokenizers?branch=upstream#13699f626ffa586ec7068f4559ca42d677c49bfc"
+dependencies = [
+ "ahash",
+ "aho-corasick",
+ "compact_str",
+ "dary_heap",
+ "derive_builder",
+ "esaxx-rs",
+ "getrandom 0.3.4",
+ "indicatif 0.18.5",
+ "itertools 0.14.0",
+ "log",
+ "macro_rules_attribute",
+ "monostate",
+ "paste",
+ "pcre2-sys",
+ "rand 0.9.4",
+ "rayon",
+ "rayon-cond",
+ "regex",
+ "regex-syntax",
+ "serde",
+ "serde_json",
+ "spm_precompiled",
+ "thiserror",
+ "unicode-normalization-alignments",
+ "unicode-segmentation",
+ "unicode_categories",
+]
+
 [[package]]
 name = "tokenizers"
 version = "0.23.1"
diff --git a/tokenizers/tk-encode/Cargo.toml b/tokenizers/tk-encode/Cargo.toml
index a946c1e1b..2c698fc5b 100644
--- a/tokenizers/tk-encode/Cargo.toml
+++ b/tokenizers/tk-encode/Cargo.toml
@@ -63,6 +63,12 @@ yada = "0.7.0"
 # Latest released tokenizers, used as the comparison baseline by the CI benchmark
 # (examples gated on `bench-baseline`). Optional so production builds never pull it.
 tokenizers-release = { package = "tokenizers", version = "=0.23.1", optional = true }
+# sebpop's performance branch, the second comparison reference. Cargo.lock pins the
+# resolved commit — `cargo update tokenizers@0.22.3-dev.0` moves it to the branch tip.
+# His default features minus `mimalloc`: its `override` feature replaces the global
+# allocator for the whole process, which would skew every series benched in the same
+# binary.
+tokenizers-sebpop = { package = "tokenizers", git = "https://github.com/sebpop/tokenizers", branch = "upstream", default-features = false, features = ["progressbar", "pcre2", "esaxx_fast"], optional = true }
 # Reference regex engines `fixture_bench` times the classify+fsm pre-tokenize against: Oniguruma and
 # PCRE2 (both C) alongside the pure-Rust `fancy-regex` (optional, above). All three are pulled in by
 # `bench-baseline` — optional + behind that feature so the C builds happen ONLY for the CI benchmark,
@@ -81,7 +87,7 @@ progressbar = ["indicatif"]
 http = ["hf-hub"]
 unstable_wasm = ["fancy-regex", "getrandom/wasm_js"]
 rustls-tls = ["hf-hub?/rustls-tls"]
-bench-baseline = ["dep:tokenizers-release", "dep:onig", "dep:pcre2", "dep:logos", "fancy-regex"]
+bench-baseline = ["dep:tokenizers-release", "dep:tokenizers-sebpop", "dep:onig", "dep:pcre2", "dep:logos", "fancy-regex"]
 
 [dev-dependencies]
 criterion = "0.6"
@@ -101,3 +107,7 @@ required-features = ["bench-baseline"]
 [[example]]
 name = "binsize_baseline"
 required-features = ["bench-baseline"]
+
+[[example]]
+name = "binsize_sebpop"
+required-features = ["bench-baseline"]
diff --git a/tokenizers/tk-encode/examples/binsize_sebpop.rs b/tokenizers/tk-encode/examples/binsize_sebpop.rs
new file mode 100644
index 000000000..0137e1851
--- /dev/null
+++ b/tokenizers/tk-encode/examples/binsize_sebpop.rs
@@ -0,0 +1,17 @@
+//! Minimal encode program measured by CI for binary size: the `tokenizers`
+//! crate from sebpop's performance branch (the second comparison reference).
+//! Structurally identical to `binsize_pipeline.rs`.
+
+use tokenizers_sebpop::Tokenizer;
+
+fn main() {
+    let mut args = std::env::args().skip(1);
+    let path = args
+        .next()
+        .expect("usage: binsize_sebpop <tokenizer.json> <text>");
+    let text = args
+        .next()
+        .expect("usage: binsize_sebpop <tokenizer.json> <text>");
+    let tok = Tokenizer::from_file(path).unwrap();
+    println!("{}", tok.encode(text.as_str(), false).unwrap().len());
+}
diff --git a/tokenizers/tk-encode/examples/fixture_bench.rs b/tokenizers/tk-encode/examples/fixture_bench.rs
index 0dcde13c2..282ae5ff6 100644
--- a/tokenizers/tk-encode/examples/fixture_bench.rs
+++ b/tokenizers/tk-encode/examples/fixture_bench.rs
@@ -1,7 +1,9 @@
-//! Comparative benchmark of the experimental `PipelineTokenizer` against the
-//! latest *released* `tokenizers` crate (the bar to beat — the in-tree legacy
-//! `Tokenizer` is on its way out, so the release is the reference), for every
-//! model in `examples/bench_models.json` across every corpus in `data/fixtures/`.
+//! Comparative benchmark of the experimental `PipelineTokenizer` against two
+//! references — the latest *released* `tokenizers` crate (the bar to beat — the
+//! in-tree legacy `Tokenizer` is on its way out, so the release is the
+//! reference) and sebpop's performance branch (`sebpop/tokenizers#upstream`) —
+//! for every model in `examples/bench_models.json` across every corpus in
+//! `data/fixtures/`.
 //!
 //! Per fixture it measures single-thread throughput on ~10 kB inputs (the regime
 //! where per-input overhead is amortized — see `pipeline_benchmark.rs` for the
@@ -21,7 +23,7 @@
 //! one-line label of the workload archetype the model exercises, passed
 //! through to the report.
 //!
-//! Emits one JSON object (`{baseline, models}`) on stdout, consumed by
+//! Emits one JSON object (`{baseline, sebpop, models}`) on stdout, consumed by
 //! `.github/scripts/render_pipeline_bench.py` in CI.
 
 use std::convert::TryFrom;
@@ -37,11 +39,14 @@ use serde_json::{json, Value};
 use tk_encode::pipeline::{Model, PipelineTokenizer};
 use tk_encode::{AddedToken, ModelWrapper, Tokenizer};
 use tokenizers_release::{AddedToken as BaselineAddedToken, Tokenizer as BaselineTokenizer};
+use tokenizers_sebpop::{AddedToken as SebpopAddedToken, Tokenizer as SebpopTokenizer};
 
 const DATA_DIR: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../data");
 const MANIFEST: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/examples/bench_models.json");
 // Keep in sync with the `tokenizers-release` pin in Cargo.toml.
 const BASELINE_VERSION: &str = "0.23.1";
+// Keep in sync with the `tokenizers-sebpop` git dependency in Cargo.toml.
+const SEBPOP_REF: &str = "sebpop/upstream";
 const CHUNK_BYTES: usize = 10 * 1024;
 const MAX_CHUNKS: usize = 100;
 const REPS: usize = 5;
@@ -93,6 +98,21 @@ fn inject_added_tokens_baseline(tok: &mut BaselineTokenizer) {
     let _ = tok.add_tokens(normalized);
 }
 
+/// Same injection through the sebpop branch's `AddedToken` (its pre-split API
+/// takes slices, not iterators).
+fn inject_added_tokens_sebpop(tok: &mut SebpopTokenizer) {
+    let special: Vec<SebpopAddedToken> = ADDED_SPECIAL
+        .iter()
+        .map(|s| SebpopAddedToken::from(*s, true).normalized(false))
+        .collect();
+    let normalized: Vec<SebpopAddedToken> = ADDED_NORMALIZED
+        .iter()
+        .map(|s| SebpopAddedToken::from(*s, false).normalized(true))
+        .collect();
+    let _ = tok.add_special_tokens(&special);
+    let _ = tok.add_tokens(&normalized);
+}
+
 fn make_chunks(text: &str) -> Vec<String> {
     let mut chunks = Vec::new();
     let mut cur = String::new();
@@ -152,19 +172,22 @@ fn par_mbps(
     bytes as f64 / median_secs(samples) / 1e6
 }
 
-/// Multi-thread throughput sweep for one model — pipeline vs the released crate at 1/2/4/8/max threads
-/// over the whole fixture corpus (thread-spawn/scheduling overhead amortized). Both impls encode the same
-/// chunk list through a fresh pool per count; interleaved so thermal drift hits them equally.
+/// Multi-thread throughput sweep for one model — pipeline vs the released crate vs sebpop's branch at
+/// 1/2/4/8/max threads over the whole fixture corpus (thread-spawn/scheduling overhead amortized). All
+/// impls encode the same chunk list through a fresh pool per count; interleaved so thermal drift hits
+/// them equally.
 fn bench_threads(
     baseline: Option<&BaselineTokenizer>,
+    sebpop: Option<&SebpopTokenizer>,
     pipeline: &PipelineTokenizer,
     chunks: &[String],
 ) -> Value {
     let bytes: usize = chunks.iter().map(String::len).sum();
     let counts = thread_counts();
-    let (mut pipe, mut base) = (Vec::new(), Vec::new());
+    let (mut pipe, mut base, mut seb) = (Vec::new(), Vec::new(), Vec::new());
     for &n in &counts {
         let b = baseline.map(|b| par_mbps(|s| b.encode(s, false).unwrap().len(), chunks, bytes, n));
+        let sp = sebpop.map(|t| par_mbps(|s| t.encode(s, false).unwrap().len(), chunks, bytes, n));
         let p = par_mbps(
             |s| pipeline.encode(s, false).unwrap().len(),
             chunks,
@@ -172,13 +195,15 @@ fn bench_threads(
             n,
         );
         eprintln!(
-            "    {n} thread(s): pipeline {p:.1} MB/s{}",
-            b.map_or(String::new(), |v| format!(", baseline {v:.1} MB/s"))
+            "    {n} thread(s): pipeline {p:.1} MB/s{}{}",
+            b.map_or(String::new(), |v| format!(", baseline {v:.1} MB/s")),
+            sp.map_or(String::new(), |v| format!(", sebpop {v:.1} MB/s"))
         );
         pipe.push(p);
         base.push(b);
+        seb.push(sp);
     }
-    json!({ "counts": counts, "pipeline_mbps": pipe, "baseline_mbps": base })
+    json!({ "counts": counts, "pipeline_mbps": pipe, "baseline_mbps": base, "sebpop_mbps": seb })
 }
 
 fn time_pass(encode: &dyn Fn(&str) -> usize, chunks: &[String]) -> f64 {
@@ -340,6 +365,15 @@ fn memory_child(which: &str, model: &Path) {
             }
             (after_load, rss_now().unwrap_or(0))
         }
+        "sebpop" => {
+            let mut tok = SebpopTokenizer::from_file(model).unwrap();
+            inject_added_tokens_sebpop(&mut tok);
+            let after_load = rss_now().unwrap_or(0);
+            for c in &chunks {
+                n += tok.encode(c.as_str(), false).unwrap().len();
+            }
+            (after_load, rss_now().unwrap_or(0))
+        }
         "pipeline" => {
             let mut tok = Tokenizer::from_file(model).unwrap();
             inject_added_tokens(&mut tok);
@@ -370,10 +404,14 @@ fn memory_child(which: &str, model: &Path) {
 
 /// Re-run this binary once per available implementation to get per-impl memory
 /// numbers that a shared address space couldn't provide.
-fn measure_memory(model: &Path, baseline_ok: bool) -> Value {
+fn measure_memory(model: &Path, baseline_ok: bool, sebpop_ok: bool) -> Value {
     let exe = std::env::current_exe().unwrap();
     let mut out = serde_json::Map::new();
-    for (key, ok) in [("baseline", baseline_ok), ("pipeline", true)] {
+    for (key, ok) in [
+        ("baseline", baseline_ok),
+        ("sebpop", sebpop_ok),
+        ("pipeline", true),
+    ] {
         if !ok {
             out.insert(key.into(), Value::Null);
             continue;
@@ -667,6 +705,7 @@ fn logos_reference_ns(regexes: &[String], text: &str) -> Option<f64> {
 
 fn bench_model(
     baseline: Option<&BaselineTokenizer>,
+    sebpop: Option<&SebpopTokenizer>,
     oracle: &Tokenizer,
     pipeline: &PipelineTokenizer,
     files: &[(String, PathBuf)],
@@ -676,6 +715,7 @@ fn bench_model(
     // per-model: the reference regex(es) onig will time on each fixture (empty for non-regex pretoks).
     let regexes = pretok_regexes(model_json);
     let base_enc = baseline.map(|b| move |s: &str| b.encode(s, false).unwrap().len());
+    let seb_enc = sebpop.map(|t| move |s: &str| t.encode(s, false).unwrap().len());
 
     let mut rows = Vec::new();
     for (group, path) in files {
@@ -704,27 +744,42 @@ fn bench_model(
                 .take(3)
                 .all(|c| b.encode(c.as_str(), false).unwrap().get_ids() == pipe_ids(c))
         });
+        // Report-only: pipeline vs sebpop's branch.
+        let ids_match_sebpop = sebpop.map(|t| {
+            chunks
+                .iter()
+                .take(3)
+                .all(|c| t.encode(c.as_str(), false).unwrap().get_ids() == pipe_ids(c))
+        });
 
-        // interleave both impls so frequency/thermal drift hits them equally
+        // interleave all impls so frequency/thermal drift hits them equally
         if let Some(be) = &base_enc {
             time_pass(be, &chunks);
         }
+        if let Some(se) = &seb_enc {
+            time_pass(se, &chunks);
+        }
         time_pass(&pipe_enc, &chunks);
-        let (mut base_s, mut pipe_s) = (Vec::new(), Vec::new());
+        let (mut base_s, mut seb_s, mut pipe_s) = (Vec::new(), Vec::new(), Vec::new());
         for _ in 0..REPS {
             if let Some(be) = &base_enc {
                 base_s.push(time_pass(be, &chunks));
             }
+            if let Some(se) = &seb_enc {
+                seb_s.push(time_pass(se, &chunks));
+            }
             pipe_s.push(time_pass(&pipe_enc, &chunks));
         }
         let mbps = |secs: f64| bytes as f64 / secs / 1e6;
         let base_mbps = (!base_s.is_empty()).then(|| mbps(median_secs(base_s)));
+        let seb_mbps = (!seb_s.is_empty()).then(|| mbps(median_secs(seb_s)));
         let pipe_mbps = mbps(median_secs(pipe_s));
 
         let fmt = |v: Option<f64>| v.map_or("—".into(), |v| format!("{v:.1}"));
         eprintln!(
-            "  {name}: baseline {} MB/s, pipeline {pipe_mbps:.1} MB/s",
-            fmt(base_mbps)
+            "  {name}: baseline {} MB/s, sebpop {} MB/s, pipeline {pipe_mbps:.1} MB/s",
+            fmt(base_mbps),
+            fmt(seb_mbps)
         );
 
         // Staged decomposition of the pipeline's own encode via the ablation ladder:
@@ -788,9 +843,10 @@ fn bench_model(
             "group": group,
             "bytes": bytes,
             "chunks": chunks.len(),
-            "mbps": { "baseline": base_mbps, "pipeline": pipe_mbps },
+            "mbps": { "baseline": base_mbps, "sebpop": seb_mbps, "pipeline": pipe_mbps },
             "ids_match": ids_match,
             "ids_match_baseline": ids_match_baseline,
+            "ids_match_sebpop": ids_match_sebpop,
             "stage_ns_per_byte": {
                 "added_split": ns_added,
                 "normalize": ns_norm,
@@ -921,9 +977,35 @@ fn main() {
             }
         };
 
-        let rows = bench_model(baseline.as_ref(), &tok, &pipeline, &files, &path);
-        let memory = measure_memory(&path, baseline.is_some());
-        let threads = bench_threads(baseline.as_ref(), &pipeline, &all_chunks);
+        // Same fallback for sebpop's branch (based on an older tree, so it may
+        // predate configs the release handles).
+        let sebpop = match SebpopTokenizer::from_file(&path) {
+            Ok(mut t) => {
+                inject_added_tokens_sebpop(&mut t);
+                match t.encode(PROBE, false) {
+                    Ok(_) => Some(t),
+                    Err(e) => {
+                        eprintln!("  {SEBPOP_REF} loads but can't encode: {e}");
+                        None
+                    }
+                }
+            }
+            Err(e) => {
+                eprintln!("  {SEBPOP_REF} can't load this config: {e}");
+                None
+            }
+        };
+
+        let rows = bench_model(
+            baseline.as_ref(),
+            sebpop.as_ref(),
+            &tok,
+            &pipeline,
+            &files,
+            &path,
+        );
+        let memory = measure_memory(&path, baseline.is_some(), sebpop.is_some());
+        let threads = bench_threads(baseline.as_ref(), sebpop.as_ref(), &pipeline, &all_chunks);
 
         models.push(json!({
             "model": name, "repo": repo, "desc": desc, "shape": shape,
@@ -933,6 +1015,7 @@ fn main() {
 
     let out = json!({
         "baseline": { "crate": "tokenizers", "version": BASELINE_VERSION },
+        "sebpop": { "crate": "tokenizers", "ref": SEBPOP_REF },
         "models": models,
     });
     println!("{}", serde_json::to_string_pretty(&out).unwrap());

From 9f74cef1cade93718f588c49e6f30c0b2e47c9fe Mon Sep 17 00:00:00 2001
From: SBrandeis <33657802+SBrandeis@users.noreply.github.com>
Date: Mon, 20 Jul 2026 17:59:13 +0200
Subject: [PATCH 2/3] bench(pipeline): bench references via encode_fast, on
 mimalloc
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Plain `encode` tracks offsets and silently bypasses sebpop's fused
byte-level fast path (gated on OffsetType::None behind encode_fast /
encode_batch_fast — the API his blog numbers measure). Switch both
references to `encode_fast`, the same offsets-free regime the
PipelineTokenizer plays in, so all three series answer the same
question.

New `bench-mimalloc` feature installs mimalloc as the bench binary's
global allocator — the allocator sebpop's branch ships by default —
uniformly for all three series, since one process can't give each
series its own allocator. CI enables it for the bench shards only
(not binsize: that chart measures the library, not the allocator).

Local gpt2 check (M3 Max, 18 fixtures): ids match across all three
implementations, fused path included; sebpop jumps from ~x1.5 to
~x7.3 geomean vs v0.23.1, landing within ~15% of the pipeline.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 .github/workflows/pipeline-bench.yml          |  9 +++--
 tokenizers/Cargo.lock                         | 19 +++++++++++
 tokenizers/tk-encode/Cargo.toml               |  6 ++++
 .../tk-encode/examples/fixture_bench.rs       | 33 +++++++++++++------
 4 files changed, 55 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/pipeline-bench.yml b/.github/workflows/pipeline-bench.yml
index 8480e88ad..836cbcb21 100644
--- a/.github/workflows/pipeline-bench.yml
+++ b/.github/workflows/pipeline-bench.yml
@@ -114,9 +114,14 @@ jobs:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
         run: make fixtures bench-models HF="uvx --from huggingface_hub hf"
 
+      # bench-mimalloc: the whole bench binary (all three series uniformly) runs on
+      # mimalloc — the allocator sebpop's branch ships by default. The references are
+      # benched through `encode_fast` (see fixture_bench.rs), so sebpop's fused
+      # fast path is actually engaged.
       - name: Run comparative benchmark (shard ${{ matrix.shard }} of ${{ env.SHARDS }})
         run: |
-          cargo run --release -p tk-encode --features tk-encode/bench-baseline \
+          cargo run --release -p tk-encode \
+            --features tk-encode/bench-baseline,tk-encode/bench-mimalloc \
             --example fixture_bench -- --shard ${{ matrix.shard }} ${{ env.SHARDS }} \
             > "pipeline_bench_${{ matrix.shard }}.json"
           cat "pipeline_bench_${{ matrix.shard }}.json"
@@ -272,7 +277,7 @@ jobs:
           fi
           python3 ${{ github.workspace }}/.github/scripts/render_pipeline_bench.py \
             pipeline_bench.json \
-            --subtitle "~10 kB inputs · single thread + 1/2/4/8/max-thread sweep" \
+            --subtitle "~10 kB inputs · refs via encode_fast · mimalloc (all series) · single thread + 1/2/4/8/max-thread sweep" \
             --revision "${{ github.event.pull_request.head.sha || github.sha }}" \
             --img-base "$base_url" \
             --run-id "${{ github.run_id }}" \
diff --git a/tokenizers/Cargo.lock b/tokenizers/Cargo.lock
index a4b582ad2..2a6712def 100644
--- a/tokenizers/Cargo.lock
+++ b/tokenizers/Cargo.lock
@@ -1217,6 +1217,15 @@ dependencies = [
  "windows-link",
 ]
 
+[[package]]
+name = "libmimalloc-sys"
+version = "0.1.49"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "libredox"
 version = "0.1.17"
@@ -1328,6 +1337,15 @@ version = "2.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4"
 
+[[package]]
+name = "mimalloc"
+version = "0.1.52"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862"
+dependencies = [
+ "libmimalloc-sys",
+]
+
 [[package]]
 name = "minimal-lexical"
 version = "0.2.1"
@@ -2286,6 +2304,7 @@ dependencies = [
  "logos",
  "macro_rules_attribute",
  "memchr",
+ "mimalloc",
  "monostate",
  "onig",
  "paste",
diff --git a/tokenizers/tk-encode/Cargo.toml b/tokenizers/tk-encode/Cargo.toml
index 2c698fc5b..fb0c36716 100644
--- a/tokenizers/tk-encode/Cargo.toml
+++ b/tokenizers/tk-encode/Cargo.toml
@@ -76,6 +76,11 @@ tokenizers-sebpop = { package = "tokenizers", git = "https://github.com/sebpop/t
 onig = { version = "6.5.1", optional = true }
 pcre2 = { version = "0.2", optional = true }
 logos = { version = "0.15", optional = true } # compile-time DFA lexer reference (pure Rust)
+# Global allocator for the CI bench binary (`bench-mimalloc`): every series runs on
+# mimalloc — the allocator sebpop's branch ships by default — because one process can't
+# give each series its own allocator. No `override` feature: `#[global_allocator]` in
+# fixture_bench.rs is explicit and scoped to that binary.
+mimalloc = { version = "0.1", optional = true }
 
 [features]
 # `fancy-regex` is the OPTIONAL system-regex backend, needed ONLY for a `Split` pre-tokenizer with an
@@ -88,6 +93,7 @@ http = ["hf-hub"]
 unstable_wasm = ["fancy-regex", "getrandom/wasm_js"]
 rustls-tls = ["hf-hub?/rustls-tls"]
 bench-baseline = ["dep:tokenizers-release", "dep:tokenizers-sebpop", "dep:onig", "dep:pcre2", "dep:logos", "fancy-regex"]
+bench-mimalloc = ["dep:mimalloc"]
 
 [dev-dependencies]
 criterion = "0.6"
diff --git a/tokenizers/tk-encode/examples/fixture_bench.rs b/tokenizers/tk-encode/examples/fixture_bench.rs
index 282ae5ff6..f18c72fa8 100644
--- a/tokenizers/tk-encode/examples/fixture_bench.rs
+++ b/tokenizers/tk-encode/examples/fixture_bench.rs
@@ -5,6 +5,13 @@
 //! for every model in `examples/bench_models.json` across every corpus in
 //! `data/fixtures/`.
 //!
+//! Both references are benched through `encode_fast` (ids, no offset tracking)
+//! — the same regime the pipeline plays in, and on sebpop's branch the entry
+//! point of his fused byte-level fast path; plain `encode` would silently
+//! bypass it. With the `bench-mimalloc` feature the whole bench binary (all
+//! three series uniformly) runs on mimalloc, matching the allocator sebpop's
+//! branch ships by default — per-series allocators can't exist in one process.
+//!
 //! Per fixture it measures single-thread throughput on ~10 kB inputs (the regime
 //! where per-input overhead is amortized — see `pipeline_benchmark.rs` for the
 //! size sweep). Per model it also (a) runs a **multi-thread throughput sweep** —
@@ -41,6 +48,10 @@ use tk_encode::{AddedToken, ModelWrapper, Tokenizer};
 use tokenizers_release::{AddedToken as BaselineAddedToken, Tokenizer as BaselineTokenizer};
 use tokenizers_sebpop::{AddedToken as SebpopAddedToken, Tokenizer as SebpopTokenizer};
 
+#[cfg(feature = "bench-mimalloc")]
+#[global_allocator]
+static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
+
 const DATA_DIR: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../data");
 const MANIFEST: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/examples/bench_models.json");
 // Keep in sync with the `tokenizers-release` pin in Cargo.toml.
@@ -186,8 +197,10 @@ fn bench_threads(
     let counts = thread_counts();
     let (mut pipe, mut base, mut seb) = (Vec::new(), Vec::new(), Vec::new());
     for &n in &counts {
-        let b = baseline.map(|b| par_mbps(|s| b.encode(s, false).unwrap().len(), chunks, bytes, n));
-        let sp = sebpop.map(|t| par_mbps(|s| t.encode(s, false).unwrap().len(), chunks, bytes, n));
+        let b = baseline
+            .map(|b| par_mbps(|s| b.encode_fast(s, false).unwrap().len(), chunks, bytes, n));
+        let sp =
+            sebpop.map(|t| par_mbps(|s| t.encode_fast(s, false).unwrap().len(), chunks, bytes, n));
         let p = par_mbps(
             |s| pipeline.encode(s, false).unwrap().len(),
             chunks,
@@ -361,7 +374,7 @@ fn memory_child(which: &str, model: &Path) {
             inject_added_tokens_baseline(&mut tok);
             let after_load = rss_now().unwrap_or(0);
             for c in &chunks {
-                n += tok.encode(c.as_str(), false).unwrap().len();
+                n += tok.encode_fast(c.as_str(), false).unwrap().len();
             }
             (after_load, rss_now().unwrap_or(0))
         }
@@ -370,7 +383,7 @@ fn memory_child(which: &str, model: &Path) {
             inject_added_tokens_sebpop(&mut tok);
             let after_load = rss_now().unwrap_or(0);
             for c in &chunks {
-                n += tok.encode(c.as_str(), false).unwrap().len();
+                n += tok.encode_fast(c.as_str(), false).unwrap().len();
             }
             (after_load, rss_now().unwrap_or(0))
         }
@@ -714,8 +727,8 @@ fn bench_model(
     let pipe_enc = |s: &str| pipeline.encode(s, false).unwrap().len();
     // per-model: the reference regex(es) onig will time on each fixture (empty for non-regex pretoks).
     let regexes = pretok_regexes(model_json);
-    let base_enc = baseline.map(|b| move |s: &str| b.encode(s, false).unwrap().len());
-    let seb_enc = sebpop.map(|t| move |s: &str| t.encode(s, false).unwrap().len());
+    let base_enc = baseline.map(|b| move |s: &str| b.encode_fast(s, false).unwrap().len());
+    let seb_enc = sebpop.map(|t| move |s: &str| t.encode_fast(s, false).unwrap().len());
 
     let mut rows = Vec::new();
     for (group, path) in files {
@@ -742,14 +755,14 @@ fn bench_model(
             chunks
                 .iter()
                 .take(3)
-                .all(|c| b.encode(c.as_str(), false).unwrap().get_ids() == pipe_ids(c))
+                .all(|c| b.encode_fast(c.as_str(), false).unwrap().get_ids() == pipe_ids(c))
         });
         // Report-only: pipeline vs sebpop's branch.
         let ids_match_sebpop = sebpop.map(|t| {
             chunks
                 .iter()
                 .take(3)
-                .all(|c| t.encode(c.as_str(), false).unwrap().get_ids() == pipe_ids(c))
+                .all(|c| t.encode_fast(c.as_str(), false).unwrap().get_ids() == pipe_ids(c))
         });
 
         // interleave all impls so frequency/thermal drift hits them equally
@@ -963,7 +976,7 @@ fn main() {
         let baseline = match BaselineTokenizer::from_file(&path) {
             Ok(mut b) => {
                 inject_added_tokens_baseline(&mut b);
-                match b.encode(PROBE, false) {
+                match b.encode_fast(PROBE, false) {
                     Ok(_) => Some(b),
                     Err(e) => {
                         eprintln!("  baseline v{BASELINE_VERSION} loads but can't encode: {e}");
@@ -982,7 +995,7 @@ fn main() {
         let sebpop = match SebpopTokenizer::from_file(&path) {
             Ok(mut t) => {
                 inject_added_tokens_sebpop(&mut t);
-                match t.encode(PROBE, false) {
+                match t.encode_fast(PROBE, false) {
                     Ok(_) => Some(t),
                     Err(e) => {
                         eprintln!("  {SEBPOP_REF} loads but can't encode: {e}");

From ae0478f5d9732b5c22a3c109ff83125a32099170 Mon Sep 17 00:00:00 2001
From: SBrandeis <33657802+SBrandeis@users.noreply.github.com>
Date: Mon, 20 Jul 2026 18:26:02 +0200
Subject: [PATCH 3/3] bench(pipeline): survive sebpop encode_fast panics, skip
 the series
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

sebpop/upstream@13699f62 guards transform_range for its no-offsets fast
path but not NormalizedString::replace, so encode_fast panics ("offset
data accessed on fast path") on any model with a Replace normalizer —
llama-2's Prepend+Replace killed bench shard 2. Probe under catch_unwind
and demote the series to None, the same fallback used for configs that
fail to load; charts render the model without a sebpop bar.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 tokenizers/tk-encode/examples/fixture_bench.rs | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/tokenizers/tk-encode/examples/fixture_bench.rs b/tokenizers/tk-encode/examples/fixture_bench.rs
index f18c72fa8..ef81fdf24 100644
--- a/tokenizers/tk-encode/examples/fixture_bench.rs
+++ b/tokenizers/tk-encode/examples/fixture_bench.rs
@@ -991,16 +991,24 @@ fn main() {
         };
 
         // Same fallback for sebpop's branch (based on an older tree, so it may
-        // predate configs the release handles).
+        // predate configs the release handles). Probed under catch_unwind: its
+        // encode_fast panics ("offset data accessed on fast path") on any model
+        // whose normalizer mutates the string, e.g. llama-2's Prepend+Replace.
         let sebpop = match SebpopTokenizer::from_file(&path) {
             Ok(mut t) => {
                 inject_added_tokens_sebpop(&mut t);
-                match t.encode_fast(PROBE, false) {
-                    Ok(_) => Some(t),
-                    Err(e) => {
+                match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+                    t.encode_fast(PROBE, false)
+                })) {
+                    Ok(Ok(_)) => Some(t),
+                    Ok(Err(e)) => {
                         eprintln!("  {SEBPOP_REF} loads but can't encode: {e}");
                         None
                     }
+                    Err(_) => {
+                        eprintln!("  {SEBPOP_REF} panics on encode_fast — skipping series");
+                        None
+                    }
                 }
             }
             Err(e) => {