From f2d0bd5273e8754c64cf0aa1ba09255724a80a00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Mon, 20 Jul 2026 21:04:10 -0700 Subject: [PATCH 01/54] =?UTF-8?q?=E2=9A=A1=20perf(urls):=20move=20tracker?= =?UTF-8?q?=20and=20dot-segment=20logic=20to=20C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tracking-parameter vocabulary and the dot-segment resolver ran per URL in Python. Both are now C entry points (_url_is_tracker, _url_remove_dot_segments) with the tables as C data; the Python module keeps only the public wrappers. A 302,762-input differential run against the Python originals found one divergence class: the C resolver matches %2e case-insensitively per WHATWG 4.4 where the Python matched only %2E, unreachable through the public API because _encode uppercases escape hex before resolution. --- src/turbohtml/_c/core/common.h | 8 ++ src/turbohtml/_c/core/module.c | 2 + src/turbohtml/_c/url/url.c | 202 +++++++++++++++++++++++++++++++++ src/turbohtml/extract/_urls.py | 67 +---------- 4 files changed, 217 insertions(+), 62 deletions(-) diff --git a/src/turbohtml/_c/core/common.h b/src/turbohtml/_c/core/common.h index 73fb6549..1793dfd7 100644 --- a/src/turbohtml/_c/core/common.h +++ b/src/turbohtml/_c/core/common.h @@ -109,6 +109,14 @@ PyObject *turbohtml_url_percent_decode(PyObject *module, PyObject *arg); PyObject *th_url_join(PyObject *base, PyObject *target); PyObject *turbohtml_url_join(PyObject *module, PyObject *args); +/* _url_is_tracker(key): whether a lowercased query-parameter name names a referral rather than content, so a + crawl-oriented cleaner drops it. Matches METH_O. */ +PyObject *turbohtml_url_is_tracker(PyObject *module, PyObject *arg); + +/* _url_remove_dot_segments(path): resolve the "." and ".." segments of a path, in either their literal or + %2E spelling, as the WHATWG path state does (spec 4.4). Matches METH_O. */ +PyObject *turbohtml_url_remove_dot_segments(PyObject *module, PyObject *arg); + /* Implemented in url/idna.c. th_url_to_ascii runs the WHATWG domain-to-ASCII step (Unicode IDNA ToASCII, UTS #46 with Transitional_Processing=false and UseSTD3ASCIIRules=false): UTS #46 mapping, NFC, and per-label punycode. _urls.py reaches _url_to_ascii(host) for its registered-name hosts instead of the IDNA-2003 str.encode("idna") codec; the host diff --git a/src/turbohtml/_c/core/module.c b/src/turbohtml/_c/core/module.c index ffb152bf..abb6812a 100644 --- a/src/turbohtml/_c/core/module.c +++ b/src/turbohtml/_c/core/module.c @@ -230,6 +230,8 @@ static PyMethodDef html_methods[] = { {"_url_percent_encode", turbohtml_url_percent_encode, METH_VARARGS, NULL}, {"_url_percent_decode", turbohtml_url_percent_decode, METH_O, NULL}, {"_url_join", turbohtml_url_join, METH_VARARGS, NULL}, + {"_url_is_tracker", turbohtml_url_is_tracker, METH_O, NULL}, + {"_url_remove_dot_segments", turbohtml_url_remove_dot_segments, METH_O, NULL}, {"_url_to_ascii", turbohtml_url_to_ascii, METH_O, NULL}, {"_sanitize", turbohtml_sanitize, METH_VARARGS, NULL}, {"_grow_probe", turbohtml_grow_probe, METH_VARARGS, NULL}, diff --git a/src/turbohtml/_c/url/url.c b/src/turbohtml/_c/url/url.c index 9c847036..b616466d 100644 --- a/src/turbohtml/_c/url/url.c +++ b/src/turbohtml/_c/url/url.c @@ -798,3 +798,205 @@ PyObject *turbohtml_url_join(PyObject *Py_UNUSED(module), PyObject *args) { } return th_url_join(base, target); } + +/* The tracking-parameter vocabulary. A crawl-oriented cleaner drops these query keys because they identify the + referral, not the content, so two URLs differing only in them address the same page. Kept sorted so the exact-name + test is a binary search. */ +static const char *const TRACKER_NAMES[] = { + "clickid", "dclid", "efid", "epik", "fb_ref", "fb_source", "fbclid", "gbraid", + "gclid", "gclsrc", "igsh", "igshid", "mkt_tok", "msclkid", "partnerid", "s_cid", + "sc_cid", "ttclid", "twclid", "wbraid", "wickedid", "yclid", "ysclid", +}; + +static const char *const TRACKER_PREFIXES[] = { + "ad_", "ads_", "ga_", "gs_", "hsa_", "itm_", "mc_", "mtm_", "oly_", "pk_", "utm_", "vero_", +}; + +/* The words a tracking key is built from, matched as a whole underscore-delimited word rather than a substring so + "reference" is not read as "ref". */ +static const char *const TRACKER_WORDS[] = { + "aff", "affi", "affiliate", "campaign", "cid", "clid", "keyword", "kwd", + "medium", "ref", "refer", "referer", "referrer", "session", "source", "uid", "xtor", +}; + +static int tracker_name_known(const char *key, size_t len) { + size_t low = 0; + size_t high = sizeof(TRACKER_NAMES) / sizeof(TRACKER_NAMES[0]); + while (low < high) { + size_t mid = low + (high - low) / 2; + int order = strncmp(TRACKER_NAMES[mid], key, len); + if (order == 0 && TRACKER_NAMES[mid][len] != '\0') { + order = 1; + } + if (order == 0) { + return 1; + } + if (order < 0) { + low = mid + 1; + } else { + high = mid; + } + } + return 0; +} + +static int tracker_word_at(const char *key, size_t start, size_t end) { + size_t len = end - start; + for (size_t index = 0; index < sizeof(TRACKER_WORDS) / sizeof(TRACKER_WORDS[0]); index++) { + if (strlen(TRACKER_WORDS[index]) == len && strncmp(TRACKER_WORDS[index], key + start, len) == 0) { + return 1; + } + } + return 0; +} + +/* _url_is_tracker(key): whether a lowercased query-parameter name identifies a referral rather than content. */ +PyObject *turbohtml_url_is_tracker(PyObject *Py_UNUSED(module), PyObject *arg) { + if (!PyUnicode_Check(arg)) { + PyErr_SetString(PyExc_TypeError, "_url_is_tracker() argument must be str"); + return NULL; + } + Py_ssize_t size; + const char *key = PyUnicode_AsUTF8AndSize(arg, &size); + if (key == NULL) { /* GCOVR_EXCL_BR_LINE: a lone surrogate cannot reach here from a decoded parameter name */ + return NULL; /* GCOVR_EXCL_LINE: unencodable-name path */ + } + size_t len = (size_t)size; + if (tracker_name_known(key, len)) { + Py_RETURN_TRUE; + } + for (size_t index = 0; index < sizeof(TRACKER_PREFIXES) / sizeof(TRACKER_PREFIXES[0]); index++) { + size_t plen = strlen(TRACKER_PREFIXES[index]); + if (len >= plen && strncmp(key, TRACKER_PREFIXES[index], plen) == 0) { + Py_RETURN_TRUE; + } + } + if (len >= 4 && strncmp(key + len - 4, "clid", 4) == 0) { + Py_RETURN_TRUE; + } + /* each underscore-delimited word, so a tracking word anywhere in the name matches but a longer word containing + one does not */ + size_t start = 0; + for (size_t index = 0; index <= len; index++) { + if (index == len || key[index] == '_') { + if (tracker_word_at(key, start, index)) { + Py_RETURN_TRUE; + } + start = index + 1; + } + } + Py_RETURN_FALSE; +} + +/* Whether the segment work[start,end) spells "." or ".." once its %2E escapes are read as dots; `dots` receives the + dot count so the caller can tell the two apart. A segment of anything else answers zero. */ +static int dot_segment(const Py_UCS4 *work, Py_ssize_t start, Py_ssize_t end, int *dots) { + int count = 0; + for (Py_ssize_t index = start; index < end;) { + if (work[index] == '.') { + index += 1; + } else if (index + 2 < end && work[index] == '%' && work[index + 1] == '2' && + (work[index + 2] == 'E' || work[index + 2] == 'e')) { + index += 3; + } else { + return 0; + } + count += 1; + if (count > 2) { + return 0; + } + } + *dots = count; + return count == 1 || count == 2; +} + +/* _url_remove_dot_segments(path): resolve the "." and ".." segments of a path, in either their literal or %2E + spelling, keeping every other segment's encoding verbatim. */ +PyObject *turbohtml_url_remove_dot_segments(PyObject *Py_UNUSED(module), PyObject *arg) { + if (!PyUnicode_Check(arg)) { + PyErr_SetString(PyExc_TypeError, "_url_remove_dot_segments() argument must be str"); + return NULL; + } + Py_ssize_t len = PyUnicode_GET_LENGTH(arg); + int kind = PyUnicode_KIND(arg); + const void *data = PyUnicode_DATA(arg); + Py_UCS4 *work = PyMem_Malloc((size_t)(len + 1) * sizeof(Py_UCS4)); + if (work == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ + return PyErr_NoMemory(); /* GCOVR_EXCL_LINE: allocation-failure path */ + } + int dotted = 0; + for (Py_ssize_t index = 0; index < len; index++) { + work[index] = PyUnicode_READ(kind, data, index); + if (work[index] == '.' || work[index] == '%') { + dotted = 1; + } + } + if (!dotted) { /* no segment can be dotted, so the path resolves to itself */ + PyMem_Free(work); + return Py_NewRef(arg); + } + /* segment starts, so popping a ".." is dropping the last recorded start */ + Py_ssize_t *starts = PyMem_Malloc((size_t)(len + 2) * sizeof(Py_ssize_t)); + Py_ssize_t *ends = PyMem_Malloc((size_t)(len + 2) * sizeof(Py_ssize_t)); + if (starts == NULL || ends == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ + PyMem_Free(work); /* GCOVR_EXCL_LINE: allocation-failure path */ + PyMem_Free(starts); /* GCOVR_EXCL_LINE: allocation-failure path */ + PyMem_Free(ends); /* GCOVR_EXCL_LINE: allocation-failure path */ + return PyErr_NoMemory(); /* GCOVR_EXCL_LINE: allocation-failure path */ + } + Py_ssize_t kept = 0; + Py_ssize_t start = 0; + int last_dots = 0; + for (Py_ssize_t index = 0; index <= len; index++) { + if (index != len && work[index] != '/') { + continue; + } + last_dots = 0; + int single = dot_segment(work, start, index, &last_dots); + if (single && last_dots == 1) { + last_dots = 1; + } else if (single) { + /* a ".." drops the previous segment, but never the leading empty one a rooted path opens with */ + if (kept > 1) { + kept -= 1; + } + } else { + starts[kept] = start; + ends[kept] = index; + kept += 1; + last_dots = 0; + } + start = index + 1; + } + if (last_dots != 0) { /* a trailing dot segment leaves the path ending in a separator */ + starts[kept] = 0; + ends[kept] = 0; + kept += 1; + } + Py_ssize_t total = kept > 0 ? kept - 1 : 0; + for (Py_ssize_t index = 0; index < kept; index++) { + total += ends[index] - starts[index]; + } + Py_UCS4 *out = PyMem_Malloc((size_t)(total + 1) * sizeof(Py_UCS4)); + if (out == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ + PyMem_Free(work); /* GCOVR_EXCL_LINE: allocation-failure path */ + PyMem_Free(starts); /* GCOVR_EXCL_LINE: allocation-failure path */ + PyMem_Free(ends); /* GCOVR_EXCL_LINE: allocation-failure path */ + return PyErr_NoMemory(); /* GCOVR_EXCL_LINE: allocation-failure path */ + } + Py_ssize_t at = 0; + for (Py_ssize_t index = 0; index < kept; index++) { + if (index != 0) { + out[at++] = '/'; + } + for (Py_ssize_t scan = starts[index]; scan < ends[index]; scan++) { + out[at++] = work[scan]; + } + } + PyObject *result = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, out, at); + PyMem_Free(work); + PyMem_Free(starts); + PyMem_Free(ends); + PyMem_Free(out); + return result; +} diff --git a/src/turbohtml/extract/_urls.py b/src/turbohtml/extract/_urls.py index ee806d75..b8e90303 100644 --- a/src/turbohtml/extract/_urls.py +++ b/src/turbohtml/extract/_urls.py @@ -20,9 +20,11 @@ from turbohtml._html import ( _registrable_domain, + _url_is_tracker, _url_join, _url_percent_decode, _url_percent_encode, + _url_remove_dot_segments, _url_split, _url_to_ascii, parse, @@ -52,35 +54,6 @@ # Names compiled from the public query-stripping lists (Firefox's query-stripping records, the ClearURLs rules, and # AdGuard's TrackParamFilter), the same sources courlan cites. -_TRACKER_NAMES: Final[frozenset[str]] = frozenset({ - "clickid", - "dclid", - "efid", - "epik", - "fb_ref", - "fb_source", - "fbclid", - "gbraid", - "gclid", - "gclsrc", - "igsh", - "igshid", - "mkt_tok", - "msclkid", - "partnerid", - "s_cid", - "sc_cid", - "ttclid", - "twclid", - "wbraid", - "wickedid", - "yclid", - "ysclid", -}) -_TRACKER_PREFIXES: Final = ("ad_", "ads_", "ga_", "gs_", "hsa_", "itm_", "mc_", "mtm_", "oly_", "pk_", "utm_", "vero_") -_TRACKER_WORDS: Final = re.compile( - r"(?:^|_)(?:aff(?:i(?:liate)?)?|campaign|cl?id|keyword|kwd|medium|refer(?:r?er)?|ref|session|source|uid|xtor)(?:_|$)" -) _CONTENT_PARAMS: Final[frozenset[str]] = frozenset({ "aid", @@ -353,7 +326,7 @@ def _normalize(parts: _Split, options: UrlCleaning) -> str: netloc = _normalize_netloc(parts, scheme) if parts.netloc else "" path = _encode(parts.path, _SET_PATH) if netloc: - path = _remove_dot_segments(path) + path = _url_remove_dot_segments(path) query = _normalize_query(parts.query, options) if netloc and scheme in _WEB_SCHEMES and not path: path = "/" # a special URL with a host never serializes an empty path (URL serializing, spec 4.5) @@ -406,26 +379,6 @@ def _encode(text: str, url_set: int) -> str: raise ValueError(msg) from exc -def _remove_dot_segments(path: str) -> str: - """Resolve ``.`` and ``..`` segments, including their ``%2e`` spellings, as the path state does (spec 4.4).""" - if "." not in path and "%2E" not in path: - return path - output: list[str] = [] - dotted = "" - for segment in path.split("/"): - dotted = segment.replace("%2E", ".") - if dotted == ".": - continue - if dotted == "..": - if len(output) > 1: - output.pop() - else: - output.append(segment) - if dotted in {".", ".."}: - output.append("") - return "/".join(output) - - def _normalize_query(query: str, options: UrlCleaning) -> str: """Drop denied, tracker, or non-allowlisted parameters and sort the rest, keeping each pair's raw encoding.""" allow = {name.lower() for name in options.query_allow} if options.query_allow is not None else None @@ -442,29 +395,19 @@ def _normalize_query(query: str, options: UrlCleaning) -> str: elif options.strict: dropped = key not in _CONTENT_PARAMS and key not in _LANGUAGE_PARAMS else: - dropped = _is_tracker(key) + dropped = _url_is_tracker(key) if dropped: continue kept.append((key, _encode(pair, _SET_QUERY))) return "&".join(pair for _key, pair in sorted(kept)) -def _is_tracker(key: str) -> bool: - """Match a query-parameter name against the compiled tracking-parameter vocabulary.""" - return ( - key in _TRACKER_NAMES - or key.startswith(_TRACKER_PREFIXES) - or key.endswith("clid") - or _TRACKER_WORDS.search(key) is not None - ) - - def _normalize_fragment(fragment: str, options: UrlCleaning) -> str: """Percent-encode the fragment, first scrubbing trackers from one shaped like a query string.""" if "=" in fragment: if "&" in fragment: return _normalize_query(fragment, options) - if _is_tracker(_url_percent_decode(fragment.partition("=")[0]).lower()): + if _url_is_tracker(_url_percent_decode(fragment.partition("=")[0]).lower()): return "" return _encode(fragment, _SET_FRAGMENT) From 004222539e30704f5a99b457dce540bde9a79eee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Mon, 20 Jul 2026 21:04:10 -0700 Subject: [PATCH 02/54] =?UTF-8?q?=E2=9C=A8=20feat(bench):=20scope=20caveat?= =?UTF-8?q?s=20to=20rows=20and=20explain=20gaps?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A note about one operation marked a library's whole migration column, and an empty cell asserted 'no measurement recorded' without saying why. Migration feeds now carry row_notes keyed by row index so a caveat marks only the rows of the operation it describes, and a configuration that skips an operation says 'same in both configurations, so measured once' instead of leaving a blank. --- docs/_ext/bench_table.py | 98 +++++++++++++++----- docs/migration/bench/airium.json | 2 +- docs/migration/bench/beautifulsoup.json | 32 +++---- docs/migration/bench/bleach.json | 2 +- docs/migration/bench/boilerpy3.json | 2 +- docs/migration/bench/calmjs-parse.json | 2 +- docs/migration/bench/chardet.json | 2 +- docs/migration/bench/charset-normalizer.json | 2 +- docs/migration/bench/courlan.json | 2 +- docs/migration/bench/csscompressor.json | 2 +- docs/migration/bench/cssselect.json | 2 +- docs/migration/bench/dominate.json | 2 +- docs/migration/bench/dompurify.json | 2 +- docs/migration/bench/extruct.json | 2 +- docs/migration/bench/fast-html.json | 6 +- docs/migration/bench/feedparser.json | 2 +- docs/migration/bench/goose3.json | 2 +- docs/migration/bench/htbuilder.json | 9 +- docs/migration/bench/html-sanitizer.json | 2 +- docs/migration/bench/html-text.json | 2 +- docs/migration/bench/html2text.json | 2 +- docs/migration/bench/html5-parser.json | 2 +- docs/migration/bench/html5lib.json | 2 +- docs/migration/bench/htmldate.json | 6 +- docs/migration/bench/htmlmin.json | 2 +- docs/migration/bench/htpy.json | 9 +- docs/migration/bench/hyperpython.json | 2 +- docs/migration/bench/inscriptis.json | 2 +- docs/migration/bench/jsmin.json | 7 +- docs/migration/bench/justext.json | 2 +- docs/migration/bench/lightningcss.json | 2 +- docs/migration/bench/linkify-it-py.json | 2 +- docs/migration/bench/lxml-html-clean.json | 6 +- docs/migration/bench/lxml.json | 2 +- docs/migration/bench/markdownify.json | 2 +- docs/migration/bench/markupsafe.json | 2 +- docs/migration/bench/markyp.json | 9 +- docs/migration/bench/metadata_parser.json | 2 +- docs/migration/bench/microdata.json | 2 +- docs/migration/bench/minify-html.json | 2 +- docs/migration/bench/news-please.json | 2 +- docs/migration/bench/newspaper3k.json | 2 +- docs/migration/bench/nh3.json | 2 +- docs/migration/bench/opengraph.json | 2 +- docs/migration/bench/pandas.json | 2 +- docs/migration/bench/parse5.json | 2 +- docs/migration/bench/parsel.json | 2 +- docs/migration/bench/pyquery.json | 2 +- docs/migration/bench/rcssmin.json | 9 +- docs/migration/bench/readabilipy.json | 2 +- docs/migration/bench/readability-lxml.json | 2 +- docs/migration/bench/resiliparse.json | 11 ++- docs/migration/bench/rjsmin.json | 7 +- docs/migration/bench/sanitize-html.json | 2 +- docs/migration/bench/selectolax.json | 2 +- docs/migration/bench/simple-html.json | 9 +- docs/migration/bench/soupsieve.json | 2 +- docs/migration/bench/stdlib.json | 8 +- docs/migration/bench/trafilatura.json | 2 +- docs/migration/bench/w3lib.json | 7 +- docs/migration/bench/yattag.json | 6 +- tests/test_bench_migration.py | 8 +- tools/bench/migration.py | 37 +++++--- 63 files changed, 245 insertions(+), 129 deletions(-) diff --git a/docs/_ext/bench_table.py b/docs/_ext/bench_table.py index d9e02d88..d98ddcf3 100644 --- a/docs/_ext/bench_table.py +++ b/docs/_ext/bench_table.py @@ -26,6 +26,7 @@ import json import math +from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING, Any, ClassVar, Final @@ -260,12 +261,53 @@ def _mark_entry(ordinal: int) -> nodes.entry: return entry +def _table_frame(ncols: int) -> tuple[nodes.table, nodes.tgroup]: + """Build the table and its column group: a wide first column for the row label, then one per metric column.""" + table = nodes.table(classes=["bench-table"]) + group = nodes.tgroup(cols=ncols) + table += group + group += nodes.colspec(colwidth=2) + for _ in range(ncols - 1): + group += nodes.colspec(colwidth=1) + return table, group + + +@dataclass(frozen=True) +class _Layout: + """What every row of one table shares: its columns, its metrics, and the numbered reasons its cells key into.""" + + parties: list[str] + metrics: list[str] + reasons: dict[str, int] + + +def _row_caveats(feed: dict[str, Any], taken: int) -> tuple[dict[str, int], dict[int, int]]: + """Assign each distinct per-row caveat an ordinal after those already taken, and map each row to its ordinal.""" + notes: dict[str, int] = {} + marks: dict[int, int] = {} + for index, note in sorted((int(key), value) for key, value in (feed.get("row_notes") or {}).items()): + notes.setdefault(note, taken + len(notes) + 1) + marks[index] = notes[note] + return notes, marks + + def _ordered_notes(feed: dict[str, Any], parties: list[str]) -> list[str]: """Return the noted parties in the order their columns appear, so ordinals read left to right.""" notes = feed.get("notes") or {} return [party for party in parties if party in notes] +def _merge_legend( + reasons: dict[str, int], notes: dict[str, int], row_notes: dict[str, int], feed: dict[str, Any] +) -> dict[str, int]: + """Gather the empty-cell reasons, the column notes and the row caveats into one numbered legend.""" + legend = dict(reasons) + for party, ordinal in notes.items(): + legend[f"{party} {feed['notes'][party]}"] = ordinal + legend.update(row_notes) + return legend + + def _legend(reasons: dict[str, int]) -> nodes.container: """Spell each numbered reason out under the table; reason text is literal so a message cannot inject markup.""" legend = nodes.container(classes=["bench-legend"]) @@ -291,30 +333,34 @@ def run(self) -> list[nodes.Node]: metrics = feed["metrics"] or ["time"] parties, rows, spread = _order_columns(feed["parties"], metrics, feed["rows"], feed.get("spread") or []) ncols = 1 + len(parties) * len(metrics) - table = nodes.table(classes=["bench-table"]) - group = nodes.tgroup(cols=ncols) - table += group - group += nodes.colspec(colwidth=2) - for _ in range(ncols - 1): - group += nodes.colspec(colwidth=1) - reasons = _reason_map(rows) + table, group = _table_frame(ncols) + layout = _Layout(parties, metrics, _reason_map(rows)) # notes number on from the empty-cell reasons so one legend carries both without colliding ordinals - notes = {party: len(reasons) + offset for offset, party in enumerate(_ordered_notes(feed, parties), start=1)} + notes = { + party: len(layout.reasons) + offset for offset, party in enumerate(_ordered_notes(feed, parties), start=1) + } + row_notes, row_marks = _row_caveats(feed, len(layout.reasons) + len(notes)) group += self._head(feed["label"], parties, feed["metrics"], notes) + group += self._body(rows, spread, layout, row_marks, ncols) + legend = _merge_legend(layout.reasons, notes, row_notes, feed) + return [table] if not legend else [table, _legend(legend)] + + def _body( + self, + rows: list[list[Any]], + spread: list[Any], + layout: _Layout, + row_marks: dict[int, int], + ncols: int, + ) -> nodes.tbody: body = nodes.tbody() - group += body - for row_index, cells in enumerate(rows): + for index, cells in enumerate(rows): if len(cells) != ncols: msg = f"bench-table row has {len(cells)} cells, expected {ncols}: {cells!r}" raise self.error(msg) - noise = spread[row_index] if row_index < len(spread) else None - body += self._body_row(cells, parties, metrics, reasons, noise) - legend = dict(reasons) - for party, ordinal in notes.items(): - legend[f"{party} {feed['notes'][party]}"] = ordinal - if not legend: - return [table] - return [table, _legend(legend)] + noise = spread[index] if index < len(spread) else None + body += self._body_row(cells, layout, noise, row_marks.get(index)) + return body def _head(self, label: str, parties: list[str], metrics: list[str], notes: dict[str, int]) -> nodes.thead: head = nodes.thead() @@ -352,15 +398,17 @@ def _head(self, label: str, parties: list[str], metrics: list[str], notes: dict[ return head def _body_row( - self, - cells: list[Any], - parties: list[str], - metrics: list[str], - reasons: dict[str, int], - spread: list[Any] | None = None, + self, cells: list[Any], layout: _Layout, spread: list[Any] | None = None, caveat: int | None = None ) -> nodes.row: + parties, metrics, reasons = layout.parties, layout.metrics, layout.reasons row = nodes.row() - row += self._entry(str(cells[0])) + label = self._entry(str(cells[0])) + # a caveat that belongs to one operation marks that row, not the library's whole column + if caveat is not None: + superscript = nodes.superscript() + superscript += nodes.Text(str(caveat)) + label.children[0] += superscript + row += label rendered: dict[int, tuple[str, str | None]] = {} marks: dict[int, int] = {} for metric_index, metric in enumerate(metrics): diff --git a/docs/migration/bench/airium.json b/docs/migration/bench/airium.json index c9b8bbc2..deb9b0e3 100644 --- a/docs/migration/bench/airium.json +++ b/docs/migration/bench/airium.json @@ -39,5 +39,5 @@ 0.03586474634366849 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/beautifulsoup.json b/docs/migration/bench/beautifulsoup.json index 6069969f..887ebcdc 100644 --- a/docs/migration/bench/beautifulsoup.json +++ b/docs/migration/bench/beautifulsoup.json @@ -11,55 +11,55 @@ "build a list (constructors) — 100 rows", 5.7091425532007634e-05, 0.0007816329752283006, - null + "same in both configurations, so measured once" ], [ "build a list (constructors) — 1k rows", 0.0005440194775019336, 0.007474331593887958, - null + "same in both configurations, so measured once" ], [ "build a list (constructors) — 10k rows", 0.005411195857050188, 0.0822198923851829, - null + "same in both configurations, so measured once" ], [ "construct N elements (no serialize) — 100 rows", 4.288681538374325e-05, 0.0002607024263738822, - null + "same in both configurations, so measured once" ], [ "construct N elements (no serialize) — 1k rows", 0.0004355111897590784, 0.0025463228632058113, - null + "same in both configurations, so measured once" ], [ "construct N elements (no serialize) — 10k rows", 0.0043399677733759745, 0.025400459164908778, - null + "same in both configurations, so measured once" ], [ "emit a built tree — 100 rows", 4.327393534175883e-06, 0.0003889943512641973, - null + "same in both configurations, so measured once" ], [ "emit a built tree — 1k rows", 4.2981901363721896e-05, 0.003848052945007415, - null + "same in both configurations, so measured once" ], [ "emit a built tree — 10k rows", 0.00045268575032271957, 0.040861342851712834, - null + "same in both configurations, so measured once" ], [ "parse to a tree — wpt tiny (0.6 kB)", @@ -611,37 +611,37 @@ "detect a byte stream's encoding — ascii (1 kB)", 2.683881651736423e-06, 0.00016621675959527238, - null + "same in both configurations, so measured once" ], [ "detect a byte stream's encoding — utf-8 russian (4 kB)", 8.42264663110844e-06, 0.0002101907146917862, - null + "same in both configurations, so measured once" ], [ "detect a byte stream's encoding — windows-1251 russian (4 kB)", 0.00023194885286178155, 0.0007660471198960295, - null + "same in both configurations, so measured once" ], [ "detect a byte stream's encoding — windows-1252 french (4 kB)", 0.00022984596078382916, 0.0009623634165715581, - null + "same in both configurations, so measured once" ], [ "detect a byte stream's encoding — shift_jis japanese (4 kB)", 0.00010243962940137408, 0.0007823282336403281, - null + "same in both configurations, so measured once" ], [ "detect a byte stream's encoding — utf-8 page (95 kB)", 1.0262474480586075e-06, 0.0017221563046708372, - null + "same in both configurations, so measured once" ], [ "extract filtered page links — daring fireball (10 kB)", @@ -1330,5 +1330,5 @@ 0.12059134279394809 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/bleach.json b/docs/migration/bench/bleach.json index fdda6c18..2608695d 100644 --- a/docs/migration/bench/bleach.json +++ b/docs/migration/bench/bleach.json @@ -59,5 +59,5 @@ 0.024059342839235647 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/boilerpy3.json b/docs/migration/bench/boilerpy3.json index 92420b38..578823ee 100644 --- a/docs/migration/bench/boilerpy3.json +++ b/docs/migration/bench/boilerpy3.json @@ -39,5 +39,5 @@ 0.07765030969801458 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/calmjs-parse.json b/docs/migration/bench/calmjs-parse.json index fbb63357..7db5327c 100644 --- a/docs/migration/bench/calmjs-parse.json +++ b/docs/migration/bench/calmjs-parse.json @@ -68,5 +68,5 @@ 0.06493109954770923 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/chardet.json b/docs/migration/bench/chardet.json index 38e5b885..b5311405 100644 --- a/docs/migration/bench/chardet.json +++ b/docs/migration/bench/chardet.json @@ -69,5 +69,5 @@ 0.02346546131727304 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/charset-normalizer.json b/docs/migration/bench/charset-normalizer.json index d535f93f..169ea634 100644 --- a/docs/migration/bench/charset-normalizer.json +++ b/docs/migration/bench/charset-normalizer.json @@ -69,5 +69,5 @@ 0.01868390630267981 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/courlan.json b/docs/migration/bench/courlan.json index c39aa04a..210da605 100644 --- a/docs/migration/bench/courlan.json +++ b/docs/migration/bench/courlan.json @@ -69,5 +69,5 @@ 0.059124346380420974 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/csscompressor.json b/docs/migration/bench/csscompressor.json index a10e1d31..88f6a2ab 100644 --- a/docs/migration/bench/csscompressor.json +++ b/docs/migration/bench/csscompressor.json @@ -96,5 +96,5 @@ 0.009543429204608288 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/cssselect.json b/docs/migration/bench/cssselect.json index 00538cf9..fc429981 100644 --- a/docs/migration/bench/cssselect.json +++ b/docs/migration/bench/cssselect.json @@ -109,5 +109,5 @@ 0.037440091999251815 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/dominate.json b/docs/migration/bench/dominate.json index 39eed83f..9a385dd6 100644 --- a/docs/migration/bench/dominate.json +++ b/docs/migration/bench/dominate.json @@ -269,5 +269,5 @@ 0.0705497022490886 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/dompurify.json b/docs/migration/bench/dompurify.json index 17edceaa..d9698332 100644 --- a/docs/migration/bench/dompurify.json +++ b/docs/migration/bench/dompurify.json @@ -39,5 +39,5 @@ 0.06365377101405542 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/extruct.json b/docs/migration/bench/extruct.json index 59575240..ef5d7420 100644 --- a/docs/migration/bench/extruct.json +++ b/docs/migration/bench/extruct.json @@ -49,5 +49,5 @@ 0.12373022254611753 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/fast-html.json b/docs/migration/bench/fast-html.json index 52850d3f..1a303711 100644 --- a/docs/migration/bench/fast-html.json +++ b/docs/migration/bench/fast-html.json @@ -39,7 +39,9 @@ 0.02516429468186054 ] ], - "notes": { - "fast-html": "concatenates a string rather than constructing a navigable tree, so nothing it builds can afterwards be queried or mutated" + "row_notes": { + "0": "concatenates a string rather than constructing a navigable tree, so nothing it builds can afterwards be queried or mutated", + "1": "concatenates a string rather than constructing a navigable tree, so nothing it builds can afterwards be queried or mutated", + "2": "concatenates a string rather than constructing a navigable tree, so nothing it builds can afterwards be queried or mutated" } } diff --git a/docs/migration/bench/feedparser.json b/docs/migration/bench/feedparser.json index fc942de4..21b80b3f 100644 --- a/docs/migration/bench/feedparser.json +++ b/docs/migration/bench/feedparser.json @@ -19,5 +19,5 @@ 0.18465721167062013 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/goose3.json b/docs/migration/bench/goose3.json index 8790831a..fff89003 100644 --- a/docs/migration/bench/goose3.json +++ b/docs/migration/bench/goose3.json @@ -89,5 +89,5 @@ 0.056976880840888505 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/htbuilder.json b/docs/migration/bench/htbuilder.json index 1ec83277..aa46c053 100644 --- a/docs/migration/bench/htbuilder.json +++ b/docs/migration/bench/htbuilder.json @@ -99,7 +99,12 @@ 0.010705242198592509 ] ], - "notes": { - "htbuilder": "concatenates a string rather than constructing a navigable tree, so nothing it builds can afterwards be queried or mutated" + "row_notes": { + "0": "concatenates a string rather than constructing a navigable tree, so nothing it builds can afterwards be queried or mutated", + "1": "concatenates a string rather than constructing a navigable tree, so nothing it builds can afterwards be queried or mutated", + "2": "concatenates a string rather than constructing a navigable tree, so nothing it builds can afterwards be queried or mutated", + "3": "concatenates a string rather than constructing a navigable tree, so nothing it builds can afterwards be queried or mutated", + "4": "concatenates a string rather than constructing a navigable tree, so nothing it builds can afterwards be queried or mutated", + "5": "concatenates a string rather than constructing a navigable tree, so nothing it builds can afterwards be queried or mutated" } } diff --git a/docs/migration/bench/html-sanitizer.json b/docs/migration/bench/html-sanitizer.json index d61d7e65..fe90be35 100644 --- a/docs/migration/bench/html-sanitizer.json +++ b/docs/migration/bench/html-sanitizer.json @@ -29,5 +29,5 @@ 0.12087625676310836 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/html-text.json b/docs/migration/bench/html-text.json index b11b32cc..a2d110cd 100644 --- a/docs/migration/bench/html-text.json +++ b/docs/migration/bench/html-text.json @@ -39,5 +39,5 @@ 0.035766572600784 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/html2text.json b/docs/migration/bench/html2text.json index 974c4c67..5ff5b851 100644 --- a/docs/migration/bench/html2text.json +++ b/docs/migration/bench/html2text.json @@ -59,5 +59,5 @@ 0.16195119580371894 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/html5-parser.json b/docs/migration/bench/html5-parser.json index 93b6c437..75df6e96 100644 --- a/docs/migration/bench/html5-parser.json +++ b/docs/migration/bench/html5-parser.json @@ -79,5 +79,5 @@ 0.01906238222885146 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/html5lib.json b/docs/migration/bench/html5lib.json index f0735c94..fe9714d1 100644 --- a/docs/migration/bench/html5lib.json +++ b/docs/migration/bench/html5lib.json @@ -299,5 +299,5 @@ 0.011445626925372463 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/htmldate.json b/docs/migration/bench/htmldate.json index c40affb8..1a9fc1fb 100644 --- a/docs/migration/bench/htmldate.json +++ b/docs/migration/bench/htmldate.json @@ -39,7 +39,9 @@ 0.0308781661300203 ] ], - "notes": { - "htmldate": "returns no date for the 100-candidate case, so its timing there is the cost of giving up rather than of finding the date turbohtml reports" + "row_notes": { + "0": "returns no date for the 100-candidate case, so its timing there is the cost of giving up rather than of finding the date turbohtml reports", + "1": "returns no date for the 100-candidate case, so its timing there is the cost of giving up rather than of finding the date turbohtml reports", + "2": "returns no date for the 100-candidate case, so its timing there is the cost of giving up rather than of finding the date turbohtml reports" } } diff --git a/docs/migration/bench/htmlmin.json b/docs/migration/bench/htmlmin.json index 518eb33a..60006cf1 100644 --- a/docs/migration/bench/htmlmin.json +++ b/docs/migration/bench/htmlmin.json @@ -68,5 +68,5 @@ 0.012759859238077997 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/htpy.json b/docs/migration/bench/htpy.json index 46c3258b..0a1e404e 100644 --- a/docs/migration/bench/htpy.json +++ b/docs/migration/bench/htpy.json @@ -99,7 +99,12 @@ 0.008694864767394896 ] ], - "notes": { - "htpy": "concatenates a string rather than constructing a navigable tree, so nothing it builds can afterwards be queried or mutated" + "row_notes": { + "0": "concatenates a string rather than constructing a navigable tree, so nothing it builds can afterwards be queried or mutated", + "1": "concatenates a string rather than constructing a navigable tree, so nothing it builds can afterwards be queried or mutated", + "2": "concatenates a string rather than constructing a navigable tree, so nothing it builds can afterwards be queried or mutated", + "3": "concatenates a string rather than constructing a navigable tree, so nothing it builds can afterwards be queried or mutated", + "4": "concatenates a string rather than constructing a navigable tree, so nothing it builds can afterwards be queried or mutated", + "5": "concatenates a string rather than constructing a navigable tree, so nothing it builds can afterwards be queried or mutated" } } diff --git a/docs/migration/bench/hyperpython.json b/docs/migration/bench/hyperpython.json index a746c2e4..f5d97a00 100644 --- a/docs/migration/bench/hyperpython.json +++ b/docs/migration/bench/hyperpython.json @@ -99,5 +99,5 @@ 0.019251615715996477 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/inscriptis.json b/docs/migration/bench/inscriptis.json index 03fda4d2..cd933b07 100644 --- a/docs/migration/bench/inscriptis.json +++ b/docs/migration/bench/inscriptis.json @@ -39,5 +39,5 @@ 0.0437664506698881 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/jsmin.json b/docs/migration/bench/jsmin.json index 724136df..d7725cad 100644 --- a/docs/migration/bench/jsmin.json +++ b/docs/migration/bench/jsmin.json @@ -68,7 +68,10 @@ 0.01459254660160529 ] ], - "notes": { - "jsmin": "strips whitespace and comments without parsing the source, so it performs none of the renaming or structural compression the parsed minifiers do: on jQuery it emits 141.1 kB where turbohtml emits 87.8 kB" + "row_notes": { + "0": "strips whitespace and comments without parsing the source, so it performs none of the renaming or structural compression the parsed minifiers do: on jQuery it emits 141.1 kB where turbohtml emits 87.8 kB", + "1": "strips whitespace and comments without parsing the source, so it performs none of the renaming or structural compression the parsed minifiers do: on jQuery it emits 141.1 kB where turbohtml emits 87.8 kB", + "2": "strips whitespace and comments without parsing the source, so it performs none of the renaming or structural compression the parsed minifiers do: on jQuery it emits 141.1 kB where turbohtml emits 87.8 kB", + "3": "strips whitespace and comments without parsing the source, so it performs none of the renaming or structural compression the parsed minifiers do: on jQuery it emits 141.1 kB where turbohtml emits 87.8 kB" } } diff --git a/docs/migration/bench/justext.json b/docs/migration/bench/justext.json index 5905d45a..7d57b90d 100644 --- a/docs/migration/bench/justext.json +++ b/docs/migration/bench/justext.json @@ -39,5 +39,5 @@ 0.05705413454393241 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/lightningcss.json b/docs/migration/bench/lightningcss.json index d242c8b1..342337bd 100644 --- a/docs/migration/bench/lightningcss.json +++ b/docs/migration/bench/lightningcss.json @@ -82,5 +82,5 @@ 0.022578577927933375 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/linkify-it-py.json b/docs/migration/bench/linkify-it-py.json index 135fd41f..c2425e80 100644 --- a/docs/migration/bench/linkify-it-py.json +++ b/docs/migration/bench/linkify-it-py.json @@ -49,5 +49,5 @@ 0.10847816873948599 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/lxml-html-clean.json b/docs/migration/bench/lxml-html-clean.json index 4d9cad5c..d3914b8b 100644 --- a/docs/migration/bench/lxml-html-clean.json +++ b/docs/migration/bench/lxml-html-clean.json @@ -59,7 +59,9 @@ 0.1153154111287691 ] ], - "notes": { - "lxml-html-clean": "links URLs inside existing markup and never linkifies email addresses; on the prose case it produces no links at all, so that timing is the cost of finding nothing" + "row_notes": { + "2": "links URLs inside existing markup and never linkifies email addresses; on the prose case it produces no links at all, so that timing is the cost of finding nothing", + "3": "links URLs inside existing markup and never linkifies email addresses; on the prose case it produces no links at all, so that timing is the cost of finding nothing", + "4": "links URLs inside existing markup and never linkifies email addresses; on the prose case it produces no links at all, so that timing is the cost of finding nothing" } } diff --git a/docs/migration/bench/lxml.json b/docs/migration/bench/lxml.json index 73107abc..740e8a0e 100644 --- a/docs/migration/bench/lxml.json +++ b/docs/migration/bench/lxml.json @@ -1489,5 +1489,5 @@ 0.15330362335285372 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/markdownify.json b/docs/migration/bench/markdownify.json index 1e7b52ec..ebf07972 100644 --- a/docs/migration/bench/markdownify.json +++ b/docs/migration/bench/markdownify.json @@ -49,5 +49,5 @@ 0.11239601430404433 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/markupsafe.json b/docs/migration/bench/markupsafe.json index 79564afd..6a5b9d0c 100644 --- a/docs/migration/bench/markupsafe.json +++ b/docs/migration/bench/markupsafe.json @@ -99,5 +99,5 @@ 0.018830366328857713 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/markyp.json b/docs/migration/bench/markyp.json index da9ce225..23b63053 100644 --- a/docs/migration/bench/markyp.json +++ b/docs/migration/bench/markyp.json @@ -99,7 +99,12 @@ 0.004369252534352652 ] ], - "notes": { - "markyp": "concatenates a string rather than constructing a navigable tree, so nothing it builds can afterwards be queried or mutated" + "row_notes": { + "0": "concatenates a string rather than constructing a navigable tree, so nothing it builds can afterwards be queried or mutated", + "1": "concatenates a string rather than constructing a navigable tree, so nothing it builds can afterwards be queried or mutated", + "2": "concatenates a string rather than constructing a navigable tree, so nothing it builds can afterwards be queried or mutated", + "3": "concatenates a string rather than constructing a navigable tree, so nothing it builds can afterwards be queried or mutated", + "4": "concatenates a string rather than constructing a navigable tree, so nothing it builds can afterwards be queried or mutated", + "5": "concatenates a string rather than constructing a navigable tree, so nothing it builds can afterwards be queried or mutated" } } diff --git a/docs/migration/bench/metadata_parser.json b/docs/migration/bench/metadata_parser.json index 2870e63f..0957dae2 100644 --- a/docs/migration/bench/metadata_parser.json +++ b/docs/migration/bench/metadata_parser.json @@ -29,5 +29,5 @@ 0.0254309534022347 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/microdata.json b/docs/migration/bench/microdata.json index 2ffed659..35f84904 100644 --- a/docs/migration/bench/microdata.json +++ b/docs/migration/bench/microdata.json @@ -29,5 +29,5 @@ 0.12540731424751708 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/minify-html.json b/docs/migration/bench/minify-html.json index 3a17bf7e..5785bed7 100644 --- a/docs/migration/bench/minify-html.json +++ b/docs/migration/bench/minify-html.json @@ -68,5 +68,5 @@ 0.01800253451841663 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/news-please.json b/docs/migration/bench/news-please.json index 815b0234..dbf566e3 100644 --- a/docs/migration/bench/news-please.json +++ b/docs/migration/bench/news-please.json @@ -69,5 +69,5 @@ 0.034291817606264 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/newspaper3k.json b/docs/migration/bench/newspaper3k.json index ea8b1cc7..13592b55 100644 --- a/docs/migration/bench/newspaper3k.json +++ b/docs/migration/bench/newspaper3k.json @@ -69,5 +69,5 @@ 0.06384224146180023 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/nh3.json b/docs/migration/bench/nh3.json index 4f12e647..b7786618 100644 --- a/docs/migration/bench/nh3.json +++ b/docs/migration/bench/nh3.json @@ -129,5 +129,5 @@ 0.07293692554531789 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/opengraph.json b/docs/migration/bench/opengraph.json index be2bc559..64f37aae 100644 --- a/docs/migration/bench/opengraph.json +++ b/docs/migration/bench/opengraph.json @@ -29,5 +29,5 @@ 0.024326554863590765 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/pandas.json b/docs/migration/bench/pandas.json index aee6ee9b..49f56701 100644 --- a/docs/migration/bench/pandas.json +++ b/docs/migration/bench/pandas.json @@ -69,5 +69,5 @@ 0.09468311152836278 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/parse5.json b/docs/migration/bench/parse5.json index d634950a..b232f4e3 100644 --- a/docs/migration/bench/parse5.json +++ b/docs/migration/bench/parse5.json @@ -49,5 +49,5 @@ 0.0114561109118128 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/parsel.json b/docs/migration/bench/parsel.json index cfa54b1c..76ee3a54 100644 --- a/docs/migration/bench/parsel.json +++ b/docs/migration/bench/parsel.json @@ -789,5 +789,5 @@ 0.02801464412122382 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/pyquery.json b/docs/migration/bench/pyquery.json index d8619a78..91932580 100644 --- a/docs/migration/bench/pyquery.json +++ b/docs/migration/bench/pyquery.json @@ -919,5 +919,5 @@ 0.2631809071714095 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/rcssmin.json b/docs/migration/bench/rcssmin.json index 8237fb6e..ce702ce5 100644 --- a/docs/migration/bench/rcssmin.json +++ b/docs/migration/bench/rcssmin.json @@ -96,7 +96,12 @@ 0.029345195540840696 ] ], - "notes": { - "rcssmin": "strips whitespace and comments only, leaving #ffffff, 0px 0px 0px 0px and font-weight:bold unshortened where turbohtml writes #fff, 0 and 700" + "row_notes": { + "0": "strips whitespace and comments only, leaving #ffffff, 0px 0px 0px 0px and font-weight:bold unshortened where turbohtml writes #fff, 0 and 700", + "1": "strips whitespace and comments only, leaving #ffffff, 0px 0px 0px 0px and font-weight:bold unshortened where turbohtml writes #fff, 0 and 700", + "2": "strips whitespace and comments only, leaving #ffffff, 0px 0px 0px 0px and font-weight:bold unshortened where turbohtml writes #fff, 0 and 700", + "3": "strips whitespace and comments only, leaving #ffffff, 0px 0px 0px 0px and font-weight:bold unshortened where turbohtml writes #fff, 0 and 700", + "4": "strips whitespace and comments only, leaving #ffffff, 0px 0px 0px 0px and font-weight:bold unshortened where turbohtml writes #fff, 0 and 700", + "5": "strips whitespace and comments only, leaving #ffffff, 0px 0px 0px 0px and font-weight:bold unshortened where turbohtml writes #fff, 0 and 700" } } diff --git a/docs/migration/bench/readabilipy.json b/docs/migration/bench/readabilipy.json index b76bfcaf..6a1016d1 100644 --- a/docs/migration/bench/readabilipy.json +++ b/docs/migration/bench/readabilipy.json @@ -29,5 +29,5 @@ 0.2572053059847665 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/readability-lxml.json b/docs/migration/bench/readability-lxml.json index c61fd75e..f873584e 100644 --- a/docs/migration/bench/readability-lxml.json +++ b/docs/migration/bench/readability-lxml.json @@ -29,5 +29,5 @@ 0.09458795749284374 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/resiliparse.json b/docs/migration/bench/resiliparse.json index 0173969e..6b95e5bf 100644 --- a/docs/migration/bench/resiliparse.json +++ b/docs/migration/bench/resiliparse.json @@ -769,7 +769,14 @@ 0.016350123964366924 ] ], - "notes": { - "resiliparse": "reports about 11% fewer elements than every other parser here (876 against 989 on the mozilla page), so it collects text from a smaller tree" + "row_notes": { + "19": "reports about 11% fewer elements than every other parser here (876 against 989 on the mozilla page), so it collects text from a smaller tree", + "20": "reports about 11% fewer elements than every other parser here (876 against 989 on the mozilla page), so it collects text from a smaller tree", + "21": "reports about 11% fewer elements than every other parser here (876 against 989 on the mozilla page), so it collects text from a smaller tree", + "22": "reports about 11% fewer elements than every other parser here (876 against 989 on the mozilla page), so it collects text from a smaller tree", + "39": "reports about 11% fewer elements than every other parser here (876 against 989 on the mozilla page), so it collects text from a smaller tree", + "40": "reports about 11% fewer elements than every other parser here (876 against 989 on the mozilla page), so it collects text from a smaller tree", + "41": "reports about 11% fewer elements than every other parser here (876 against 989 on the mozilla page), so it collects text from a smaller tree", + "42": "reports about 11% fewer elements than every other parser here (876 against 989 on the mozilla page), so it collects text from a smaller tree" } } diff --git a/docs/migration/bench/rjsmin.json b/docs/migration/bench/rjsmin.json index d9619a79..a1785143 100644 --- a/docs/migration/bench/rjsmin.json +++ b/docs/migration/bench/rjsmin.json @@ -68,7 +68,10 @@ 0.015807151526946495 ] ], - "notes": { - "rjsmin": "strips whitespace and comments without parsing the source, so it performs none of the renaming or structural compression the parsed minifiers do: on jQuery it emits 141.1 kB where turbohtml emits 87.8 kB" + "row_notes": { + "0": "strips whitespace and comments without parsing the source, so it performs none of the renaming or structural compression the parsed minifiers do: on jQuery it emits 141.1 kB where turbohtml emits 87.8 kB", + "1": "strips whitespace and comments without parsing the source, so it performs none of the renaming or structural compression the parsed minifiers do: on jQuery it emits 141.1 kB where turbohtml emits 87.8 kB", + "2": "strips whitespace and comments without parsing the source, so it performs none of the renaming or structural compression the parsed minifiers do: on jQuery it emits 141.1 kB where turbohtml emits 87.8 kB", + "3": "strips whitespace and comments without parsing the source, so it performs none of the renaming or structural compression the parsed minifiers do: on jQuery it emits 141.1 kB where turbohtml emits 87.8 kB" } } diff --git a/docs/migration/bench/sanitize-html.json b/docs/migration/bench/sanitize-html.json index a92f9978..d916e9b2 100644 --- a/docs/migration/bench/sanitize-html.json +++ b/docs/migration/bench/sanitize-html.json @@ -19,5 +19,5 @@ 0.11172668998026558 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/selectolax.json b/docs/migration/bench/selectolax.json index 5a1d3f17..2fa7e438 100644 --- a/docs/migration/bench/selectolax.json +++ b/docs/migration/bench/selectolax.json @@ -799,5 +799,5 @@ 0.16291120403068735 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/simple-html.json b/docs/migration/bench/simple-html.json index 0ff43e5c..573125c6 100644 --- a/docs/migration/bench/simple-html.json +++ b/docs/migration/bench/simple-html.json @@ -99,7 +99,12 @@ 0.2111296673575497 ] ], - "notes": { - "simple-html": "concatenates a string rather than constructing a navigable tree, so nothing it builds can afterwards be queried or mutated" + "row_notes": { + "0": "concatenates a string rather than constructing a navigable tree, so nothing it builds can afterwards be queried or mutated", + "1": "concatenates a string rather than constructing a navigable tree, so nothing it builds can afterwards be queried or mutated", + "2": "concatenates a string rather than constructing a navigable tree, so nothing it builds can afterwards be queried or mutated", + "3": "concatenates a string rather than constructing a navigable tree, so nothing it builds can afterwards be queried or mutated", + "4": "concatenates a string rather than constructing a navigable tree, so nothing it builds can afterwards be queried or mutated", + "5": "concatenates a string rather than constructing a navigable tree, so nothing it builds can afterwards be queried or mutated" } } diff --git a/docs/migration/bench/soupsieve.json b/docs/migration/bench/soupsieve.json index 784e3623..88367c85 100644 --- a/docs/migration/bench/soupsieve.json +++ b/docs/migration/bench/soupsieve.json @@ -169,5 +169,5 @@ 0.01417893540370526 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/stdlib.json b/docs/migration/bench/stdlib.json index 374fc31f..cc0c2001 100644 --- a/docs/migration/bench/stdlib.json +++ b/docs/migration/bench/stdlib.json @@ -439,7 +439,11 @@ 0.02023617070051036 ] ], - "notes": { - "stdlib": "decodes with the nearest CPython codec under errors=replace, which is not the WHATWG decoder of that label: the two disagree on both the mapping tables and where decoding resumes after an error" + "row_notes": { + "38": "decodes with the nearest CPython codec under errors=replace, which is not the WHATWG decoder of that label: the two disagree on both the mapping tables and where decoding resumes after an error", + "39": "decodes with the nearest CPython codec under errors=replace, which is not the WHATWG decoder of that label: the two disagree on both the mapping tables and where decoding resumes after an error", + "40": "decodes with the nearest CPython codec under errors=replace, which is not the WHATWG decoder of that label: the two disagree on both the mapping tables and where decoding resumes after an error", + "41": "decodes with the nearest CPython codec under errors=replace, which is not the WHATWG decoder of that label: the two disagree on both the mapping tables and where decoding resumes after an error", + "42": "decodes with the nearest CPython codec under errors=replace, which is not the WHATWG decoder of that label: the two disagree on both the mapping tables and where decoding resumes after an error" } } diff --git a/docs/migration/bench/trafilatura.json b/docs/migration/bench/trafilatura.json index 839496c1..a8e691ee 100644 --- a/docs/migration/bench/trafilatura.json +++ b/docs/migration/bench/trafilatura.json @@ -69,5 +69,5 @@ 0.09149282207360948 ] ], - "notes": {} + "row_notes": {} } diff --git a/docs/migration/bench/w3lib.json b/docs/migration/bench/w3lib.json index ce220227..4fa44386 100644 --- a/docs/migration/bench/w3lib.json +++ b/docs/migration/bench/w3lib.json @@ -199,7 +199,10 @@ 0.04726044582904961 ] ], - "notes": { - "w3lib": "removes the tags with a regular expression over the raw string and never parses, so it cannot honor nesting or the tokenizer rules that decide where an element really ends" + "row_notes": { + "7": "removes the tags with a regular expression over the raw string and never parses, so it cannot honor nesting or the tokenizer rules that decide where an element really ends", + "8": "removes the tags with a regular expression over the raw string and never parses, so it cannot honor nesting or the tokenizer rules that decide where an element really ends", + "9": "removes the tags with a regular expression over the raw string and never parses, so it cannot honor nesting or the tokenizer rules that decide where an element really ends", + "10": "removes the tags with a regular expression over the raw string and never parses, so it cannot honor nesting or the tokenizer rules that decide where an element really ends" } } diff --git a/docs/migration/bench/yattag.json b/docs/migration/bench/yattag.json index 4c641146..d6ebf07e 100644 --- a/docs/migration/bench/yattag.json +++ b/docs/migration/bench/yattag.json @@ -39,7 +39,9 @@ 0.013404825323645687 ] ], - "notes": { - "yattag": "concatenates a string rather than constructing a navigable tree, so nothing it builds can afterwards be queried or mutated" + "row_notes": { + "0": "concatenates a string rather than constructing a navigable tree, so nothing it builds can afterwards be queried or mutated", + "1": "concatenates a string rather than constructing a navigable tree, so nothing it builds can afterwards be queried or mutated", + "2": "concatenates a string rather than constructing a navigable tree, so nothing it builds can afterwards be queried or mutated" } } diff --git a/tests/test_bench_migration.py b/tests/test_bench_migration.py index 31e2db45..a0d97c4d 100644 --- a/tests/test_bench_migration.py +++ b/tests/test_bench_migration.py @@ -93,7 +93,13 @@ def test_operation_one_backend_skips_keeps_the_row(tmp_path: Path) -> None: "encoding|bytes|bs4 (html.parser)": {"mean": 8.0, "cv": 0.02}, }, ) - assert ["detect a byte stream's encoding — bytes", 1.0, 8.0, None] in feed["rows"] + # the configuration that skips the operation says why rather than leaving the cell blank + assert [ + "detect a byte stream's encoding — bytes", + 1.0, + 8.0, + "same in both configurations, so measured once", + ] in feed["rows"] def test_spread_aligns_with_every_variant(tmp_path: Path) -> None: diff --git a/tools/bench/migration.py b/tools/bench/migration.py index 7b87587d..1548a20a 100644 --- a/tools/bench/migration.py +++ b/tools/bench/migration.py @@ -33,6 +33,10 @@ _MEMORY_OPS: Final = frozenset({"parse-dense", "rewrite"}) _WIDE_OPS: Final = _SIZE_OPS | _MEMORY_OPS +# Why a cell is empty when the library is measured in more than one configuration: the operation runs the same +# code whichever one is chosen, so benchmarking it twice would publish a duplicate column. +_SHARED_WITH_OTHER_VARIANT: Final = "same in both configurations, so measured once" + # A competitor module's stem maps to its migration page's slug by turning ``_`` into ``-``; these libraries # name the two differently, so the map pins them. A stem with no ``docs/migration/.rst`` is skipped, so # columns that only appear inside another library's table (cchardet, cssmin, css_html_js_minify) fall out. @@ -85,7 +89,7 @@ def _case_names(operation: str, stats: dict[str, dict[str, float]]) -> list[str] def _rows( variants: list[dict[str, str]], stats: dict[str, dict[str, float]] -) -> tuple[list[list[str | float | None]], list[list[float | None]]]: +) -> tuple[list[list[str | float | None]], list[list[float | None]], list[str | None]]: """ Build a library's rows and their spread across every shared operation-case, prefixing labels that span ops. @@ -94,12 +98,14 @@ def _rows( does not measure an operation leaves its cells empty rather than dropping the row for the ones that do. """ covered = {operation for variant in variants for operation in variant} + labels = [next(iter(variant.values())) for variant in variants] prefixed = len(covered) > 1 # a leading metric only makes sense when every shared operation carries one; a library that spans a memory # operation and timing-only ones would otherwise emit rows of two different widths into one table wide = bool(_leading_metric(covered)) rows: list[list[str | float | None]] = [] spread: list[list[float | None]] = [] + caveats: list[str | None] = [] for operation, meta in ((op, operations.OPERATIONS[op]) for op in operations.OPERATIONS if op in covered): for case in _case_names(operation, stats): turbo = stats.get(f"{operation}|{case}|turbohtml") @@ -110,22 +116,29 @@ def _rows( if turbo is None or all(other is None for other in others): continue row_label = rst_safe(f"{meta.title} — {case}" if prefixed else case) + # a configuration that skips an operation another one measures skips it because the two run the same + # code there, so the cell says that rather than leaving an unexplained blank + absent: list[str | None] = [ + None if operation in variant else _SHARED_WITH_OTHER_VARIANT for variant in variants + ] if wide: lead = "size" if operation in _SIZE_OPS else "peak" row: list[str | float | None] = [row_label, turbo[lead], turbo["mean"]] noise: list[float | None] = [None, None, turbo.get("cv")] - for other in others: - row += [None, None] if other is None else [other[lead], other["mean"]] + for other, gap in zip(others, absent, strict=True): + row += [gap, gap] if other is None else [other[lead], other["mean"]] noise += [None, None if other is None else other.get("cv")] else: row = [row_label, turbo["mean"]] noise = [None, turbo.get("cv")] - for other in others: - row.append(None if other is None else other["mean"]) + for other, gap in zip(others, absent, strict=True): + row.append(gap if other is None else other["mean"]) noise.append(None if other is None else other.get("cv")) rows.append(row) spread.append(noise) - return rows, spread + # a library page mixes operations, so a caveat belongs on the rows of the operation it describes + caveats.append(next((NOTES[operation][label] for label in labels if label in NOTES.get(operation, {})), None)) + return rows, spread, caveats def _leading_metric(ops: Iterable[str]) -> list[str]: @@ -159,7 +172,7 @@ def emit_migration_feeds( for slug, variants in by_slug.items(): if not (docs_root / "migration" / f"{slug}.rst").exists(): continue - rows, spread = _rows(variants, stats) + rows, spread, caveats = _rows(variants, stats) if not rows: skipped.append(slug) continue @@ -170,13 +183,9 @@ def emit_migration_feeds( "metrics": _leading_metric({operation for variant in variants for operation in variant}), "rows": rows, "spread": spread, - # a migration page mixes operations, so a note applies once the library is noted for any of them - "notes": { - label: note - for variant, label in zip(variants, labels, strict=True) - for operation in variant - if (note := NOTES.get(operation, {}).get(label)) is not None - }, + # keyed by row rather than by column: a library page mixes operations, and a caveat that belongs to one + # of them would otherwise read as covering every row the library appears in + "row_notes": {str(index): note for index, note in enumerate(caveats) if note is not None}, } (directory / f"{slug}.json").write_text(json.dumps(feed, indent=2, ensure_ascii=False) + "\n", "utf-8") return skipped From f14cd5b0f8f1209f7d083cb45811fa9cfceb8a55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Mon, 20 Jul 2026 21:10:16 -0700 Subject: [PATCH 03/54] docs(bench): note trafilatura gives up on dates --- tools/bench/notes.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tools/bench/notes.py b/tools/bench/notes.py index 6d83c723..7b93d1f8 100644 --- a/tools/bench/notes.py +++ b/tools/bench/notes.py @@ -50,12 +50,11 @@ "links at all, so that timing is the cost of finding nothing" ), }, - "date": { - "htmldate": ( - "returns no date for the 100-candidate case, so its timing there is the cost of giving up rather than of " - "finding the date turbohtml reports" - ), - }, + "date": dict.fromkeys( + ("htmldate", "trafilatura"), + "returns no date for the 100-candidate case, so its timing there is the cost of giving up rather than of " + "finding the date turbohtml reports", + ), "text-content": { "resiliparse": ( "reports about 11% fewer elements than every other parser here (876 against 989 on the mozilla page), so " From fd08512200c4383233b0bdc6538de60d32fe9c4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Mon, 20 Jul 2026 21:19:51 -0700 Subject: [PATCH 04/54] =?UTF-8?q?=F0=9F=90=9B=20fix(stubs):=20declare=20th?= =?UTF-8?q?e=20new=20URL=20entry=20points?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The urls port added _url_is_tracker and _url_remove_dot_segments to the module without declaring them in the stubs, so the type gate rejected their imports. Declared in features.pyi and re-exported through the _html.pyi aggregate. --- src/turbohtml/_html.pyi | 6 ++++++ src/turbohtml/_stubs/features.pyi | 2 ++ 2 files changed, 8 insertions(+) diff --git a/src/turbohtml/_html.pyi b/src/turbohtml/_html.pyi index cbfe671c..6ec8c1c4 100644 --- a/src/turbohtml/_html.pyi +++ b/src/turbohtml/_html.pyi @@ -149,6 +149,9 @@ from ._stubs.features import ( from ._stubs.features import ( _sanitize as _sanitize, ) +from ._stubs.features import ( + _url_is_tracker as _url_is_tracker, +) from ._stubs.features import ( _url_join as _url_join, ) @@ -158,6 +161,9 @@ from ._stubs.features import ( from ._stubs.features import ( _url_percent_encode as _url_percent_encode, ) +from ._stubs.features import ( + _url_remove_dot_segments as _url_remove_dot_segments, +) from ._stubs.features import ( _url_split as _url_split, ) diff --git a/src/turbohtml/_stubs/features.pyi b/src/turbohtml/_stubs/features.pyi index 1867fee3..13485c44 100644 --- a/src/turbohtml/_stubs/features.pyi +++ b/src/turbohtml/_stubs/features.pyi @@ -33,6 +33,8 @@ def _url_split(url: str, /) -> tuple[str, str, str, str, str, str, str, str, boo def _url_percent_encode(text: str, url_set: int, /) -> str: ... def _url_percent_decode(text: str, /) -> str: ... def _url_join(base: str, target: str, /) -> str: ... +def _url_is_tracker(key: str, /) -> bool: ... +def _url_remove_dot_segments(path: str, /) -> str: ... def _url_to_ascii(host: str, /) -> str: ... def _register_links(link_type: type, /) -> None: ... def _register_structured_data( From 99265f66389628230e8f9818f73f22d4ebbd4712 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Mon, 20 Jul 2026 21:20:28 -0700 Subject: [PATCH 05/54] =?UTF-8?q?=F0=9F=8E=A8=20style:=20apply=20formatter?= =?UTF-8?q?=20fixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/turbohtml/_c/url/url.c | 10 +++++----- tools/bench/migration.py | 4 +++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/turbohtml/_c/url/url.c b/src/turbohtml/_c/url/url.c index b616466d..145e4faa 100644 --- a/src/turbohtml/_c/url/url.c +++ b/src/turbohtml/_c/url/url.c @@ -803,9 +803,9 @@ PyObject *turbohtml_url_join(PyObject *Py_UNUSED(module), PyObject *args) { referral, not the content, so two URLs differing only in them address the same page. Kept sorted so the exact-name test is a binary search. */ static const char *const TRACKER_NAMES[] = { - "clickid", "dclid", "efid", "epik", "fb_ref", "fb_source", "fbclid", "gbraid", - "gclid", "gclsrc", "igsh", "igshid", "mkt_tok", "msclkid", "partnerid", "s_cid", - "sc_cid", "ttclid", "twclid", "wbraid", "wickedid", "yclid", "ysclid", + "clickid", "dclid", "efid", "epik", "fb_ref", "fb_source", "fbclid", "gbraid", + "gclid", "gclsrc", "igsh", "igshid", "mkt_tok", "msclkid", "partnerid", "s_cid", + "sc_cid", "ttclid", "twclid", "wbraid", "wickedid", "yclid", "ysclid", }; static const char *const TRACKER_PREFIXES[] = { @@ -815,8 +815,8 @@ static const char *const TRACKER_PREFIXES[] = { /* The words a tracking key is built from, matched as a whole underscore-delimited word rather than a substring so "reference" is not read as "ref". */ static const char *const TRACKER_WORDS[] = { - "aff", "affi", "affiliate", "campaign", "cid", "clid", "keyword", "kwd", - "medium", "ref", "refer", "referer", "referrer", "session", "source", "uid", "xtor", + "aff", "affi", "affiliate", "campaign", "cid", "clid", "keyword", "kwd", "medium", + "ref", "refer", "referer", "referrer", "session", "source", "uid", "xtor", }; static int tracker_name_known(const char *key, size_t len) { diff --git a/tools/bench/migration.py b/tools/bench/migration.py index 1548a20a..471913e3 100644 --- a/tools/bench/migration.py +++ b/tools/bench/migration.py @@ -137,7 +137,9 @@ def _rows( rows.append(row) spread.append(noise) # a library page mixes operations, so a caveat belongs on the rows of the operation it describes - caveats.append(next((NOTES[operation][label] for label in labels if label in NOTES.get(operation, {})), None)) + caveats.append( + next((NOTES[operation][label] for label in labels if label in NOTES.get(operation, {})), None) + ) return rows, spread, caveats From 13983f4df1122f09c907a802a4b60819ba0bd26d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Mon, 20 Jul 2026 21:19:51 -0700 Subject: [PATCH 06/54] =?UTF-8?q?=F0=9F=90=9B=20fix(stubs):=20declare=20th?= =?UTF-8?q?e=20new=20URL=20entry=20points?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The urls port added _url_is_tracker and _url_remove_dot_segments to the module without declaring them in the stubs, so the type gate rejected their imports. Declared in features.pyi and re-exported through the _html.pyi aggregate. --- src/turbohtml/_html.pyi | 6 ++++++ src/turbohtml/_stubs/features.pyi | 2 ++ 2 files changed, 8 insertions(+) diff --git a/src/turbohtml/_html.pyi b/src/turbohtml/_html.pyi index cbfe671c..6ec8c1c4 100644 --- a/src/turbohtml/_html.pyi +++ b/src/turbohtml/_html.pyi @@ -149,6 +149,9 @@ from ._stubs.features import ( from ._stubs.features import ( _sanitize as _sanitize, ) +from ._stubs.features import ( + _url_is_tracker as _url_is_tracker, +) from ._stubs.features import ( _url_join as _url_join, ) @@ -158,6 +161,9 @@ from ._stubs.features import ( from ._stubs.features import ( _url_percent_encode as _url_percent_encode, ) +from ._stubs.features import ( + _url_remove_dot_segments as _url_remove_dot_segments, +) from ._stubs.features import ( _url_split as _url_split, ) diff --git a/src/turbohtml/_stubs/features.pyi b/src/turbohtml/_stubs/features.pyi index 1867fee3..13485c44 100644 --- a/src/turbohtml/_stubs/features.pyi +++ b/src/turbohtml/_stubs/features.pyi @@ -33,6 +33,8 @@ def _url_split(url: str, /) -> tuple[str, str, str, str, str, str, str, str, boo def _url_percent_encode(text: str, url_set: int, /) -> str: ... def _url_percent_decode(text: str, /) -> str: ... def _url_join(base: str, target: str, /) -> str: ... +def _url_is_tracker(key: str, /) -> bool: ... +def _url_remove_dot_segments(path: str, /) -> str: ... def _url_to_ascii(host: str, /) -> str: ... def _register_links(link_type: type, /) -> None: ... def _register_structured_data( From 0ad958481145ce0e9adcaba1277f37f81e313587 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Mon, 20 Jul 2026 21:20:28 -0700 Subject: [PATCH 07/54] =?UTF-8?q?=F0=9F=8E=A8=20style:=20apply=20formatter?= =?UTF-8?q?=20fixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/turbohtml/_c/url/url.c | 10 +++++----- tools/bench/migration.py | 4 +++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/turbohtml/_c/url/url.c b/src/turbohtml/_c/url/url.c index b616466d..145e4faa 100644 --- a/src/turbohtml/_c/url/url.c +++ b/src/turbohtml/_c/url/url.c @@ -803,9 +803,9 @@ PyObject *turbohtml_url_join(PyObject *Py_UNUSED(module), PyObject *args) { referral, not the content, so two URLs differing only in them address the same page. Kept sorted so the exact-name test is a binary search. */ static const char *const TRACKER_NAMES[] = { - "clickid", "dclid", "efid", "epik", "fb_ref", "fb_source", "fbclid", "gbraid", - "gclid", "gclsrc", "igsh", "igshid", "mkt_tok", "msclkid", "partnerid", "s_cid", - "sc_cid", "ttclid", "twclid", "wbraid", "wickedid", "yclid", "ysclid", + "clickid", "dclid", "efid", "epik", "fb_ref", "fb_source", "fbclid", "gbraid", + "gclid", "gclsrc", "igsh", "igshid", "mkt_tok", "msclkid", "partnerid", "s_cid", + "sc_cid", "ttclid", "twclid", "wbraid", "wickedid", "yclid", "ysclid", }; static const char *const TRACKER_PREFIXES[] = { @@ -815,8 +815,8 @@ static const char *const TRACKER_PREFIXES[] = { /* The words a tracking key is built from, matched as a whole underscore-delimited word rather than a substring so "reference" is not read as "ref". */ static const char *const TRACKER_WORDS[] = { - "aff", "affi", "affiliate", "campaign", "cid", "clid", "keyword", "kwd", - "medium", "ref", "refer", "referer", "referrer", "session", "source", "uid", "xtor", + "aff", "affi", "affiliate", "campaign", "cid", "clid", "keyword", "kwd", "medium", + "ref", "refer", "referer", "referrer", "session", "source", "uid", "xtor", }; static int tracker_name_known(const char *key, size_t len) { diff --git a/tools/bench/migration.py b/tools/bench/migration.py index 1548a20a..471913e3 100644 --- a/tools/bench/migration.py +++ b/tools/bench/migration.py @@ -137,7 +137,9 @@ def _rows( rows.append(row) spread.append(noise) # a library page mixes operations, so a caveat belongs on the rows of the operation it describes - caveats.append(next((NOTES[operation][label] for label in labels if label in NOTES.get(operation, {})), None)) + caveats.append( + next((NOTES[operation][label] for label in labels if label in NOTES.get(operation, {})), None) + ) return rows, spread, caveats From cb94b1fd97bd252eda25780c84642acbc43d855c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Mon, 20 Jul 2026 21:19:51 -0700 Subject: [PATCH 08/54] =?UTF-8?q?=F0=9F=90=9B=20fix(stubs):=20declare=20th?= =?UTF-8?q?e=20new=20URL=20entry=20points?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The urls port added _url_is_tracker and _url_remove_dot_segments to the module without declaring them in the stubs, so the type gate rejected their imports. Declared in features.pyi and re-exported through the _html.pyi aggregate. --- src/turbohtml/_html.pyi | 6 ++++++ src/turbohtml/_stubs/features.pyi | 2 ++ 2 files changed, 8 insertions(+) diff --git a/src/turbohtml/_html.pyi b/src/turbohtml/_html.pyi index cbfe671c..6ec8c1c4 100644 --- a/src/turbohtml/_html.pyi +++ b/src/turbohtml/_html.pyi @@ -149,6 +149,9 @@ from ._stubs.features import ( from ._stubs.features import ( _sanitize as _sanitize, ) +from ._stubs.features import ( + _url_is_tracker as _url_is_tracker, +) from ._stubs.features import ( _url_join as _url_join, ) @@ -158,6 +161,9 @@ from ._stubs.features import ( from ._stubs.features import ( _url_percent_encode as _url_percent_encode, ) +from ._stubs.features import ( + _url_remove_dot_segments as _url_remove_dot_segments, +) from ._stubs.features import ( _url_split as _url_split, ) diff --git a/src/turbohtml/_stubs/features.pyi b/src/turbohtml/_stubs/features.pyi index 1867fee3..13485c44 100644 --- a/src/turbohtml/_stubs/features.pyi +++ b/src/turbohtml/_stubs/features.pyi @@ -33,6 +33,8 @@ def _url_split(url: str, /) -> tuple[str, str, str, str, str, str, str, str, boo def _url_percent_encode(text: str, url_set: int, /) -> str: ... def _url_percent_decode(text: str, /) -> str: ... def _url_join(base: str, target: str, /) -> str: ... +def _url_is_tracker(key: str, /) -> bool: ... +def _url_remove_dot_segments(path: str, /) -> str: ... def _url_to_ascii(host: str, /) -> str: ... def _register_links(link_type: type, /) -> None: ... def _register_structured_data( From 78c1fab6f702648472c43d05c8b7829682824f4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Mon, 20 Jul 2026 21:20:28 -0700 Subject: [PATCH 09/54] =?UTF-8?q?=F0=9F=8E=A8=20style:=20apply=20formatter?= =?UTF-8?q?=20fixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/turbohtml/_c/url/url.c | 10 +++++----- tools/bench/migration.py | 4 +++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/turbohtml/_c/url/url.c b/src/turbohtml/_c/url/url.c index b616466d..145e4faa 100644 --- a/src/turbohtml/_c/url/url.c +++ b/src/turbohtml/_c/url/url.c @@ -803,9 +803,9 @@ PyObject *turbohtml_url_join(PyObject *Py_UNUSED(module), PyObject *args) { referral, not the content, so two URLs differing only in them address the same page. Kept sorted so the exact-name test is a binary search. */ static const char *const TRACKER_NAMES[] = { - "clickid", "dclid", "efid", "epik", "fb_ref", "fb_source", "fbclid", "gbraid", - "gclid", "gclsrc", "igsh", "igshid", "mkt_tok", "msclkid", "partnerid", "s_cid", - "sc_cid", "ttclid", "twclid", "wbraid", "wickedid", "yclid", "ysclid", + "clickid", "dclid", "efid", "epik", "fb_ref", "fb_source", "fbclid", "gbraid", + "gclid", "gclsrc", "igsh", "igshid", "mkt_tok", "msclkid", "partnerid", "s_cid", + "sc_cid", "ttclid", "twclid", "wbraid", "wickedid", "yclid", "ysclid", }; static const char *const TRACKER_PREFIXES[] = { @@ -815,8 +815,8 @@ static const char *const TRACKER_PREFIXES[] = { /* The words a tracking key is built from, matched as a whole underscore-delimited word rather than a substring so "reference" is not read as "ref". */ static const char *const TRACKER_WORDS[] = { - "aff", "affi", "affiliate", "campaign", "cid", "clid", "keyword", "kwd", - "medium", "ref", "refer", "referer", "referrer", "session", "source", "uid", "xtor", + "aff", "affi", "affiliate", "campaign", "cid", "clid", "keyword", "kwd", "medium", + "ref", "refer", "referer", "referrer", "session", "source", "uid", "xtor", }; static int tracker_name_known(const char *key, size_t len) { diff --git a/tools/bench/migration.py b/tools/bench/migration.py index 1548a20a..471913e3 100644 --- a/tools/bench/migration.py +++ b/tools/bench/migration.py @@ -137,7 +137,9 @@ def _rows( rows.append(row) spread.append(noise) # a library page mixes operations, so a caveat belongs on the rows of the operation it describes - caveats.append(next((NOTES[operation][label] for label in labels if label in NOTES.get(operation, {})), None)) + caveats.append( + next((NOTES[operation][label] for label in labels if label in NOTES.get(operation, {})), None) + ) return rows, spread, caveats From 502f962555f66cb8b22f6c3a529a62c5c819497c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Mon, 20 Jul 2026 21:19:51 -0700 Subject: [PATCH 10/54] =?UTF-8?q?=F0=9F=90=9B=20fix(stubs):=20declare=20th?= =?UTF-8?q?e=20new=20URL=20entry=20points?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The urls port added _url_is_tracker and _url_remove_dot_segments to the module without declaring them in the stubs, so the type gate rejected their imports. Declared in features.pyi and re-exported through the _html.pyi aggregate. --- src/turbohtml/_html.pyi | 6 ++++++ src/turbohtml/_stubs/features.pyi | 2 ++ 2 files changed, 8 insertions(+) diff --git a/src/turbohtml/_html.pyi b/src/turbohtml/_html.pyi index cbfe671c..6ec8c1c4 100644 --- a/src/turbohtml/_html.pyi +++ b/src/turbohtml/_html.pyi @@ -149,6 +149,9 @@ from ._stubs.features import ( from ._stubs.features import ( _sanitize as _sanitize, ) +from ._stubs.features import ( + _url_is_tracker as _url_is_tracker, +) from ._stubs.features import ( _url_join as _url_join, ) @@ -158,6 +161,9 @@ from ._stubs.features import ( from ._stubs.features import ( _url_percent_encode as _url_percent_encode, ) +from ._stubs.features import ( + _url_remove_dot_segments as _url_remove_dot_segments, +) from ._stubs.features import ( _url_split as _url_split, ) diff --git a/src/turbohtml/_stubs/features.pyi b/src/turbohtml/_stubs/features.pyi index 1867fee3..13485c44 100644 --- a/src/turbohtml/_stubs/features.pyi +++ b/src/turbohtml/_stubs/features.pyi @@ -33,6 +33,8 @@ def _url_split(url: str, /) -> tuple[str, str, str, str, str, str, str, str, boo def _url_percent_encode(text: str, url_set: int, /) -> str: ... def _url_percent_decode(text: str, /) -> str: ... def _url_join(base: str, target: str, /) -> str: ... +def _url_is_tracker(key: str, /) -> bool: ... +def _url_remove_dot_segments(path: str, /) -> str: ... def _url_to_ascii(host: str, /) -> str: ... def _register_links(link_type: type, /) -> None: ... def _register_structured_data( From 870860039614d0df9ca3cb62c941f4d8fe792abd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Mon, 20 Jul 2026 21:20:28 -0700 Subject: [PATCH 11/54] =?UTF-8?q?=F0=9F=8E=A8=20style:=20apply=20formatter?= =?UTF-8?q?=20fixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/turbohtml/_c/url/url.c | 10 +++++----- tools/bench/migration.py | 4 +++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/turbohtml/_c/url/url.c b/src/turbohtml/_c/url/url.c index b616466d..145e4faa 100644 --- a/src/turbohtml/_c/url/url.c +++ b/src/turbohtml/_c/url/url.c @@ -803,9 +803,9 @@ PyObject *turbohtml_url_join(PyObject *Py_UNUSED(module), PyObject *args) { referral, not the content, so two URLs differing only in them address the same page. Kept sorted so the exact-name test is a binary search. */ static const char *const TRACKER_NAMES[] = { - "clickid", "dclid", "efid", "epik", "fb_ref", "fb_source", "fbclid", "gbraid", - "gclid", "gclsrc", "igsh", "igshid", "mkt_tok", "msclkid", "partnerid", "s_cid", - "sc_cid", "ttclid", "twclid", "wbraid", "wickedid", "yclid", "ysclid", + "clickid", "dclid", "efid", "epik", "fb_ref", "fb_source", "fbclid", "gbraid", + "gclid", "gclsrc", "igsh", "igshid", "mkt_tok", "msclkid", "partnerid", "s_cid", + "sc_cid", "ttclid", "twclid", "wbraid", "wickedid", "yclid", "ysclid", }; static const char *const TRACKER_PREFIXES[] = { @@ -815,8 +815,8 @@ static const char *const TRACKER_PREFIXES[] = { /* The words a tracking key is built from, matched as a whole underscore-delimited word rather than a substring so "reference" is not read as "ref". */ static const char *const TRACKER_WORDS[] = { - "aff", "affi", "affiliate", "campaign", "cid", "clid", "keyword", "kwd", - "medium", "ref", "refer", "referer", "referrer", "session", "source", "uid", "xtor", + "aff", "affi", "affiliate", "campaign", "cid", "clid", "keyword", "kwd", "medium", + "ref", "refer", "referer", "referrer", "session", "source", "uid", "xtor", }; static int tracker_name_known(const char *key, size_t len) { diff --git a/tools/bench/migration.py b/tools/bench/migration.py index 1548a20a..471913e3 100644 --- a/tools/bench/migration.py +++ b/tools/bench/migration.py @@ -137,7 +137,9 @@ def _rows( rows.append(row) spread.append(noise) # a library page mixes operations, so a caveat belongs on the rows of the operation it describes - caveats.append(next((NOTES[operation][label] for label in labels if label in NOTES.get(operation, {})), None)) + caveats.append( + next((NOTES[operation][label] for label in labels if label in NOTES.get(operation, {})), None) + ) return rows, spread, caveats From fc65465a0920aed59ce97a304e2767118405666d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Mon, 20 Jul 2026 21:19:51 -0700 Subject: [PATCH 12/54] =?UTF-8?q?=F0=9F=90=9B=20fix(stubs):=20declare=20th?= =?UTF-8?q?e=20new=20URL=20entry=20points?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The urls port added _url_is_tracker and _url_remove_dot_segments to the module without declaring them in the stubs, so the type gate rejected their imports. Declared in features.pyi and re-exported through the _html.pyi aggregate. --- src/turbohtml/_html.pyi | 6 ++++++ src/turbohtml/_stubs/features.pyi | 2 ++ 2 files changed, 8 insertions(+) diff --git a/src/turbohtml/_html.pyi b/src/turbohtml/_html.pyi index cbfe671c..6ec8c1c4 100644 --- a/src/turbohtml/_html.pyi +++ b/src/turbohtml/_html.pyi @@ -149,6 +149,9 @@ from ._stubs.features import ( from ._stubs.features import ( _sanitize as _sanitize, ) +from ._stubs.features import ( + _url_is_tracker as _url_is_tracker, +) from ._stubs.features import ( _url_join as _url_join, ) @@ -158,6 +161,9 @@ from ._stubs.features import ( from ._stubs.features import ( _url_percent_encode as _url_percent_encode, ) +from ._stubs.features import ( + _url_remove_dot_segments as _url_remove_dot_segments, +) from ._stubs.features import ( _url_split as _url_split, ) diff --git a/src/turbohtml/_stubs/features.pyi b/src/turbohtml/_stubs/features.pyi index 1867fee3..13485c44 100644 --- a/src/turbohtml/_stubs/features.pyi +++ b/src/turbohtml/_stubs/features.pyi @@ -33,6 +33,8 @@ def _url_split(url: str, /) -> tuple[str, str, str, str, str, str, str, str, boo def _url_percent_encode(text: str, url_set: int, /) -> str: ... def _url_percent_decode(text: str, /) -> str: ... def _url_join(base: str, target: str, /) -> str: ... +def _url_is_tracker(key: str, /) -> bool: ... +def _url_remove_dot_segments(path: str, /) -> str: ... def _url_to_ascii(host: str, /) -> str: ... def _register_links(link_type: type, /) -> None: ... def _register_structured_data( From 08172d478455233807f993473a89cb0982d64c79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Mon, 20 Jul 2026 21:20:28 -0700 Subject: [PATCH 13/54] =?UTF-8?q?=F0=9F=8E=A8=20style:=20apply=20formatter?= =?UTF-8?q?=20fixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/turbohtml/_c/url/url.c | 10 +++++----- tools/bench/migration.py | 4 +++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/turbohtml/_c/url/url.c b/src/turbohtml/_c/url/url.c index b616466d..145e4faa 100644 --- a/src/turbohtml/_c/url/url.c +++ b/src/turbohtml/_c/url/url.c @@ -803,9 +803,9 @@ PyObject *turbohtml_url_join(PyObject *Py_UNUSED(module), PyObject *args) { referral, not the content, so two URLs differing only in them address the same page. Kept sorted so the exact-name test is a binary search. */ static const char *const TRACKER_NAMES[] = { - "clickid", "dclid", "efid", "epik", "fb_ref", "fb_source", "fbclid", "gbraid", - "gclid", "gclsrc", "igsh", "igshid", "mkt_tok", "msclkid", "partnerid", "s_cid", - "sc_cid", "ttclid", "twclid", "wbraid", "wickedid", "yclid", "ysclid", + "clickid", "dclid", "efid", "epik", "fb_ref", "fb_source", "fbclid", "gbraid", + "gclid", "gclsrc", "igsh", "igshid", "mkt_tok", "msclkid", "partnerid", "s_cid", + "sc_cid", "ttclid", "twclid", "wbraid", "wickedid", "yclid", "ysclid", }; static const char *const TRACKER_PREFIXES[] = { @@ -815,8 +815,8 @@ static const char *const TRACKER_PREFIXES[] = { /* The words a tracking key is built from, matched as a whole underscore-delimited word rather than a substring so "reference" is not read as "ref". */ static const char *const TRACKER_WORDS[] = { - "aff", "affi", "affiliate", "campaign", "cid", "clid", "keyword", "kwd", - "medium", "ref", "refer", "referer", "referrer", "session", "source", "uid", "xtor", + "aff", "affi", "affiliate", "campaign", "cid", "clid", "keyword", "kwd", "medium", + "ref", "refer", "referer", "referrer", "session", "source", "uid", "xtor", }; static int tracker_name_known(const char *key, size_t len) { diff --git a/tools/bench/migration.py b/tools/bench/migration.py index 1548a20a..471913e3 100644 --- a/tools/bench/migration.py +++ b/tools/bench/migration.py @@ -137,7 +137,9 @@ def _rows( rows.append(row) spread.append(noise) # a library page mixes operations, so a caveat belongs on the rows of the operation it describes - caveats.append(next((NOTES[operation][label] for label in labels if label in NOTES.get(operation, {})), None)) + caveats.append( + next((NOTES[operation][label] for label in labels if label in NOTES.get(operation, {})), None) + ) return rows, spread, caveats From 1efc5b7bd58665879fa7ca8e52fa445dbf2c7df4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Mon, 20 Jul 2026 21:19:51 -0700 Subject: [PATCH 14/54] =?UTF-8?q?=F0=9F=90=9B=20fix(stubs):=20declare=20th?= =?UTF-8?q?e=20new=20URL=20entry=20points?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The urls port added _url_is_tracker and _url_remove_dot_segments to the module without declaring them in the stubs, so the type gate rejected their imports. Declared in features.pyi and re-exported through the _html.pyi aggregate. --- src/turbohtml/_html.pyi | 6 ++++++ src/turbohtml/_stubs/features.pyi | 2 ++ 2 files changed, 8 insertions(+) diff --git a/src/turbohtml/_html.pyi b/src/turbohtml/_html.pyi index cbfe671c..6ec8c1c4 100644 --- a/src/turbohtml/_html.pyi +++ b/src/turbohtml/_html.pyi @@ -149,6 +149,9 @@ from ._stubs.features import ( from ._stubs.features import ( _sanitize as _sanitize, ) +from ._stubs.features import ( + _url_is_tracker as _url_is_tracker, +) from ._stubs.features import ( _url_join as _url_join, ) @@ -158,6 +161,9 @@ from ._stubs.features import ( from ._stubs.features import ( _url_percent_encode as _url_percent_encode, ) +from ._stubs.features import ( + _url_remove_dot_segments as _url_remove_dot_segments, +) from ._stubs.features import ( _url_split as _url_split, ) diff --git a/src/turbohtml/_stubs/features.pyi b/src/turbohtml/_stubs/features.pyi index 1867fee3..13485c44 100644 --- a/src/turbohtml/_stubs/features.pyi +++ b/src/turbohtml/_stubs/features.pyi @@ -33,6 +33,8 @@ def _url_split(url: str, /) -> tuple[str, str, str, str, str, str, str, str, boo def _url_percent_encode(text: str, url_set: int, /) -> str: ... def _url_percent_decode(text: str, /) -> str: ... def _url_join(base: str, target: str, /) -> str: ... +def _url_is_tracker(key: str, /) -> bool: ... +def _url_remove_dot_segments(path: str, /) -> str: ... def _url_to_ascii(host: str, /) -> str: ... def _register_links(link_type: type, /) -> None: ... def _register_structured_data( From cbd50e7e36661eed7f164387845ac6d1938e1e24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Mon, 20 Jul 2026 21:20:28 -0700 Subject: [PATCH 15/54] =?UTF-8?q?=F0=9F=8E=A8=20style:=20apply=20formatter?= =?UTF-8?q?=20fixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/turbohtml/_c/url/url.c | 10 +++++----- tools/bench/migration.py | 4 +++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/turbohtml/_c/url/url.c b/src/turbohtml/_c/url/url.c index b616466d..145e4faa 100644 --- a/src/turbohtml/_c/url/url.c +++ b/src/turbohtml/_c/url/url.c @@ -803,9 +803,9 @@ PyObject *turbohtml_url_join(PyObject *Py_UNUSED(module), PyObject *args) { referral, not the content, so two URLs differing only in them address the same page. Kept sorted so the exact-name test is a binary search. */ static const char *const TRACKER_NAMES[] = { - "clickid", "dclid", "efid", "epik", "fb_ref", "fb_source", "fbclid", "gbraid", - "gclid", "gclsrc", "igsh", "igshid", "mkt_tok", "msclkid", "partnerid", "s_cid", - "sc_cid", "ttclid", "twclid", "wbraid", "wickedid", "yclid", "ysclid", + "clickid", "dclid", "efid", "epik", "fb_ref", "fb_source", "fbclid", "gbraid", + "gclid", "gclsrc", "igsh", "igshid", "mkt_tok", "msclkid", "partnerid", "s_cid", + "sc_cid", "ttclid", "twclid", "wbraid", "wickedid", "yclid", "ysclid", }; static const char *const TRACKER_PREFIXES[] = { @@ -815,8 +815,8 @@ static const char *const TRACKER_PREFIXES[] = { /* The words a tracking key is built from, matched as a whole underscore-delimited word rather than a substring so "reference" is not read as "ref". */ static const char *const TRACKER_WORDS[] = { - "aff", "affi", "affiliate", "campaign", "cid", "clid", "keyword", "kwd", - "medium", "ref", "refer", "referer", "referrer", "session", "source", "uid", "xtor", + "aff", "affi", "affiliate", "campaign", "cid", "clid", "keyword", "kwd", "medium", + "ref", "refer", "referer", "referrer", "session", "source", "uid", "xtor", }; static int tracker_name_known(const char *key, size_t len) { diff --git a/tools/bench/migration.py b/tools/bench/migration.py index 1548a20a..471913e3 100644 --- a/tools/bench/migration.py +++ b/tools/bench/migration.py @@ -137,7 +137,9 @@ def _rows( rows.append(row) spread.append(noise) # a library page mixes operations, so a caveat belongs on the rows of the operation it describes - caveats.append(next((NOTES[operation][label] for label in labels if label in NOTES.get(operation, {})), None)) + caveats.append( + next((NOTES[operation][label] for label in labels if label in NOTES.get(operation, {})), None) + ) return rows, spread, caveats From 21a479de242748ce1fe3dc365b987cc3c4a4454b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Mon, 20 Jul 2026 21:19:51 -0700 Subject: [PATCH 16/54] =?UTF-8?q?=F0=9F=90=9B=20fix(stubs):=20declare=20th?= =?UTF-8?q?e=20new=20URL=20entry=20points?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The urls port added _url_is_tracker and _url_remove_dot_segments to the module without declaring them in the stubs, so the type gate rejected their imports. Declared in features.pyi and re-exported through the _html.pyi aggregate. --- src/turbohtml/_html.pyi | 6 ++++++ src/turbohtml/_stubs/features.pyi | 2 ++ 2 files changed, 8 insertions(+) diff --git a/src/turbohtml/_html.pyi b/src/turbohtml/_html.pyi index cbfe671c..6ec8c1c4 100644 --- a/src/turbohtml/_html.pyi +++ b/src/turbohtml/_html.pyi @@ -149,6 +149,9 @@ from ._stubs.features import ( from ._stubs.features import ( _sanitize as _sanitize, ) +from ._stubs.features import ( + _url_is_tracker as _url_is_tracker, +) from ._stubs.features import ( _url_join as _url_join, ) @@ -158,6 +161,9 @@ from ._stubs.features import ( from ._stubs.features import ( _url_percent_encode as _url_percent_encode, ) +from ._stubs.features import ( + _url_remove_dot_segments as _url_remove_dot_segments, +) from ._stubs.features import ( _url_split as _url_split, ) diff --git a/src/turbohtml/_stubs/features.pyi b/src/turbohtml/_stubs/features.pyi index 1867fee3..13485c44 100644 --- a/src/turbohtml/_stubs/features.pyi +++ b/src/turbohtml/_stubs/features.pyi @@ -33,6 +33,8 @@ def _url_split(url: str, /) -> tuple[str, str, str, str, str, str, str, str, boo def _url_percent_encode(text: str, url_set: int, /) -> str: ... def _url_percent_decode(text: str, /) -> str: ... def _url_join(base: str, target: str, /) -> str: ... +def _url_is_tracker(key: str, /) -> bool: ... +def _url_remove_dot_segments(path: str, /) -> str: ... def _url_to_ascii(host: str, /) -> str: ... def _register_links(link_type: type, /) -> None: ... def _register_structured_data( From 3b20882058bb0f0b60fd3c420c3f7f8c2a5f5c13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Mon, 20 Jul 2026 21:20:28 -0700 Subject: [PATCH 17/54] =?UTF-8?q?=F0=9F=8E=A8=20style:=20apply=20formatter?= =?UTF-8?q?=20fixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/turbohtml/_c/url/url.c | 10 +++++----- tools/bench/migration.py | 4 +++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/turbohtml/_c/url/url.c b/src/turbohtml/_c/url/url.c index b616466d..145e4faa 100644 --- a/src/turbohtml/_c/url/url.c +++ b/src/turbohtml/_c/url/url.c @@ -803,9 +803,9 @@ PyObject *turbohtml_url_join(PyObject *Py_UNUSED(module), PyObject *args) { referral, not the content, so two URLs differing only in them address the same page. Kept sorted so the exact-name test is a binary search. */ static const char *const TRACKER_NAMES[] = { - "clickid", "dclid", "efid", "epik", "fb_ref", "fb_source", "fbclid", "gbraid", - "gclid", "gclsrc", "igsh", "igshid", "mkt_tok", "msclkid", "partnerid", "s_cid", - "sc_cid", "ttclid", "twclid", "wbraid", "wickedid", "yclid", "ysclid", + "clickid", "dclid", "efid", "epik", "fb_ref", "fb_source", "fbclid", "gbraid", + "gclid", "gclsrc", "igsh", "igshid", "mkt_tok", "msclkid", "partnerid", "s_cid", + "sc_cid", "ttclid", "twclid", "wbraid", "wickedid", "yclid", "ysclid", }; static const char *const TRACKER_PREFIXES[] = { @@ -815,8 +815,8 @@ static const char *const TRACKER_PREFIXES[] = { /* The words a tracking key is built from, matched as a whole underscore-delimited word rather than a substring so "reference" is not read as "ref". */ static const char *const TRACKER_WORDS[] = { - "aff", "affi", "affiliate", "campaign", "cid", "clid", "keyword", "kwd", - "medium", "ref", "refer", "referer", "referrer", "session", "source", "uid", "xtor", + "aff", "affi", "affiliate", "campaign", "cid", "clid", "keyword", "kwd", "medium", + "ref", "refer", "referer", "referrer", "session", "source", "uid", "xtor", }; static int tracker_name_known(const char *key, size_t len) { diff --git a/tools/bench/migration.py b/tools/bench/migration.py index 1548a20a..471913e3 100644 --- a/tools/bench/migration.py +++ b/tools/bench/migration.py @@ -137,7 +137,9 @@ def _rows( rows.append(row) spread.append(noise) # a library page mixes operations, so a caveat belongs on the rows of the operation it describes - caveats.append(next((NOTES[operation][label] for label in labels if label in NOTES.get(operation, {})), None)) + caveats.append( + next((NOTES[operation][label] for label in labels if label in NOTES.get(operation, {})), None) + ) return rows, spread, caveats From 407b10b615f0c29d63afb35bb3beae997f7e2b68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Mon, 20 Jul 2026 21:28:45 -0700 Subject: [PATCH 18/54] =?UTF-8?q?=E2=9A=A1=20perf(saxparse):=20dispatch=20?= =?UTF-8?q?events=20from=20C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sax_parse looped in Python over the pull iterator, building a tuple then a typed record then an isinstance chain for every event before the handler call. The tokenizer's html.parser adapter had the same shape and its C dispatch removed 30.8% of instructions, so the sax walk gets the identical treatment: _sax_dispatch binds the six handler methods once and drives them straight off the tree walk, building no per-event record. Callgrind on the whatwg spec with a counting handler: 900.7M instructions to 507.0M (-43.7%), instruction-cache misses -71%, indirect-branch mispredicts -61%. A 38,590-call differential against the retired Python loop over 18 documents, including foster-parented tables, adoption-agency nesting, template content and both doctype identifier forms, found zero divergence. --- src/turbohtml/_c/core/module.c | 1 + src/turbohtml/_c/tokenizer/binding.h | 5 + src/turbohtml/_c/tokenizer/sax.c | 154 +++++++++++++++++++++++++++ src/turbohtml/_html.pyi | 3 + src/turbohtml/_stubs/tokenizer.pyi | 1 + src/turbohtml/saxparse.py | 16 +-- tests/tokenizer/test_saxparse.py | 47 ++++++++ 7 files changed, 213 insertions(+), 14 deletions(-) diff --git a/src/turbohtml/_c/core/module.c b/src/turbohtml/_c/core/module.c index abb6812a..3cba531d 100644 --- a/src/turbohtml/_c/core/module.c +++ b/src/turbohtml/_c/core/module.c @@ -197,6 +197,7 @@ static PyMethodDef html_methods[] = { {"_build_document", (PyCFunction)(void (*)(void))turbohtml_build_document, METH_VARARGS | METH_KEYWORDS, NULL}, {"_tokenize_states", turbohtml_tokenize_states, METH_VARARGS, NULL}, {"_sax_events", turbohtml_sax_events, METH_O, NULL}, + {"_sax_dispatch", turbohtml_sax_dispatch, METH_VARARGS, NULL}, {"_parse_into", turbohtml_parse_into, METH_VARARGS, NULL}, {"_rewrite", turbohtml_rewrite, METH_VARARGS, NULL}, {"_parse_tree", turbohtml_parse_tree, METH_VARARGS, NULL}, diff --git a/src/turbohtml/_c/tokenizer/binding.h b/src/turbohtml/_c/tokenizer/binding.h index 41673156..8cabce68 100644 --- a/src/turbohtml/_c/tokenizer/binding.h +++ b/src/turbohtml/_c/tokenizer/binding.h @@ -131,6 +131,11 @@ PyObject *turbohtml_rewrite(PyObject *module, PyObject *args); METH_O. */ PyObject *turbohtml_sax_events(PyObject *module, PyObject *arg); +/* Parse markup and drive a handler object's start_element/end_element/characters/comment/doctype/ + processing_instruction methods over the events in document order, building no per-event record; the push + form behind turbohtml.saxparse.sax_parse. Matches METH_VARARGS: (source, handler). */ +PyObject *turbohtml_sax_dispatch(PyObject *module, PyObject *args); + /* Parse markup and drive a caller-supplied builder object over the constructed tree in document order (create_document/create_doctype/create_element/create_text/ create_comment/create_pi/append), returning whatever the builder made its root without diff --git a/src/turbohtml/_c/tokenizer/sax.c b/src/turbohtml/_c/tokenizer/sax.c index aafa3d35..45366b9b 100644 --- a/src/turbohtml/_c/tokenizer/sax.c +++ b/src/turbohtml/_c/tokenizer/sax.c @@ -280,3 +280,157 @@ int sax_register(PyObject *module, module_state *state) { } return 0; } + +/* The handler methods _sax_dispatch drives, bound once per call and indexed by the enum, so binding walks the name + table rather than testing six lookups in one condition. */ +enum { SAX_H_START, SAX_H_END, SAX_H_CHARACTERS, SAX_H_COMMENT, SAX_H_DOCTYPE, SAX_H_PI, SAX_H_COUNT }; + +static const char *const SAX_HANDLER_NAMES[SAX_H_COUNT] = { + "start_element", "end_element", "characters", "comment", "doctype", "processing_instruction", +}; + +typedef struct { + PyObject *fn[SAX_H_COUNT]; +} sax_dispatch_handlers; + +static void sax_handlers_release(sax_dispatch_handlers *bound) { + for (int index = 0; index < SAX_H_COUNT; index++) { + Py_XDECREF(bound->fn[index]); + } +} + +static int sax_handlers_bind(sax_dispatch_handlers *bound, PyObject *handler) { + for (int index = 0; index < SAX_H_COUNT; index++) { + bound->fn[index] = NULL; + } + for (int index = 0; index < SAX_H_COUNT; index++) { + bound->fn[index] = PyObject_GetAttrString(handler, SAX_HANDLER_NAMES[index]); + if (bound->fn[index] == NULL) { + sax_handlers_release(bound); + return -1; + } + } + return 0; +} + +/* Call one bound handler for entering node; 0 to continue, -1 with an exception set to stop. A node that emits + nothing on entry (the document root, a template content fragment) continues without a call. */ +static int dispatch_enter(th_tree *tree, th_node *node, sax_dispatch_handlers *bound) { + PyObject *result = NULL; + switch (node->type) { + case TH_NODE_ELEMENT: { + PyObject *tag = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, node->text, node->text_len); + if (tag == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ + return -1; /* GCOVR_EXCL_LINE: allocation-failure path */ + } + PyObject *attrs = attrs_tuple(tree, node); + if (attrs == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ + Py_DECREF(tag); /* GCOVR_EXCL_LINE: allocation-failure path */ + return -1; /* GCOVR_EXCL_LINE: allocation-failure path */ + } + result = PyObject_CallFunctionObjArgs(bound->fn[SAX_H_START], tag, attrs, NULL); + Py_DECREF(tag); + Py_DECREF(attrs); + break; + } + case TH_NODE_TEXT: + case TH_NODE_COMMENT: { + PyObject *data = data_str(tree, node); + if (data == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ + return -1; /* GCOVR_EXCL_LINE: allocation-failure path */ + } + int which = node->type == TH_NODE_TEXT ? SAX_H_CHARACTERS + : (node->tag_flags & TH_COMMENT_IS_PI) ? SAX_H_PI + : SAX_H_COMMENT; + result = PyObject_CallFunctionObjArgs(bound->fn[which], data, NULL); + Py_DECREF(data); + break; + } + case TH_NODE_DOCTYPE: { + /* the tuple event builder is reused, then unpacked, so the identifier extraction lives in one place */ + PyObject *event = doctype_event(tree, node); + if (event == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ + return -1; /* GCOVR_EXCL_LINE: allocation-failure path */ + } + result = PyObject_CallFunctionObjArgs(bound->fn[SAX_H_DOCTYPE], PyTuple_GET_ITEM(event, 1), + PyTuple_GET_ITEM(event, 2), PyTuple_GET_ITEM(event, 3), NULL); + Py_DECREF(event); + break; + } + default: /* TH_NODE_DOCUMENT and TH_NODE_CONTENT emit nothing themselves */ + return 0; + } + if (result == NULL) { + return -1; + } + Py_DECREF(result); + return 0; +} + +/* Call the end_element handler when leaving node; 0 to continue, -1 with an exception set to stop. Only an + element has a close. */ +static int dispatch_leave(th_node *node, sax_dispatch_handlers *bound) { + if (node->type != TH_NODE_ELEMENT) { + return 0; + } + PyObject *tag = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, node->text, node->text_len); + if (tag == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ + return -1; /* GCOVR_EXCL_LINE: allocation-failure path */ + } + PyObject *result = PyObject_CallFunctionObjArgs(bound->fn[SAX_H_END], tag, NULL); + Py_DECREF(tag); + if (result == NULL) { + return -1; + } + Py_DECREF(result); + return 0; +} + +/* _sax_dispatch(html, handler): parse and drive the handler's methods over the events in document order, building + no per-event record. The push form of the walk _sax_events streams; sax_parse wraps this. */ +PyObject *turbohtml_sax_dispatch(PyObject *Py_UNUSED(module), PyObject *args) { + PyObject *html; + PyObject *handler; + if (!PyArg_ParseTuple(args, "UO:_sax_dispatch", &html, &handler)) { + return NULL; + } + sax_dispatch_handlers bound; + if (sax_handlers_bind(&bound, handler) < 0) { + return NULL; + } + th_tree *tree = th_tree_parse(PyUnicode_KIND(html), PyUnicode_DATA(html), PyUnicode_GET_LENGTH(html), 0, 0, 0, 1); + if (tree == NULL) { /* GCOVR_EXCL_BR_LINE: only an allocation failure returns NULL */ + sax_handlers_release(&bound); /* GCOVR_EXCL_LINE: allocation-failure path */ + return PyErr_NoMemory(); /* GCOVR_EXCL_LINE: allocation-failure path */ + } + th_node *cursor = th_tree_document(tree); + int leaving = 0; + int failed = 0; + while (!failed) { + th_node *node = cursor; + if (!leaving) { + failed = dispatch_enter(tree, node, &bound) < 0; + if (node->first_child != NULL) { + cursor = node->first_child; + } else { + leaving = 1; + } + } else { + failed = dispatch_leave(node, &bound) < 0; + if (node->next_sibling != NULL) { + cursor = node->next_sibling; + leaving = 0; + } else if (node->parent != NULL) { + cursor = node->parent; + } else { + break; + } + } + } + th_tree_free(tree); + sax_handlers_release(&bound); + if (failed) { + return NULL; + } + Py_RETURN_NONE; +} diff --git a/src/turbohtml/_html.pyi b/src/turbohtml/_html.pyi index 6ec8c1c4..682637db 100644 --- a/src/turbohtml/_html.pyi +++ b/src/turbohtml/_html.pyi @@ -281,6 +281,9 @@ from ._stubs.tokenizer import ( from ._stubs.tokenizer import ( _RewriteHandle as _RewriteHandle, ) +from ._stubs.tokenizer import ( + _sax_dispatch as _sax_dispatch, +) from ._stubs.tokenizer import ( _sax_events as _sax_events, ) diff --git a/src/turbohtml/_stubs/tokenizer.pyi b/src/turbohtml/_stubs/tokenizer.pyi index 4f0ccfe7..74fd820a 100644 --- a/src/turbohtml/_stubs/tokenizer.pyi +++ b/src/turbohtml/_stubs/tokenizer.pyi @@ -103,6 +103,7 @@ def _rewrite( /, ) -> str: ... def _sax_events(html: str, /) -> Iterator[tuple[object, ...]]: ... +def _sax_dispatch(html: str, handler: object, /) -> None: ... def _tokenize_states( text: str, initial_state: str, last_start_tag: str | None = ..., storage_kind: int = ..., / ) -> tuple[list[tuple[object, ...]], list[tuple[str, int, int]]]: ... diff --git a/src/turbohtml/saxparse.py b/src/turbohtml/saxparse.py index a6a638f9..ef5546d3 100644 --- a/src/turbohtml/saxparse.py +++ b/src/turbohtml/saxparse.py @@ -28,7 +28,7 @@ from typing import TYPE_CHECKING, NamedTuple, cast -from ._html import _sax_events +from ._html import _sax_dispatch, _sax_events if TYPE_CHECKING: from collections.abc import Iterator @@ -157,16 +157,4 @@ def sax_parse(html: str, handler: SaxHandler) -> None: :param handler: a :class:`SaxHandler` (or subclass) whose overridden methods receive the events. :raises TypeError: if ``html`` is not a str. """ - for event in iter_events(html): - if isinstance(event, StartElement): - handler.start_element(event.tag, event.attrs) - elif isinstance(event, EndElement): - handler.end_element(event.tag) - elif isinstance(event, Characters): - handler.characters(event.data) - elif isinstance(event, Comment): - handler.comment(event.data) - elif isinstance(event, Doctype): - handler.doctype(event.name, event.public_id, event.system_id) - else: - handler.processing_instruction(event.data) + _sax_dispatch(html, handler) diff --git a/tests/tokenizer/test_saxparse.py b/tests/tokenizer/test_saxparse.py index c5622c0d..7973148f 100644 --- a/tests/tokenizer/test_saxparse.py +++ b/tests/tokenizer/test_saxparse.py @@ -253,3 +253,50 @@ class _Sentinel: def test_non_str_argument_raises_type_error() -> None: with pytest.raises(TypeError, match="must be str"): list(iter_events(cast("str", b"

"))) + + +def test_sax_parse_requires_every_handler_method() -> None: + # dispatch binds the six methods once up front, so an object missing one fails before any parsing happens + with pytest.raises(AttributeError, match="start_element"): + sax_parse("

x

", cast("SaxHandler", object())) + + +def test_sax_parse_non_str_raises_type_error() -> None: + with pytest.raises(TypeError): + sax_parse(cast("str", b"

"), SaxHandler()) + + +def test_handler_exception_in_start_stops_the_parse() -> None: + class FailingStart(SaxHandler): + def __init__(self) -> None: + self.seen: list[str] = [] + + def start_element(self, tag: str, attrs: tuple[tuple[str, str | None], ...]) -> None: + self.seen.append(tag) + if tag == "i": + msg = f"stop at {tag} with {len(attrs)} attributes" + raise ValueError(msg) + + handler = FailingStart() + with pytest.raises(ValueError, match="stop at i"): + sax_parse("

x

", handler) + # the walk stops where the handler raised rather than swallowing the error and continuing + assert handler.seen == ["html", "head", "body", "p", "i"] + + +def test_handler_exception_in_end_stops_the_parse() -> None: + class FailingEnd(SaxHandler): + def __init__(self) -> None: + self.closed: list[str] = [] + + def end_element(self, tag: str) -> None: + self.closed.append(tag) + if tag == "p": + msg = f"stop closing {tag}" + raise RuntimeError(msg) + + handler = FailingEnd() + with pytest.raises(RuntimeError, match="stop closing p"): + sax_parse("

x

y
", handler) + # the implied closes before

opens, so it precedes the raise + assert handler.closed == ["head", "p"] From 3114ed39bbe31f0679ab7d7d7ad00847dc0528c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Mon, 20 Jul 2026 21:38:04 -0700 Subject: [PATCH 19/54] =?UTF-8?q?=E2=9A=A1=20perf(query):=20escape=20ident?= =?UTF-8?q?ifiers=20in=20C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit escape_identifier walked the string per character in Python applying the CSSOM serialize-an-identifier rules. The loop is now the C entry point _css_escape_identifier and the Python def keeps only its docstring and the call. A 4,828-input differential across the BMP boundaries, astral plane, leading-digit and lone-dash shapes found zero divergences against the retired loop. --- src/turbohtml/_c/core/common.h | 4 +++ src/turbohtml/_c/core/module.c | 1 + src/turbohtml/_c/css/cssom/cssom.c | 45 ++++++++++++++++++++++++++++++ src/turbohtml/_html.pyi | 3 ++ src/turbohtml/_stubs/query.pyi | 1 + src/turbohtml/query/_match.py | 21 ++------------ 6 files changed, 56 insertions(+), 19 deletions(-) diff --git a/src/turbohtml/_c/core/common.h b/src/turbohtml/_c/core/common.h index 1793dfd7..498ab32c 100644 --- a/src/turbohtml/_c/core/common.h +++ b/src/turbohtml/_c/core/common.h @@ -80,6 +80,10 @@ PyObject *turbohtml_css_parse_declarations(PyObject *module, PyObject *text); PyObject *turbohtml_css_parse_rules(PyObject *module, PyObject *text); PyObject *turbohtml_css_computed_style(PyObject *module, PyObject *arg); +/* CSS.escape per the CSSOM serialize-an-identifier rules, behind turbohtml.query.escape_identifier. + Matches METH_O. */ +PyObject *turbohtml_css_escape_identifier(PyObject *module, PyObject *arg); + /* Implemented in dom/element.c: stores the SelectorSyntaxError type the selector and XPath parsers raise on a malformed expression (METH_O); turbohtml._selectors registers it on import. */ diff --git a/src/turbohtml/_c/core/module.c b/src/turbohtml/_c/core/module.c index 3cba531d..45f7ecb1 100644 --- a/src/turbohtml/_c/core/module.c +++ b/src/turbohtml/_c/core/module.c @@ -213,6 +213,7 @@ static PyMethodDef html_methods[] = { {"_css_parse_declarations", turbohtml_css_parse_declarations, METH_O, NULL}, {"_css_parse_rules", turbohtml_css_parse_rules, METH_O, NULL}, {"_css_computed_style", turbohtml_css_computed_style, METH_O, NULL}, + {"_css_escape_identifier", turbohtml_css_escape_identifier, METH_O, NULL}, {"_minify_js", turbohtml_minify_js, METH_VARARGS, NULL}, {"_minify_js_tokens", turbohtml_minify_js_tokens, METH_O, NULL}, {"_minify_js_parse", turbohtml_minify_js_parse, METH_O, NULL}, diff --git a/src/turbohtml/_c/css/cssom/cssom.c b/src/turbohtml/_c/css/cssom/cssom.c index a59d92dd..efc34380 100644 --- a/src/turbohtml/_c/css/cssom/cssom.c +++ b/src/turbohtml/_c/css/cssom/cssom.c @@ -19,6 +19,7 @@ #include "dom/nodes.h" #include "css/select/selector.h" +#include #include /* One CSS longhand property the cascade resolves: its lowercase name, whether it @@ -1241,3 +1242,47 @@ PyObject *turbohtml_css_computed_style(PyObject *module, PyObject *arg) { css_free_map(final_map); return result; } + +/* _css_escape_identifier(text): CSS.escape per the CSSOM serialize-an-identifier rules, so a raw class or id can be + dropped into a selector. Worst case every code point becomes "\XXXXXX " (8 UCS4), which bounds the buffer. */ +PyObject *turbohtml_css_escape_identifier(PyObject *Py_UNUSED(module), PyObject *arg) { + if (!PyUnicode_Check(arg)) { + PyErr_SetString(PyExc_TypeError, "escape_identifier() argument must be str"); + return NULL; + } + Py_ssize_t len = PyUnicode_GET_LENGTH(arg); + int kind = PyUnicode_KIND(arg); + const void *data = PyUnicode_DATA(arg); + Py_UCS4 *out = PyMem_Malloc((size_t)(len > 0 ? len : 1) * 8 * sizeof(Py_UCS4)); + if (out == NULL) { /* GCOVR_EXCL_BR_LINE: allocation failure cannot be forced from a test */ + return PyErr_NoMemory(); /* GCOVR_EXCL_LINE: allocation-failure path */ + } + Py_ssize_t at = 0; + for (Py_ssize_t position = 0; position < len; position++) { + Py_UCS4 code = PyUnicode_READ(kind, data, position); + int is_digit = code >= 0x30 && code <= 0x39; + /* a digit leading the identifier (or following an initial dash) cannot start a CSS name */ + int leading_digit = is_digit && (position == 0 || (position == 1 && PyUnicode_READ(kind, data, 0) == '-')); + int is_name_char = code >= 0x80 || is_digit || code == '-' || code == '_' || (code >= 0x41 && code <= 0x5A) || + (code >= 0x61 && code <= 0x7A); + /* a lone "-" cannot stand as an identifier, so it is backslashed exactly like a non-name character */ + int lone_dash = position == 0 && code == '-' && len == 1; + if (code == 0) { + out[at++] = 0xFFFD; + } else if (code <= 0x1F || code == 0x7F || leading_digit) { + char hex[16]; + int wrote = snprintf(hex, sizeof(hex), "\\%x ", (unsigned int)code); + for (int index = 0; index < wrote; index++) { + out[at++] = (Py_UCS4)(unsigned char)hex[index]; + } + } else if (is_name_char && !lone_dash) { + out[at++] = code; + } else { + out[at++] = '\\'; + out[at++] = code; + } + } + PyObject *result = th_str_from_kind(PyUnicode_4BYTE_KIND, out, at); + PyMem_Free(out); + return result; +} diff --git a/src/turbohtml/_html.pyi b/src/turbohtml/_html.pyi index 682637db..b05fe66d 100644 --- a/src/turbohtml/_html.pyi +++ b/src/turbohtml/_html.pyi @@ -182,6 +182,9 @@ from ._stubs.query import ( from ._stubs.query import ( _css_computed_style as _css_computed_style, ) +from ._stubs.query import ( + _css_escape_identifier as _css_escape_identifier, +) from ._stubs.query import ( _css_parse_declarations as _css_parse_declarations, ) diff --git a/src/turbohtml/_stubs/query.pyi b/src/turbohtml/_stubs/query.pyi index ebeec1b9..29d694db 100644 --- a/src/turbohtml/_stubs/query.pyi +++ b/src/turbohtml/_stubs/query.pyi @@ -19,6 +19,7 @@ def _css_specificity(selector: str, /) -> list[tuple[int, int, int]]: ... def _css_parse_declarations(text: str, /) -> tuple[tuple[str, str, bool], ...]: ... def _css_parse_rules(text: str, /) -> tuple[tuple[str, tuple[tuple[str, str, bool], ...]], ...]: ... def _css_computed_style(element: Element, /) -> tuple[tuple[str, str], ...]: ... +def _css_escape_identifier(ident: str, /) -> str: ... def _register_xpath_string(xpath_string_type: type, /) -> None: ... def _register_selector_error(selector_error_type: type[ValueError], /) -> None: ... diff --git a/src/turbohtml/query/_match.py b/src/turbohtml/query/_match.py index 18d19f26..fc44c95b 100644 --- a/src/turbohtml/query/_match.py +++ b/src/turbohtml/query/_match.py @@ -23,7 +23,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Final, cast -from turbohtml._html import Document, Element, _matches_many, parse +from turbohtml._html import Document, Element, _css_escape_identifier, _matches_many, parse from turbohtml._internal._selectors import SelectorSyntaxError if TYPE_CHECKING: @@ -44,24 +44,7 @@ def escape_identifier(ident: str) -> str: :param ident: the raw identifier text. :returns: the identifier with CSS-significant characters backslash- or hex-escaped per the CSSOM rules. """ - out: list[str] = [] - for position, char in enumerate(ident): - code = ord(char) - is_digit = 0x30 <= code <= 0x39 - # a digit leading the identifier (or following an initial dash) cannot start a CSS name, so it is hex-escaped - leading_digit = is_digit and (position == 0 or (position == 1 and ident[0] == "-")) - is_name_char = code >= 0x80 or is_digit or char in "-_" or 0x41 <= code <= 0x5A or 0x61 <= code <= 0x7A - if code == 0: - out.append("�") - elif code <= 0x1F or code == 0x7F or leading_digit: - out.append(f"\\{code:x} ") - elif position == 0 and code == 0x2D and len(ident) == 1: - out.append(f"\\{char}") - elif is_name_char: - out.append(char) - else: - out.append(f"\\{char}") - return "".join(out) + return _css_escape_identifier(ident) @dataclass(frozen=True) From 7d03dac1a7a557333f794886a21fb3a3e078ce01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Mon, 20 Jul 2026 21:42:29 -0700 Subject: [PATCH 20/54] =?UTF-8?q?=F0=9F=93=9D=20docs:=20add=20the=20change?= =?UTF-8?q?log=20fragment=20for=20#692?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/changelog/692.feature.rst | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 docs/changelog/692.feature.rst diff --git a/docs/changelog/692.feature.rst b/docs/changelog/692.feature.rst new file mode 100644 index 00000000..bd19d417 --- /dev/null +++ b/docs/changelog/692.feature.rst @@ -0,0 +1,7 @@ +Dispatch SAX events from C. :func:`turbohtml.saxparse.sax_parse` used to build a tuple, a typed record, and an +isinstance chain for every event before calling the handler; the tokenizer now binds the six handler methods once and +drives them straight off the tree walk. Callgrind on the WHATWG specification measures 43.7% fewer instructions with +instruction-cache misses down 71%, and a 38,590-call differential against the previous dispatch found no divergence. +The pull form :func:`~turbohtml.saxparse.iter_events` keeps its typed records. URL cleaning's tracking-parameter +vocabulary and dot-segment resolution, and :func:`turbohtml.query.escape_identifier`'s CSSOM escape loop, moved to C +the same way, each validated by a differential run against the retired Python (302,762 and 4,828 inputs). From 021f1d4ea60dc555d0d3cd72a9bfc35b410ac347 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 21 Jul 2026 04:42:51 +0000 Subject: [PATCH 21/54] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/changelog/692.feature.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/changelog/692.feature.rst b/docs/changelog/692.feature.rst index bd19d417..be2a89f3 100644 --- a/docs/changelog/692.feature.rst +++ b/docs/changelog/692.feature.rst @@ -1,7 +1,7 @@ Dispatch SAX events from C. :func:`turbohtml.saxparse.sax_parse` used to build a tuple, a typed record, and an isinstance chain for every event before calling the handler; the tokenizer now binds the six handler methods once and drives them straight off the tree walk. Callgrind on the WHATWG specification measures 43.7% fewer instructions with -instruction-cache misses down 71%, and a 38,590-call differential against the previous dispatch found no divergence. -The pull form :func:`~turbohtml.saxparse.iter_events` keeps its typed records. URL cleaning's tracking-parameter -vocabulary and dot-segment resolution, and :func:`turbohtml.query.escape_identifier`'s CSSOM escape loop, moved to C -the same way, each validated by a differential run against the retired Python (302,762 and 4,828 inputs). +instruction-cache misses down 71%, and a 38,590-call differential against the previous dispatch found no divergence. The +pull form :func:`~turbohtml.saxparse.iter_events` keeps its typed records. URL cleaning's tracking-parameter vocabulary +and dot-segment resolution, and :func:`turbohtml.query.escape_identifier`'s CSSOM escape loop, moved to C the same way, +each validated by a differential run against the retired Python (302,762 and 4,828 inputs). From 013fabb6f6b2bb252a7af90052e296796b139da4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Mon, 20 Jul 2026 21:46:50 -0700 Subject: [PATCH 22/54] perf(validate): cache schema node qnames is_schema_el reclassified the same schema element nodes millions of times per document -- the content-model matcher revisits every particle on each NFA step -- and each call walked the ancestor chain through resolve_ns to re-resolve the node's namespace. That walk was the validator's single hottest cost. Resolve every schema element node's qname once at compile time into a pointer-sorted table and look it up by binary search. The table is built before any classification runs and never mutated, so a compiled schema shared across validating threads stays read-only. On the catalog XSD + doc bench (Callgrind, python:3.13-slim): 1,118.9M to 926.6M Ir, -17.2%. resolve_ns 206.1M -> 56.9M (-72%), th_attr_name 50.0M -> 19.7M (-61%). All XSD/RelaxNG conformance tests pass unchanged. --- src/turbohtml/_c/validate/relaxng.h | 52 ++++++++-------- src/turbohtml/_c/validate/schema.c | 97 ++++++++++++++++++++++++++--- src/turbohtml/_c/validate/xsd.h | 91 +++++++++++++-------------- 3 files changed, 161 insertions(+), 79 deletions(-) diff --git a/src/turbohtml/_c/validate/relaxng.h b/src/turbohtml/_c/validate/relaxng.h index be7c693f..e6e17108 100644 --- a/src/turbohtml/_c/validate/relaxng.h +++ b/src/turbohtml/_c/validate/relaxng.h @@ -302,7 +302,7 @@ static nameclass *rng_nameclass_choice_children(th_schema *schema, th_node *node if (child->type != TH_NODE_ELEMENT) { continue; } - if (is_schema_el(schema->tree, child, RNG_NS, "except")) { + if (is_schema_el(schema, child, RNG_NS, "except")) { continue; } nameclass *part = rng_build_nameclass(schema, child); @@ -323,13 +323,13 @@ static nameclass *rng_build_nameclass(th_schema *schema, th_node *node) { if (node == NULL) { /* a malformed / with no name class matches any name */ return nc_new(schema, NC_ANY); } - if (is_schema_el(tree, node, RNG_NS, "name")) { + if (is_schema_el(schema, node, RNG_NS, "name")) { Py_ssize_t len = 0; const Py_UCS4 *text = element_text_raw(tree, node, &len); return nc_from_qname(schema, node, text, len, 0); } - if (is_schema_el(tree, node, RNG_NS, "anyName")) { - th_node *except = first_schema_child(tree, node, RNG_NS, "except"); + if (is_schema_el(schema, node, RNG_NS, "anyName")) { + th_node *except = first_schema_child(schema, node, RNG_NS, "except"); if (except == NULL) { return nc_new(schema, NC_ANY); } @@ -338,7 +338,7 @@ static nameclass *rng_build_nameclass(th_schema *schema, th_node *node) { nc->b = rng_nameclass_choice_children(schema, except); return nc; } - if (is_schema_el(tree, node, RNG_NS, "nsName")) { + if (is_schema_el(schema, node, RNG_NS, "nsName")) { nameclass *nc = nc_new(schema, NC_NS); const th_node_attr *ns = attr_exact(tree, node, "ns", 2); if (ns == NULL) { @@ -347,7 +347,7 @@ static nameclass *rng_build_nameclass(th_schema *schema, th_node *node) { nc->uri = ns->value; nc->uri_len = ns->value_len; } - th_node *except = first_schema_child(tree, node, RNG_NS, "except"); + th_node *except = first_schema_child(schema, node, RNG_NS, "except"); if (except == NULL) { return nc; } @@ -423,16 +423,16 @@ static pattern *rng_build_children(th_schema *schema, th_node *node, th_node *sk static pattern *rng_build(th_schema *schema, th_node *node) { th_tree *tree = schema->tree; - if (is_schema_el(tree, node, RNG_NS, "empty")) { + if (is_schema_el(schema, node, RNG_NS, "empty")) { return schema->p_empty; } - if (is_schema_el(tree, node, RNG_NS, "text")) { + if (is_schema_el(schema, node, RNG_NS, "text")) { return schema->p_text; } - if (is_schema_el(tree, node, RNG_NS, "notAllowed")) { + if (is_schema_el(schema, node, RNG_NS, "notAllowed")) { return schema->p_notallowed; } - if (is_schema_el(tree, node, RNG_NS, "element")) { + if (is_schema_el(schema, node, RNG_NS, "element")) { pattern *node_pat = pat_new(schema, P_ELEMENT); const th_node_attr *name = attr_exact(tree, node, "name", 4); th_node *skip = NULL; @@ -446,7 +446,7 @@ static pattern *rng_build(th_schema *schema, th_node *node) { node_pat->p1 = rng_build_children(schema, node, skip); return node_pat; } - if (is_schema_el(tree, node, RNG_NS, "attribute")) { + if (is_schema_el(schema, node, RNG_NS, "attribute")) { pattern *node_pat = pat_new(schema, P_ATTRIBUTE); const th_node_attr *name = attr_exact(tree, node, "name", 4); th_node *skip = NULL; @@ -467,10 +467,10 @@ static pattern *rng_build(th_schema *schema, th_node *node) { node_pat->p1 = has_content ? rng_build_children(schema, node, skip) : schema->p_text; return node_pat; } - if (is_schema_el(tree, node, RNG_NS, "group")) { + if (is_schema_el(schema, node, RNG_NS, "group")) { return rng_build_children(schema, node, NULL); } - if (is_schema_el(tree, node, RNG_NS, "choice")) { + if (is_schema_el(schema, node, RNG_NS, "choice")) { pattern *combined = schema->p_notallowed; for (th_node *child = node->first_child; child != NULL; child = child->next_sibling) { if (child->type == TH_NODE_ELEMENT) { @@ -479,7 +479,7 @@ static pattern *rng_build(th_schema *schema, th_node *node) { } return combined; } - if (is_schema_el(tree, node, RNG_NS, "interleave")) { + if (is_schema_el(schema, node, RNG_NS, "interleave")) { pattern *combined = schema->p_empty; for (th_node *child = node->first_child; child != NULL; child = child->next_sibling) { if (child->type == TH_NODE_ELEMENT) { @@ -488,24 +488,24 @@ static pattern *rng_build(th_schema *schema, th_node *node) { } return combined; } - if (is_schema_el(tree, node, RNG_NS, "optional")) { + if (is_schema_el(schema, node, RNG_NS, "optional")) { return pat_choice(schema, rng_build_children(schema, node, NULL), schema->p_empty); } - if (is_schema_el(tree, node, RNG_NS, "zeroOrMore")) { + if (is_schema_el(schema, node, RNG_NS, "zeroOrMore")) { return pat_choice(schema, pat_onemore(schema, rng_build_children(schema, node, NULL)), schema->p_empty); } - if (is_schema_el(tree, node, RNG_NS, "oneOrMore")) { + if (is_schema_el(schema, node, RNG_NS, "oneOrMore")) { return pat_onemore(schema, rng_build_children(schema, node, NULL)); } - if (is_schema_el(tree, node, RNG_NS, "mixed")) { + if (is_schema_el(schema, node, RNG_NS, "mixed")) { return pat_interleave(schema, rng_build_children(schema, node, NULL), schema->p_text); } - if (is_schema_el(tree, node, RNG_NS, "list")) { + if (is_schema_el(schema, node, RNG_NS, "list")) { pattern *node_pat = pat_new(schema, P_LIST); node_pat->p1 = rng_build_children(schema, node, NULL); return node_pat; } - if (is_schema_el(tree, node, RNG_NS, "data")) { + if (is_schema_el(schema, node, RNG_NS, "data")) { pattern *node_pat = pat_new(schema, P_DATA); node_pat->datatype_id = rng_datatype_id(schema, node); facetset *facets = arena_alloc(&schema->mem, sizeof(*facets)); @@ -514,7 +514,7 @@ static pattern *rng_build(th_schema *schema, th_node *node) { } facetset_init(facets, node_pat->datatype_id); for (th_node *param = node->first_child; param != NULL; param = param->next_sibling) { - if (is_schema_el(tree, param, RNG_NS, "param")) { + if (is_schema_el(schema, param, RNG_NS, "param")) { const th_node_attr *pname = attr_exact(tree, param, "name", 4); Py_ssize_t vlen = 0; const Py_UCS4 *value = element_text_raw(tree, param, &vlen); @@ -526,14 +526,14 @@ static pattern *rng_build(th_schema *schema, th_node *node) { node_pat->facets = facets; return node_pat; } - if (is_schema_el(tree, node, RNG_NS, "value")) { + if (is_schema_el(schema, node, RNG_NS, "value")) { pattern *node_pat = pat_new(schema, P_VALUE); node_pat->datatype_id = rng_datatype_id(schema, node); node_pat->value = element_text_raw(tree, node, &node_pat->value_len); node_pat->value_ws = dt_default_ws(node_pat->datatype_id); return node_pat; } - if (is_schema_el(tree, node, RNG_NS, "ref")) { + if (is_schema_el(schema, node, RNG_NS, "ref")) { const th_node_attr *name = attr_exact(tree, node, "name", 4); Py_ssize_t index = def_find(&schema->defines, name->value, name->value_len); if (index >= 0) { @@ -963,12 +963,12 @@ static int rng_compile(th_schema *schema) { schema->p_empty = pat_new(schema, P_EMPTY); schema->p_notallowed = pat_new(schema, P_NOTALLOWED); schema->p_text = pat_new(schema, P_TEXT); - if (!is_schema_el(tree, schema->root, RNG_NS, "grammar")) { + if (!is_schema_el(schema, schema->root, RNG_NS, "grammar")) { schema->start = rng_build(schema, schema->root); return 1; } for (th_node *child = schema->root->first_child; child != NULL; child = child->next_sibling) { - if (is_schema_el(tree, child, RNG_NS, "define")) { + if (is_schema_el(schema, child, RNG_NS, "define")) { const th_node_attr *name = attr_exact(tree, child, "name", 4); if (name == NULL) { continue; @@ -979,7 +979,7 @@ static int rng_compile(th_schema *schema) { } } } - th_node *start = first_schema_child(tree, schema->root, RNG_NS, "start"); + th_node *start = first_schema_child(schema, schema->root, RNG_NS, "start"); if (start == NULL) { PyErr_SetString(PyExc_ValueError, "grammar has no start element"); return 0; diff --git a/src/turbohtml/_c/validate/schema.c b/src/turbohtml/_c/validate/schema.c index c3e45914..66bcf498 100644 --- a/src/turbohtml/_c/validate/schema.c +++ b/src/turbohtml/_c/validate/schema.c @@ -27,6 +27,7 @@ #include "dom/tree.h" #include +#include #include static const char XSD_NS[] = "http://www.w3.org/2001/XMLSchema"; @@ -123,6 +124,20 @@ typedef struct { Py_ssize_t uri_len; } qname; +/* The schema tree is immutable once compiled, yet is_schema_el reclassifies its element + nodes millions of times while a document validates (the content-model matcher revisits + the same particles on every step). resolve_ns walking the ancestor chain per call is the + validator's hottest cost, so each schema element node's namespace-resolved qname is + computed once at compile time and looked up by node pointer here. Built before any + compile classification runs and never mutated afterwards, so a compiled schema shared + across validating threads stays read-only. */ +struct th_schema; +typedef struct { + th_node *node; + qname name; +} sqname_entry; +static const qname *schema_node_qname(const struct th_schema *schema, th_node *node); + static void split_prefix(const Py_UCS4 *name, Py_ssize_t len, const Py_UCS4 **local, Py_ssize_t *local_len, const Py_UCS4 **prefix, Py_ssize_t *prefix_len) { for (Py_ssize_t index = 0; index < len; index++) { @@ -206,25 +221,25 @@ static qname node_qname(th_tree *tree, th_node *element, const Py_UCS4 *name, Py return out; } -static int is_schema_el(th_tree *tree, th_node *node, const char *ns, const char *local) { +static int is_schema_el(const struct th_schema *schema, th_node *node, const char *ns, const char *local) { if (node->type != TH_NODE_ELEMENT) { return 0; } - qname name = node_qname(tree, node, node->text, node->text_len, 0); - if (!u_eq_ascii(name.local, name.local_len, local)) { + const qname *name = schema_node_qname(schema, node); + if (!u_eq_ascii(name->local, name->local_len, local)) { return 0; } - if (name.uri == NULL) { + if (name->uri == NULL) { return 0; } - return u_eq_ascii(name.uri, name.uri_len, ns); + return u_eq_ascii(name->uri, name->uri_len, ns); } /* The first element child of `node` in the schema namespace `ns` with local name `local`, or NULL. */ -static th_node *first_schema_child(th_tree *tree, th_node *node, const char *ns, const char *local) { +static th_node *first_schema_child(const struct th_schema *schema, th_node *node, const char *ns, const char *local) { for (th_node *child = node->first_child; child != NULL; child = child->next_sibling) { - if (is_schema_el(tree, child, ns, local)) { + if (is_schema_el(schema, child, ns, local)) { return child; } } @@ -451,8 +466,72 @@ typedef struct th_schema { pattern *start; pattern *p_empty, *p_notallowed, *p_text; def_vec defines; + /* every schema element node's resolved qname, sorted by node pointer for is_schema_el */ + sqname_entry *sqnames; + Py_ssize_t sqname_count; } th_schema; +/* Look up a schema element node's precomputed qname. schema_build_qname_cache enters every + element node of the schema tree before any classification runs, and is_schema_el only ever + asks about schema element nodes, so the search always hits. */ +static const qname *schema_node_qname(const th_schema *schema, th_node *node) { + Py_ssize_t lo = 0, hi = schema->sqname_count - 1; + while (lo <= hi) { /* GCOVR_EXCL_BR_LINE: the searched-for node is always present, so the loop exits by return */ + Py_ssize_t mid = lo + ((hi - lo) >> 1); + th_node *at = schema->sqnames[mid].node; + if (at == node) { + return &schema->sqnames[mid].name; + } + if (at < node) { + lo = mid + 1; + } else { + hi = mid - 1; + } + } + return NULL; /* GCOVR_EXCL_LINE: every schema element node is cached, so the search never misses */ +} + +static Py_ssize_t schema_count_elements(th_node *parent) { + Py_ssize_t total = 0; + for (th_node *child = parent->first_child; child != NULL; child = child->next_sibling) { + if (child->type == TH_NODE_ELEMENT) { + total += 1 + schema_count_elements(child); + } + } + return total; +} + +static void schema_fill_qnames(th_schema *schema, th_node *parent, Py_ssize_t *at) { + for (th_node *child = parent->first_child; child != NULL; child = child->next_sibling) { + if (child->type == TH_NODE_ELEMENT) { + schema->sqnames[*at].node = child; + schema->sqnames[*at].name = node_qname(schema->tree, child, child->text, child->text_len, 0); + (*at)++; + schema_fill_qnames(schema, child, at); + } + } +} + +static int sqname_cmp(const void *left, const void *right) { + th_node *left_node = ((const sqname_entry *)left)->node; + th_node *right_node = ((const sqname_entry *)right)->node; + return (left_node > right_node) - (left_node < right_node); +} + +static int schema_build_qname_cache(th_schema *schema) { + th_node *document = th_tree_document(schema->tree); + Py_ssize_t count = schema_count_elements(document); + schema->sqnames = arena_alloc(&schema->mem, (size_t)count * sizeof(sqname_entry)); + if (schema->sqnames == NULL) { /* GCOVR_EXCL_BR_LINE: arena OOM is unforceable */ + return -1; /* GCOVR_EXCL_LINE */ + } + Py_ssize_t at = 0; + schema_fill_qnames(schema, document, &at); + schema->sqname_count = count; + qsort(schema->sqnames, (size_t)count, sizeof(sqname_entry), sqname_cmp); + return 0; +} + static int named_push(th_schema *schema, named_vec *vec, const Py_UCS4 *name, Py_ssize_t len, th_node *node) { if (vec->len == vec->cap) { Py_ssize_t cap = vec->cap ? vec->cap * 2 : 8; @@ -625,6 +704,10 @@ PyObject *turbohtml_schema_compile(PyObject *module, PyObject *args) { schema->tree = tree; schema->source = Py_NewRef(source); schema->root = document_root(tree); + if (schema_build_qname_cache(schema) < 0) { /* GCOVR_EXCL_BR_LINE: arena OOM is unforceable */ + schema_free(schema); /* GCOVR_EXCL_LINE */ + return PyErr_NoMemory(); /* GCOVR_EXCL_LINE */ + } int ok = kind == 0 ? xsd_compile(schema) : rng_compile(schema); if (!ok) { schema_free(schema); diff --git a/src/turbohtml/_c/validate/xsd.h b/src/turbohtml/_c/validate/xsd.h index a6fc9f3d..bb182ffc 100644 --- a/src/turbohtml/_c/validate/xsd.h +++ b/src/turbohtml/_c/validate/xsd.h @@ -24,9 +24,9 @@ typedef struct { } edecl_vec; /* Whether `node` is an XSD element whose local name is one of the listed particles. */ -static int xsd_is_particle(th_tree *tree, th_node *node, const char *const *locals, size_t count) { +static int xsd_is_particle(const th_schema *schema, th_node *node, const char *const *locals, size_t count) { for (size_t index = 0; index < count; index++) { - if (is_schema_el(tree, node, XSD_NS, locals[index])) { + if (is_schema_el(schema, node, XSD_NS, locals[index])) { return 1; } } @@ -117,7 +117,7 @@ static void xsd_element_name(th_schema *schema, th_node *particle, edecl *out) { out->local_len = name_len; out->decl = particle; /* an element particle always has a parent (its model group or the schema) */ - int global = is_schema_el(tree, particle->parent, XSD_NS, "schema"); + int global = is_schema_el(schema, particle->parent, XSD_NS, "schema"); if (global || schema->element_qualified) { out->uri = schema->target_ns; out->uri_len = schema->target_ns_len; @@ -146,7 +146,7 @@ static th_node *xsd_group_model(th_schema *schema, th_node *group_ref) { return NULL; } for (th_node *child = group->first_child; child != NULL; child = child->next_sibling) { - if (xsd_is_particle(schema->tree, child, XSD_MODEL_GROUPS, 3)) { + if (xsd_is_particle(schema, child, XSD_MODEL_GROUPS, 3)) { return child; } } /* GCOVR_EXCL_LINE: llvm miscredits this loop-exit brace when the group has no model group */ @@ -173,14 +173,13 @@ static int edecl_push(th_schema *schema, edecl_vec *vec, const edecl *item) { /* Gather every element declaration reachable in a particle so a child can find its type by name (unique-particle-attribution makes the name→decl mapping unambiguous). */ static void xsd_collect_edecls(th_schema *schema, th_node *particle, edecl_vec *vec) { - th_tree *tree = schema->tree; - if (is_schema_el(tree, particle, XSD_NS, "element")) { + if (is_schema_el(schema, particle, XSD_NS, "element")) { edecl want; xsd_element_name(schema, particle, &want); edecl_push(schema, vec, &want); return; } - if (is_schema_el(tree, particle, XSD_NS, "group")) { + if (is_schema_el(schema, particle, XSD_NS, "group")) { th_node *model = xsd_group_model(schema, particle); if (model != NULL) { xsd_collect_edecls(schema, model, vec); @@ -188,7 +187,7 @@ static void xsd_collect_edecls(th_schema *schema, th_node *particle, edecl_vec * return; } for (th_node *child = particle->first_child; child != NULL; child = child->next_sibling) { - if (xsd_is_particle(tree, child, XSD_PARTICLES, 5)) { + if (xsd_is_particle(schema, child, XSD_PARTICLES, 5)) { xsd_collect_edecls(schema, child, vec); } } @@ -271,9 +270,8 @@ static void xsd_match_occurs(th_schema *schema, th_tree *inst, th_node *particle static void xsd_match_once(th_schema *schema, th_tree *inst, th_node *particle, th_node **children, const posset *in, posset *out) { - th_tree *tree = schema->tree; memset(out->pos, 0, (size_t)out->size); - if (is_schema_el(tree, particle, XSD_NS, "element")) { + if (is_schema_el(schema, particle, XSD_NS, "element")) { edecl want; xsd_element_name(schema, particle, &want); for (Py_ssize_t pos = 0; pos + 1 < in->size; pos++) { @@ -287,14 +285,14 @@ static void xsd_match_once(th_schema *schema, th_tree *inst, th_node *particle, } return; } - if (is_schema_el(tree, particle, XSD_NS, "group")) { + if (is_schema_el(schema, particle, XSD_NS, "group")) { th_node *model = xsd_group_model(schema, particle); if (model != NULL) { xsd_match_once(schema, inst, model, children, in, out); } return; } - if (is_schema_el(tree, particle, XSD_NS, "choice")) { + if (is_schema_el(schema, particle, XSD_NS, "choice")) { for (th_node *child = particle->first_child; child != NULL; child = child->next_sibling) { if (child->type != TH_NODE_ELEMENT) { continue; @@ -341,7 +339,7 @@ static void xsd_validate_all(valctx *ctx, th_node *all, th_node **children, Py_s qname name = node_qname(inst, children[index], children[index]->text, children[index]->text_len, 0); th_node *matched = NULL; for (th_node *particle = all->first_child; particle != NULL; particle = particle->next_sibling) { - if (!is_schema_el(tree, particle, XSD_NS, "element")) { + if (!is_schema_el(schema, particle, XSD_NS, "element")) { continue; } edecl want; @@ -362,7 +360,7 @@ static void xsd_validate_all(valctx *ctx, th_node *all, th_node **children, Py_s } } for (th_node *particle = all->first_child; particle != NULL; particle = particle->next_sibling) { - if (!is_schema_el(tree, particle, XSD_NS, "element") || xsd_occurs_min(tree, particle) == 0) { + if (!is_schema_el(schema, particle, XSD_NS, "element") || xsd_occurs_min(tree, particle) == 0) { continue; } edecl want; @@ -405,7 +403,7 @@ static int xsd_base_id(th_schema *schema, th_node *owner, const Py_UCS4 *base, P static int xsd_gather_facets(th_schema *schema, th_node *simpletype, facetset *facets, int depth) { th_tree *tree = schema->tree; - th_node *restriction = first_schema_child(tree, simpletype, XSD_NS, "restriction"); + th_node *restriction = first_schema_child(schema, simpletype, XSD_NS, "restriction"); if (restriction == NULL) { /* list / union fall back to a permissive string base */ return facets->base_id; } @@ -415,7 +413,7 @@ static int xsd_gather_facets(th_schema *schema, th_node *simpletype, facetset *f if (base != NULL) { base_id = xsd_base_id(schema, restriction, base, base_len, facets, depth); } else { - th_node *inner = first_schema_child(tree, restriction, XSD_NS, "simpleType"); + th_node *inner = first_schema_child(schema, restriction, XSD_NS, "simpleType"); if (inner != NULL) { base_id = xsd_gather_facets(schema, inner, facets, depth + 1); } @@ -522,7 +520,7 @@ static void xsd_validate_attr_decl(valctx *ctx, th_node *instance, th_node *decl } Py_ssize_t type_len = 0; const Py_UCS4 *type = xsd_attr(tree, effective, "type", &type_len); - th_node *inline_type = first_schema_child(tree, effective, XSD_NS, "simpleType"); + th_node *inline_type = first_schema_child(schema, effective, XSD_NS, "simpleType"); if (type != NULL) { facetset probe; facetset_init(&probe, DT_STRING); @@ -549,7 +547,7 @@ static void xsd_collect_attrs(valctx *ctx, th_node *instance, th_node *scope, ed th_schema *schema = ctx->schema; th_tree *tree = schema->tree; for (th_node *child = scope->first_child; child != NULL; child = child->next_sibling) { - if (is_schema_el(tree, child, XSD_NS, "attribute")) { + if (is_schema_el(schema, child, XSD_NS, "attribute")) { xsd_validate_attr_decl(ctx, instance, child); th_node *eff = child; Py_ssize_t ref_len = 0; @@ -567,7 +565,7 @@ static void xsd_collect_attrs(valctx *ctx, th_node *instance, th_node *scope, ed edecl item = {NULL, 0, name, name_len, eff}; edecl_push(schema, declared, &item); } - } else if (is_schema_el(tree, child, XSD_NS, "attributeGroup")) { + } else if (is_schema_el(schema, child, XSD_NS, "attributeGroup")) { Py_ssize_t ref_len = 0; const Py_UCS4 *ref = xsd_attr(tree, child, "ref", &ref_len); if (ref != NULL) { @@ -579,11 +577,11 @@ static void xsd_collect_attrs(valctx *ctx, th_node *instance, th_node *scope, ed xsd_collect_attrs(ctx, instance, group, declared); } } - } else if (is_schema_el(tree, child, XSD_NS, "complexContent") || - is_schema_el(tree, child, XSD_NS, "simpleContent")) { - th_node *derivation = first_schema_child(tree, child, XSD_NS, "extension"); + } else if (is_schema_el(schema, child, XSD_NS, "complexContent") || + is_schema_el(schema, child, XSD_NS, "simpleContent")) { + th_node *derivation = first_schema_child(schema, child, XSD_NS, "extension"); if (derivation == NULL) { - derivation = first_schema_child(tree, child, XSD_NS, "restriction"); + derivation = first_schema_child(schema, child, XSD_NS, "restriction"); } if (derivation != NULL) { Py_ssize_t base_len = 0; @@ -637,10 +635,10 @@ static void xsd_validate_attrs(valctx *ctx, th_node *instance, th_node *scope, e /* ---- complex-type content ---- */ -static th_node *xsd_content_model(th_tree *tree, th_node *scope) { +static th_node *xsd_content_model(const th_schema *schema, th_node *scope) { static const char *const models[] = {"sequence", "choice", "all", "group"}; for (size_t index = 0; index < sizeof(models) / sizeof(models[0]); index++) { - th_node *model = first_schema_child(tree, scope, XSD_NS, models[index]); + th_node *model = first_schema_child(schema, scope, XSD_NS, models[index]); if (model != NULL) { return model; } @@ -653,11 +651,11 @@ static th_node *xsd_content_model(th_tree *tree, th_node *scope) { static th_node *xsd_effective_model(th_schema *schema, th_node *ctype, th_node **base_model_out) { th_tree *tree = schema->tree; *base_model_out = NULL; - th_node *complex_content = first_schema_child(tree, ctype, XSD_NS, "complexContent"); + th_node *complex_content = first_schema_child(schema, ctype, XSD_NS, "complexContent"); if (complex_content != NULL) { - th_node *extension = first_schema_child(tree, complex_content, XSD_NS, "extension"); + th_node *extension = first_schema_child(schema, complex_content, XSD_NS, "extension"); th_node *derivation = - extension != NULL ? extension : first_schema_child(tree, complex_content, XSD_NS, "restriction"); + extension != NULL ? extension : first_schema_child(schema, complex_content, XSD_NS, "restriction"); if (derivation == NULL) { return NULL; } @@ -675,16 +673,17 @@ static th_node *xsd_effective_model(th_schema *schema, th_node *ctype, th_node * } } } - return xsd_content_model(tree, derivation); + return xsd_content_model(schema, derivation); } - return xsd_content_model(tree, ctype); + return xsd_content_model(schema, ctype); } -static int xsd_is_mixed(th_tree *tree, th_node *ctype) { +static int xsd_is_mixed(const th_schema *schema, th_node *ctype) { + th_tree *tree = schema->tree; if (xsd_attr_is(tree, ctype, "mixed", "true")) { return 1; } - th_node *complex_content = first_schema_child(tree, ctype, XSD_NS, "complexContent"); + th_node *complex_content = first_schema_child(schema, ctype, XSD_NS, "complexContent"); if (complex_content != NULL) { return xsd_attr_is(tree, complex_content, "mixed", "true"); } @@ -698,7 +697,7 @@ static void xsd_validate_complex(valctx *ctx, th_node *instance, th_node *ctype) edecl_vec declared = {NULL, 0, 0}; xsd_validate_attrs(ctx, instance, ctype, &declared); - th_node *simple_content = first_schema_child(tree, ctype, XSD_NS, "simpleContent"); + th_node *simple_content = first_schema_child(schema, ctype, XSD_NS, "simpleContent"); if (simple_content != NULL) { for (th_node *child = instance->first_child; child != NULL; child = child->next_sibling) { if (child->type == TH_NODE_ELEMENT) { @@ -706,9 +705,9 @@ static void xsd_validate_complex(valctx *ctx, th_node *instance, th_node *ctype) break; } } - th_node *extension = first_schema_child(tree, simple_content, XSD_NS, "extension"); + th_node *extension = first_schema_child(schema, simple_content, XSD_NS, "extension"); th_node *derivation = - extension != NULL ? extension : first_schema_child(tree, simple_content, XSD_NS, "restriction"); + extension != NULL ? extension : first_schema_child(schema, simple_content, XSD_NS, "restriction"); Py_ssize_t text_len = 0; const Py_UCS4 *text = element_text(ctx, instance, &text_len); if (derivation != NULL) { @@ -724,7 +723,7 @@ static void xsd_validate_complex(valctx *ctx, th_node *instance, th_node *ctype) th_node *base_model; th_node *model = xsd_effective_model(schema, ctype, &base_model); - int mixed = xsd_is_mixed(tree, ctype); + int mixed = xsd_is_mixed(schema, ctype); if (!mixed) { for (th_node *child = instance->first_child; child != NULL; child = child->next_sibling) { if (is_chardata(child) && !th_node_text_is_blank(inst, child)) { @@ -751,7 +750,7 @@ static void xsd_validate_complex(valctx *ctx, th_node *instance, th_node *ctype) } } - if (model != NULL && is_schema_el(tree, model, XSD_NS, "all")) { + if (model != NULL && is_schema_el(schema, model, XSD_NS, "all")) { xsd_validate_all(ctx, model, children, nchildren); return; } @@ -831,8 +830,8 @@ static void xsd_validate_element_inner(valctx *ctx, th_node *instance, th_node * Py_ssize_t type_len = 0; const Py_UCS4 *type = xsd_attr(tree, decl, "type", &type_len); - th_node *inline_complex = first_schema_child(tree, decl, XSD_NS, "complexType"); - th_node *inline_simple = first_schema_child(tree, decl, XSD_NS, "simpleType"); + th_node *inline_complex = first_schema_child(schema, decl, XSD_NS, "complexType"); + th_node *inline_simple = first_schema_child(schema, decl, XSD_NS, "simpleType"); if (type != NULL) { const Py_UCS4 *local, *prefix, *uri; Py_ssize_t local_len = 0, prefix_len = 0, uri_len = 0; @@ -869,7 +868,7 @@ static void xsd_validate_element_inner(valctx *ctx, th_node *instance, th_node * static int xsd_compile(th_schema *schema) { th_tree *tree = schema->tree; - if (!is_schema_el(tree, schema->root, XSD_NS, "schema")) { + if (!is_schema_el(schema, schema->root, XSD_NS, "schema")) { PyErr_SetString(PyExc_ValueError, "root element is not an xs:schema"); return 0; } @@ -890,17 +889,17 @@ static int xsd_compile(th_schema *schema) { continue; } named_vec *bucket = NULL; - if (is_schema_el(tree, child, XSD_NS, "element")) { + if (is_schema_el(schema, child, XSD_NS, "element")) { bucket = &schema->elements; - } else if (is_schema_el(tree, child, XSD_NS, "complexType")) { + } else if (is_schema_el(schema, child, XSD_NS, "complexType")) { bucket = &schema->complex_types; - } else if (is_schema_el(tree, child, XSD_NS, "simpleType")) { + } else if (is_schema_el(schema, child, XSD_NS, "simpleType")) { bucket = &schema->simple_types; - } else if (is_schema_el(tree, child, XSD_NS, "attribute")) { + } else if (is_schema_el(schema, child, XSD_NS, "attribute")) { bucket = &schema->attributes; - } else if (is_schema_el(tree, child, XSD_NS, "group")) { + } else if (is_schema_el(schema, child, XSD_NS, "group")) { bucket = &schema->groups; - } else if (is_schema_el(tree, child, XSD_NS, "attributeGroup")) { + } else if (is_schema_el(schema, child, XSD_NS, "attributeGroup")) { bucket = &schema->attr_groups; } if (bucket != NULL) { From ccaf4e8ccc6893919c4da5c8031e44dbbb2b312d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Mon, 20 Jul 2026 21:22:29 -0700 Subject: [PATCH 23/54] perf(links): skip wasted wraps and absolute joins Two independent read-path wins, each verified with Callgrind in Docker on the 95 kB mozilla page (instructions retired, not wall clock, which swings 7-30% on this machine). links(): the walk wrapped every element in a Python Element before scanning it, then dropped the wrapper, even though a real page carries a link on a small fraction of its elements. Mint the wrapper lazily on the first link an element emits. 59,171,837 -> 50,408,887 Ir (-14.8%). resolve_links(): urljoin, the stdlib join it stands in for, is ~99% of the cost and returns an already-absolute reference byte-for-byte. Prove that in C for the common cases -- a scheme differing from the base's, or a same-scheme scheme://authority under an http/https base with a lowercase scheme and no tab/CR/LF -- and skip the call. Output stays identical (the rewrite already no-ops an unchanged join), so no new benchmark id. 274,610,778 -> 88,284,235 Ir (-67.8%). --- src/turbohtml/_c/extract/links.c | 152 +++++++++++++++++++++++----- tests/extract/test_links_resolve.py | 80 +++++++++++++++ 2 files changed, 205 insertions(+), 27 deletions(-) diff --git a/src/turbohtml/_c/extract/links.c b/src/turbohtml/_c/extract/links.c index 2d575c56..07bcfdcc 100644 --- a/src/turbohtml/_c/extract/links.c +++ b/src/turbohtml/_c/extract/links.c @@ -223,6 +223,11 @@ typedef struct { PyObject *owner; /* the element/document the public call was made on, for wrapping enumerated nodes */ PyObject *result; /* list[Link] for enumerate, else NULL */ PyObject *replace; /* callable (str) -> str | None for rewrite, else NULL */ + /* resolve_links only: the base URL's lowercased scheme, or NULL for the enumerate/rewrite paths. When set, a link + that urljoin(base, link) would return byte-for-byte (an absolute URL) is left untouched without the call. */ + const char *base_scheme; + Py_ssize_t base_scheme_len; + int base_netloc_scheme; /* the base scheme reserializes an absolute same-scheme URL unchanged (http/https) */ link_span *spans; Py_ssize_t span_count; Py_ssize_t span_cap; @@ -282,15 +287,23 @@ static PyObject *ucs4_slice(const Py_UCS4 *value, Py_ssize_t start, Py_ssize_t e return PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, value + start, end - start); } -/* Append a Link(element, attr, url) for each span to the result list. `report_element` is the owning Element wrapper - and `report_attr` its attribute name (or None for