From 3584c27f3f2c2e2fe26cf8cdeaf9beaeba1114bb Mon Sep 17 00:00:00 2001 From: Tyler Riccio Date: Tue, 9 Jun 2026 18:58:27 -0400 Subject: [PATCH 1/4] test: add append-only public API contract and surface snapshot Freeze the public contract so it can't change silently within a major version: - tests/contract/test_surface.py pins __all__ and the exact signature of every exported function. - tests/contract/test_behavior.py freezes documented round-trips, dtypes, errors, the length-prefixed header shape, and the token wire format via a frozen golden token (cross-version decode stability). - CONTRACT.md documents the SemVer promise and the append-only rule for tests/contract/. --- CONTRACT.md | 50 ++++++++++++++ tests/contract/__init__.py | 0 tests/contract/test_behavior.py | 113 ++++++++++++++++++++++++++++++++ tests/contract/test_surface.py | 59 +++++++++++++++++ 4 files changed, 222 insertions(+) create mode 100644 CONTRACT.md create mode 100644 tests/contract/__init__.py create mode 100644 tests/contract/test_behavior.py create mode 100644 tests/contract/test_surface.py diff --git a/CONTRACT.md b/CONTRACT.md new file mode 100644 index 0000000..9c6ecaa --- /dev/null +++ b/CONTRACT.md @@ -0,0 +1,50 @@ +# Public API Contract + +This document defines the stability promise `pl-row-encode` makes to its users and the +rules that keep that promise mechanically enforceable. + +## The promise (SemVer) + +We follow [Semantic Versioning](https://semver.org/). For a given **major** version: + +- The **public surface** is frozen: the names exported from `pl_row_encode.__all__`, and + their call signatures, do not change in a breaking way. +- The **documented behavior** of those functions does not change in a breaking way. +- The **token wire format** is stable: a token produced by any release within a major + version decodes correctly with any later release in the same major version. + +Additions (new functions, new *optional* keyword arguments, new accepted input types) are +allowed in **minor** releases. Bug fixes go in **patch** releases. + +Anything that breaks one of the three guarantees above requires a **major** version bump +and an entry in the changelog. + +## How it is enforced + +The contract is not a gentleman's agreement — it is executable. Two layers live under +`tests/contract/`: + +| File | Guards | +|------|--------| +| `test_surface.py` | The exported symbols and their exact signatures. | +| `test_behavior.py` | Documented round-trip behavior, dtypes, errors, and the token wire format (via a frozen golden token). | + +### The append-only rule + +Within a major version, the tests in `tests/contract/` are **append-only**: + +- You **may** add new contract tests (covering new, additive API). +- You **may not** weaken, change, or delete an existing contract test. + +If a change requires editing an existing contract assertion, that change is breaking by +definition. Do not edit the assertion to make CI green — instead, bump the major version +and record the break here and in the changelog. + +A red contract suite means "this is a breaking change." That is the whole point. + +## Updating the surface snapshot + +`test_surface.py` pins exact signatures. When you make an **additive** change (e.g. a new +function, or a new optional keyword argument), update the snapshot in the same PR and +release it as a minor version. When you find yourself wanting to update it to *remove* or +*retype* something, that is a major change. diff --git a/tests/contract/__init__.py b/tests/contract/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/contract/test_behavior.py b/tests/contract/test_behavior.py new file mode 100644 index 0000000..4d6f6f7 --- /dev/null +++ b/tests/contract/test_behavior.py @@ -0,0 +1,113 @@ +"""Freeze the *behavior* users depend on: round-trips, dtypes, errors, wire format. + +APPEND-ONLY within a major version. You may add tests for new, additive behavior; you may +not weaken or delete an assertion here. A red test in this file means a breaking change. +See CONTRACT.md. +""" + +import polars as pl +import pytest +from polars.testing import assert_series_equal + +from pl_row_encode import ( + decode, + decode_peek, + decode_series, + encode, + encode_series, + get_header, +) + +# --- round-trip behavior -------------------------------------------------- + + +def test_encode_produces_binary_token(): + df = pl.DataFrame({"x": [1, 2], "y": ["a", "b"]}) + out = df.select(encode("x", "y").alias("tok")) + assert out.schema == pl.Schema({"tok": pl.Binary}) + + +def test_lazy_roundtrip_via_decode_with_header(): + df = pl.DataFrame({"x": [1, 2], "y": ["a", "b"]}).select(encode("x", "y").alias("tok")) + header = get_header(df.to_series()) + rows = df.select(decode("tok", schema_header=header).alias("row")).to_series().to_list() + assert rows == [{"x": 1, "y": "a"}, {"x": 2, "y": "b"}] + + +def test_lazy_roundtrip_via_decode_peek(): + df = pl.DataFrame({"x": [1, 2], "y": ["a", "b"]}).select(encode("x", "y").alias("tok")) + rows = df.select(decode_peek(df, "tok").alias("row")).to_series().to_list() + assert rows == [{"x": 1, "y": "a"}, {"x": 2, "y": "b"}] + + +def test_eager_roundtrip_via_series(): + tok = encode_series(pl.Series("x", [10, 20])) + assert tok.dtype == pl.Binary + assert decode_series(tok).to_list() == [{"x": 10}, {"x": 20}] + + +def test_encode_series_matches_encode_expr(): + df = pl.DataFrame({"a": [1, 2, 3], "b": ["x", "y", "z"]}) + via_expr = df.select(tok=encode("a", "b"))["tok"] + via_series = encode_series(df["a"], df["b"]) + assert_series_equal(via_series.rename("tok"), via_expr) + + +# --- documented errors ---------------------------------------------------- + + +def test_encode_requires_input(): + with pytest.raises(ValueError, match="at least one column"): + encode() + + +def test_encode_series_requires_input(): + with pytest.raises(ValueError, match="at least one series"): + encode_series() + + +def test_get_header_rejects_all_null_series(): + with pytest.raises(ValueError, match="all-null"): + get_header(pl.Series("x", [None], dtype=pl.Binary)) + + +def test_get_header_rejects_short_token(): + with pytest.raises(ValueError, match="too short"): + get_header(b"\x00") + + +def test_decode_peek_rejects_no_non_null_token(): + df = pl.DataFrame({"tok": pl.Series([None], dtype=pl.Binary)}) + with pytest.raises(ValueError, match="no non-null token"): + decode_peek(df, "tok") + + +# --- header shape --------------------------------------------------------- + + +def test_header_is_length_prefixed(): + import struct + + tok = encode_series(pl.Series("x", [1, 2, 3])) + header = get_header(tok) + (header_len,) = struct.unpack_from(" 'pl.Expr'", + "encode_series": "(*series: 'pl.Series') -> 'pl.Series'", + "get_header": "(token: 'bytes | pl.Series') -> 'bytes'", + "decode": "(expr: 'IntoExpr', *, schema_header: 'bytes') -> 'pl.Expr'", + "decode_peek": "(frame: 'Frame', column: 'str') -> 'pl.Expr'", + "decode_series": "(s: 'pl.Series') -> 'pl.Series'", +} + + +def test_all_matches_frozen_set(): + assert set(m.__all__) == EXPORTED + + +def test_exported_symbols_are_importable(): + for name in EXPORTED: + assert hasattr(m, name), f"{name} is exported in __all__ but not importable" + + +def test_no_undeclared_signatures(): + # Every exported symbol must have a pinned signature below, so new public API can't + # slip in without consciously freezing its signature. + assert set(SIGNATURES) == EXPORTED + + +def test_signatures_frozen(): + for name, expected in SIGNATURES.items(): + actual = str(inspect.signature(getattr(m, name))) + assert actual == expected, ( + f"signature of {name} changed: {actual!r} != frozen {expected!r}. " + "If additive, update SIGNATURES (minor release); if not, bump major." + ) From c535d2da3a9d1e14e864cd2a16eec4137c033624 Mon Sep 17 00:00:00 2001 From: Tyler Riccio Date: Tue, 9 Jun 2026 19:03:50 -0400 Subject: [PATCH 2/4] test: run public docstring examples as doctests Add --doctest-modules so the >>> examples in pl_row_encode's public docstrings execute under pytest. This makes the documented usage a third contract layer (alongside tests/contract/) and guarantees the examples stay runnable. All 6 module doctests pass with no repr brittleness. --- pyproject.toml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 13b47e0..b1f708e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,12 @@ build-backend = "maturin" python-source = "." module-name = "pl_row_encode._internal" +[tool.pytest.ini_options] +# Run the >>> examples in the public docstrings as tests, so the documented +# usage can't silently rot. This is the third contract layer alongside +# tests/contract/ (see CONTRACT.md). +addopts = "--doctest-modules" + [dependency-groups] dev = [ "ruff>=0.15.15", From 1f0ed35488e9476727f656a40eea6af0d37be719 Mon Sep 17 00:00:00 2001 From: Tyler Riccio Date: Tue, 9 Jun 2026 19:27:27 -0400 Subject: [PATCH 3/4] format --- tests/contract/test_behavior.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/contract/test_behavior.py b/tests/contract/test_behavior.py index 4d6f6f7..5bec28f 100644 --- a/tests/contract/test_behavior.py +++ b/tests/contract/test_behavior.py @@ -28,14 +28,22 @@ def test_encode_produces_binary_token(): def test_lazy_roundtrip_via_decode_with_header(): - df = pl.DataFrame({"x": [1, 2], "y": ["a", "b"]}).select(encode("x", "y").alias("tok")) + df = pl.DataFrame({"x": [1, 2], "y": ["a", "b"]}).select( + encode("x", "y").alias("tok") + ) header = get_header(df.to_series()) - rows = df.select(decode("tok", schema_header=header).alias("row")).to_series().to_list() + rows = ( + df.select(decode("tok", schema_header=header).alias("row")) + .to_series() + .to_list() + ) assert rows == [{"x": 1, "y": "a"}, {"x": 2, "y": "b"}] def test_lazy_roundtrip_via_decode_peek(): - df = pl.DataFrame({"x": [1, 2], "y": ["a", "b"]}).select(encode("x", "y").alias("tok")) + df = pl.DataFrame({"x": [1, 2], "y": ["a", "b"]}).select( + encode("x", "y").alias("tok") + ) rows = df.select(decode_peek(df, "tok").alias("row")).to_series().to_list() assert rows == [{"x": 1, "y": "a"}, {"x": 2, "y": "b"}] From 931a7116b192df8b0d953cdbdb01032da697bb60 Mon Sep 17 00:00:00 2001 From: Tyler Riccio Date: Wed, 10 Jun 2026 15:59:31 -0400 Subject: [PATCH 4/4] feat: three-tier dtype guard so encodable implies decodable Replace the categorical-only denylist with an allowlist-based classifier: known-good dtypes encode silently, Categorical is hard-rejected, and any unvetted dtype emits a UserWarning that it may fail or panic on decode. This keeps the encodable-implies-decodable invariant general without maintaining an exhaustive denylist, while never running the dangerous decode probe on an unknown type. --- CONTRACT.md | 20 ++++++++++ src/lib.rs | 84 ++++++++++++++++++++++++++++++++++++------ tests/test_property.py | 33 +++++++++++++++++ 3 files changed, 126 insertions(+), 11 deletions(-) diff --git a/CONTRACT.md b/CONTRACT.md index 9c6ecaa..fade2b3 100644 --- a/CONTRACT.md +++ b/CONTRACT.md @@ -42,6 +42,26 @@ and record the break here and in the changelog. A red contract suite means "this is a breaking change." That is the whole point. +## Dtype safety: encodable implies decodable + +The core invariant is that a token we hand back can always be decoded — we never emit +bytes that only blow up on the return trip. `encode` classifies every (recursively +nested) input dtype into three tiers at encode time: + +- **Known-good** — dtypes verified to round-trip losslessly (the allowlist in + `classify_dtype`, guarded by the dtype-matrix property tests). These encode silently. +- **Known-bad** — `Categorical`. Its category→string mapping lives in an external string + cache, not in the token, so decoding would panic. Hard-rejected at encode with an + actionable `ComputeError` (use `Enum`, or cast to `String`). +- **Unknown** — anything we haven't vetted. We cannot safely probe it (the probe is the + dangerous decode itself), so encode emits a `UserWarning` that the token may fail or + panic on decode, then proceeds. + +The allowlist is keyed to the **compiled** polars crate version (`Cargo.toml`), not the +user's runtime polars: the plugin decodes with its own embedded `polars-row`, so the set +of safe dtypes is fixed at build time. Bumping the polars crate is the trigger to re-run +the dtype-matrix tests and re-validate the allowlist. + ## Updating the surface snapshot `test_surface.py` pins exact signatures. When you make an **additive** change (e.g. a new diff --git a/src/lib.rs b/src/lib.rs index f6136fa..2d6b18a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -41,30 +41,88 @@ fn fields_from_prefixed(buf: &[u8]) -> PolarsResult> { deserialize_header(&buf[4..end]) } -/// Categorical's category->string mapping lives in an external string cache, not in the -/// dtype or the row bytes, so a token can't carry it -- decoding would panic in -/// `cat_to_str`. Reject it (recursing through nested containers) at encode time with an -/// actionable message instead of emitting a token that only blows up on decode. Enum is -/// fine: its categories live in the dtype and ride along in the serialized header. -fn reject_categorical(dtype: &DataType) -> PolarsResult<()> { +/// Classify a dtype against three tiers to uphold the invariant that anything we encode +/// can be decoded (see CONTRACT.md): +/// +/// * **known-good** -- dtypes whose polars-row encode/decode we've verified round-trips +/// losslessly (the allowlist below, guarded by the dtype-matrix property tests). These +/// encode silently. +/// * **known-bad** -- `Categorical`. Its category->string mapping lives in an external +/// string cache, not in the dtype or row bytes, so a token can't carry it; decoding +/// panics in `cat_to_str`. Hard-rejected with an actionable message instead of +/// emitting a token that blows up on decode. (Enum is fine: its categories live in the +/// dtype and ride along in the serialized header.) +/// * **unknown** -- anything else (new/exotic dtypes we haven't vetted). We can't safely +/// probe it (the probe is the dangerous decode), so its name is collected and the +/// caller emits a warning: the token may fail or panic on decode. +/// +/// Recurses through `List` / `Array` / `Struct` so a nested offender is caught too. The +/// allowlist is keyed to the compiled polars crate version (Cargo.toml), not the user's +/// runtime polars, since the plugin decodes with its own embedded `polars-row`. +fn classify_dtype(dtype: &DataType, unknown: &mut Vec) -> PolarsResult<()> { match dtype { DataType::Categorical(_, _) => polars_bail!( ComputeError: "Categorical columns cannot be encoded: the category mapping is not embeddable \ in a token. Use Enum, or cast to String before encoding." ), - DataType::List(inner) => reject_categorical(inner), - DataType::Array(inner, _) => reject_categorical(inner), + + // Allowlist: leaves verified to round-trip (see tests/test_property.py). + DataType::Boolean + | DataType::UInt8 + | DataType::UInt16 + | DataType::UInt32 + | DataType::UInt64 + | DataType::Int8 + | DataType::Int16 + | DataType::Int32 + | DataType::Int64 + | DataType::Int128 + | DataType::Float32 + | DataType::Float64 + | DataType::Decimal(_, _) + | DataType::String + | DataType::Binary + | DataType::Date + | DataType::Time + | DataType::Datetime(_, _) + | DataType::Duration(_) + | DataType::Enum(_, _) => Ok(()), + + // Containers: known-good iff their inner dtype(s) are. + DataType::List(inner) | DataType::Array(inner, _) => classify_dtype(inner, unknown), DataType::Struct(fields) => { for f in fields { - reject_categorical(f.dtype())?; + classify_dtype(f.dtype(), unknown)?; } Ok(()) } - _ => Ok(()), + + other => { + unknown.push(format!("{other}")); + Ok(()) + } } } +/// Emit a Python `UserWarning` naming dtypes that aren't on the known-good allowlist. +/// Reaching Python from inside a polars expr requires grabbing the GIL; failures to warn +/// are swallowed (a warning must never break an encode). +fn warn_unknown_dtypes(names: &[String]) { + use pyo3::prelude::*; + + let msg = format!( + "pl_row_encode: dtype(s) [{}] are not in the known round-trip-safe set; the \ + resulting token may fail or panic on decode. If decode works, please report it \ + so the dtype can be allowlisted.", + names.join(", "), + ); + let _ = Python::attach(|py| -> PyResult<()> { + py.import("warnings")?.call_method1("warn", (msg,))?; + Ok(()) + }); +} + #[polars_expr(output_type=Binary)] fn row_encode(inputs: &[Series]) -> PolarsResult { if inputs.is_empty() { @@ -74,8 +132,12 @@ fn row_encode(inputs: &[Series]) -> PolarsResult { let columns: Vec = inputs.iter().cloned().map(Column::from).collect(); let fields: Vec = inputs.iter().map(|s| s.field().into_owned()).collect(); + let mut unknown: Vec = Vec::new(); for field in &fields { - reject_categorical(field.dtype())?; + classify_dtype(field.dtype(), &mut unknown)?; + } + if !unknown.is_empty() { + warn_unknown_dtypes(&unknown); } let header = serialize_header(&fields)?; diff --git a/tests/test_property.py b/tests/test_property.py index d9dc4f0..ee60072 100644 --- a/tests/test_property.py +++ b/tests/test_property.py @@ -6,6 +6,8 @@ header plumbing, these will find it. """ +import warnings + import polars as pl import pytest from hypothesis import given, settings @@ -188,3 +190,34 @@ def test_nested_categorical_rejected_at_encode() -> None: pl.exceptions.ComputeError, match="Categorical columns cannot be encoded" ): df.select(tok=encode("a")) + + +# --- unknown-dtype tier: warn-and-proceed ------------------------------------------- +# +# The encode-time guard has three tiers (see CONTRACT.md / classify_dtype in src/lib.rs): +# known-good dtypes encode silently, Categorical is hard-rejected, and anything we haven't +# vetted is "unknown" -- we can't safely probe it (the probe is the dangerous decode), so +# we warn that the token may fail or panic on decode but still proceed. Null is a stable +# example of a dtype that crosses the FFI but isn't on the allowlist. + + +def test_unknown_dtype_warns_at_encode() -> None: + df = pl.DataFrame({"a": pl.Series("a", [None, None], dtype=pl.Null)}) + with pytest.warns(UserWarning, match="not in the known round-trip-safe set"): + df.select(tok=encode("a")) + + +def test_nested_unknown_dtype_warns_at_encode() -> None: + # Classification recurses through containers, so an unknown inner dtype warns too. + df = pl.DataFrame({"a": pl.Series("a", [[None]], dtype=pl.List(pl.Null))}) + with pytest.warns(UserWarning, match=r"\[null\]"): + df.select(tok=encode("a")) + + +@pytest.mark.parametrize("dtype", ROUNDTRIP_DTYPES) +def test_allowlisted_dtypes_do_not_warn(dtype: pl.DataType) -> None: + # Known-good dtypes must encode silently -- no spurious unknown-dtype warning. + df = pl.DataFrame({"a": pl.Series("a", [None], dtype=dtype)}) + with warnings.catch_warnings(): + warnings.simplefilter("error") # any warning becomes an exception + df.select(tok=encode("a"))