From 1d45e89696022c7cdf3f5e8969513b2c2938f7cf Mon Sep 17 00:00:00 2001 From: zolizoli Date: Tue, 30 Jun 2026 23:58:15 +0200 Subject: [PATCH] Phase 1: the spine (segment -> layout -> encode -> render) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit End-to-end pipeline on the bundled text, returning a matplotlib Figure. - segment/units.py: segment() over chars/tokens/sentences. Offline, abbreviation- and dialogue-aware regex sentence splitter by default (keeps "Mr. Bennet" and '"Is it let?" she asked.' whole); punkt=True opts into NLTK Punkt. - encode/channels.py: the plain-array data contract — normalize_size, categorical_colors (tab20/hsv, stable by first appearance), continuous_colors, and the Channels container. - layout/linear.py: a trivial reading-order grid layout. - render/mpl.py: render_points and render_path build Figure directly (no pyplot, never show()); equal aspect, framed, headless-safe. - Public API re-exported to the top level; submodule __init__ re-exports. - Tests: unit + Hypothesis property tests per module, plus an end-to-end spine test; examples/spine_demo.py; docs pages and nav. make ci green (57 tests); mkdocs build --strict green. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 13 ++ docs/api/channels.md | 11 ++ docs/api/layout.md | 6 + docs/api/render.md | 7 + docs/api/segment.md | 11 ++ docs/examples/spine_demo.md | 19 +++ examples/spine_demo.py | 37 +++++ mkdocs.yml | 6 + src/lexograph/__init__.py | 17 +++ src/lexograph/encode/__init__.py | 16 +++ src/lexograph/encode/channels.py | 164 +++++++++++++++++++++++ src/lexograph/layout/__init__.py | 4 + src/lexograph/layout/linear.py | 65 +++++++++ src/lexograph/render/__init__.py | 4 + src/lexograph/render/mpl.py | 215 ++++++++++++++++++++++++++++++ src/lexograph/segment/__init__.py | 4 + src/lexograph/segment/units.py | 188 ++++++++++++++++++++++++++ tests/test_channels.py | 76 +++++++++++ tests/test_linear.py | 42 ++++++ tests/test_render.py | 60 +++++++++ tests/test_segment.py | 67 ++++++++++ tests/test_spine.py | 39 ++++++ 22 files changed, 1071 insertions(+) create mode 100644 docs/api/channels.md create mode 100644 docs/api/layout.md create mode 100644 docs/api/render.md create mode 100644 docs/api/segment.md create mode 100644 docs/examples/spine_demo.md create mode 100644 examples/spine_demo.py create mode 100644 src/lexograph/encode/channels.py create mode 100644 src/lexograph/layout/linear.py create mode 100644 src/lexograph/render/mpl.py create mode 100644 src/lexograph/segment/units.py create mode 100644 tests/test_channels.py create mode 100644 tests/test_linear.py create mode 100644 tests/test_render.py create mode 100644 tests/test_segment.py create mode 100644 tests/test_spine.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0938185..9efe695 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,19 @@ semantic versioning. ### Added +- Phase 1 spine: the four-step pipeline end to end. + - `segment` — split text into characters, tokens, or sentences. Sentence + splitting defaults to a self-contained, abbreviation-aware offline splitter + (keeps `Mr. Bennet` and `"Is it let?" she asked.` whole); `punkt=True` opts + into NLTK Punkt. + - `encode.channels` — the plain-array data contract: `normalize_size`, + `categorical_colors`, `continuous_colors`, and the `Channels` container. + - `layout.linear_layout` — a trivial reading-order grid layout. + - `render` — `render_points` and `render_path` return a matplotlib `Figure` + (no `pyplot`, never `show()`), rendering inline in Jupyter and saving with + `fig.savefig(...)`. + - `examples/spine_demo.py` renders the bundled chapter as a field of sentence + tiles. - Phase 0 scaffold: package layout mirroring the corpus-lx family (keyflux), uv build backend, ruff + ty + pytest (`--doctest-modules`) config, CI and publish workflows, mkdocs documentation skeleton, and the bundled *Pride and Prejudice* demo text diff --git a/docs/api/channels.md b/docs/api/channels.md new file mode 100644 index 0000000..da2a5f4 --- /dev/null +++ b/docs/api/channels.md @@ -0,0 +1,11 @@ +# Encode channels + +Map plain per-unit arrays onto visual channels — the package's data contract. + +::: lexograph.encode.channels.normalize_size + +::: lexograph.encode.channels.categorical_colors + +::: lexograph.encode.channels.continuous_colors + +::: lexograph.encode.channels.Channels diff --git a/docs/api/layout.md b/docs/api/layout.md new file mode 100644 index 0000000..ca73232 --- /dev/null +++ b/docs/api/layout.md @@ -0,0 +1,6 @@ +# Layout + +Place ordered units in space. The trivial reading-order layout is below; the +walk, spiral, and grid layouts land in later phases. + +::: lexograph.layout.linear.linear_layout diff --git a/docs/api/render.md b/docs/api/render.md new file mode 100644 index 0000000..26c7d07 --- /dev/null +++ b/docs/api/render.md @@ -0,0 +1,7 @@ +# Render + +Draw laid-out, encoded units as a matplotlib `Figure`. + +::: lexograph.render.mpl.render_points + +::: lexograph.render.mpl.render_path diff --git a/docs/api/segment.md b/docs/api/segment.md new file mode 100644 index 0000000..41d6671 --- /dev/null +++ b/docs/api/segment.md @@ -0,0 +1,11 @@ +# Segment + +Turn raw text into ordered units: characters, tokens, or sentences. + +::: lexograph.segment.units.segment + +::: lexograph.segment.units.sentences + +::: lexograph.segment.units.tokens + +::: lexograph.segment.units.characters diff --git a/docs/examples/spine_demo.md b/docs/examples/spine_demo.md new file mode 100644 index 0000000..f009787 --- /dev/null +++ b/docs/examples/spine_demo.md @@ -0,0 +1,19 @@ +# Spine demo + +The four steps of the spine — **segment → layout → encode → render** — composed +end to end on the bundled *Pride and Prejudice* text. The chapter's sentences are +laid out as a field of tiles, each tile sized by sentence length and coloured by +its row in the chapter. + +```python +--8<-- "examples/spine_demo.py" +``` + +Run it with: + +```bash +uv run python examples/spine_demo.py +``` + +It writes `spine_demo.png` to the current directory. The function returns a +matplotlib `Figure`, so in a notebook you can display it inline instead of saving. diff --git a/examples/spine_demo.py b/examples/spine_demo.py new file mode 100644 index 0000000..5958f8e --- /dev/null +++ b/examples/spine_demo.py @@ -0,0 +1,37 @@ +"""Spine demo: segment -> layout -> encode -> render on the bundled text. + +Run with: uv run python examples/spine_demo.py +Writes spine_demo.png to the current directory. +""" + +from lexograph import ( + Channels, + categorical_colors, + linear_layout, + normalize_size, + render_points, + segment, +) +from lexograph.datasets import load_demo_text + + +def main() -> None: + """Lay Pride and Prejudice's first chapter out as a field of sentence tiles.""" + text = load_demo_text() + units = segment(text, unit="sentences") + print(f"Segmented {len(units)} sentences.") + + coords = linear_layout(len(units), columns=8) + channels = Channels( + # Size each tile by sentence length; colour by position in the chapter. + sizes=normalize_size([len(u) for u in units], lo=6.0, hi=26.0, power=1.5), + colors=categorical_colors([i // 8 for i in range(len(units))]), + ) + + fig = render_points(coords, channels=channels, figsize=(9.0, 9.0)) + fig.savefig("spine_demo.png", dpi=120) + print("Saved spine_demo.png") + + +if __name__ == "__main__": + main() diff --git a/mkdocs.yml b/mkdocs.yml index 66b8ba1..3a7c14a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -48,6 +48,12 @@ nav: - Getting Started: - index.md - quickstart.md + - Examples: + - examples/spine_demo.md - Troubleshooting: troubleshooting.md - API Reference: + - api/segment.md + - api/layout.md + - api/channels.md + - api/render.md - api/datasets.md diff --git a/src/lexograph/__init__.py b/src/lexograph/__init__.py index 52f9a31..1b526e9 100644 --- a/src/lexograph/__init__.py +++ b/src/lexograph/__init__.py @@ -11,8 +11,25 @@ __version__ = "0.1.0" from lexograph.datasets import load_demo_text +from lexograph.encode import ( + Channels, + categorical_colors, + continuous_colors, + normalize_size, +) +from lexograph.layout import linear_layout +from lexograph.render import render_path, render_points +from lexograph.segment import segment __all__ = [ + "segment", + "linear_layout", + "normalize_size", + "categorical_colors", + "continuous_colors", + "Channels", + "render_points", + "render_path", "load_demo_text", "__version__", ] diff --git a/src/lexograph/encode/__init__.py b/src/lexograph/encode/__init__.py index d24946f..89e9b3c 100644 --- a/src/lexograph/encode/__init__.py +++ b/src/lexograph/encode/__init__.py @@ -1 +1,17 @@ """Encode: map per-unit attributes onto visual channels (size, colour, glyph).""" + +from lexograph.encode.channels import ( + RGBA, + Channels, + categorical_colors, + continuous_colors, + normalize_size, +) + +__all__ = [ + "RGBA", + "Channels", + "normalize_size", + "categorical_colors", + "continuous_colors", +] diff --git a/src/lexograph/encode/channels.py b/src/lexograph/encode/channels.py new file mode 100644 index 0000000..b0d160c --- /dev/null +++ b/src/lexograph/encode/channels.py @@ -0,0 +1,164 @@ +"""Map per-unit attributes onto visual channels — the package's data seam. + +Every function here accepts a **plain per-unit array** and returns matplotlib-ready +values. ``size`` comes from a scalar array; ``colour`` comes from either an array +of category labels (mapped to a qualitative palette) or an array of numeric values +(mapped through a continuous colormap). Nothing here knows where the numbers came +from, which is exactly what lets a caller drive a figure from ``length``, +``frequency``, a graph centrality, or their own column. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass +from typing import TYPE_CHECKING + +import matplotlib as mpl +import numpy as np +from matplotlib.colors import Normalize + +from lexograph._types import FloatArray + +if TYPE_CHECKING: + from collections.abc import Iterable + +__all__ = [ + "RGBA", + "Channels", + "normalize_size", + "categorical_colors", + "continuous_colors", +] + +RGBA = tuple[float, float, float, float] +"""A matplotlib ``(red, green, blue, alpha)`` colour, each component in ``[0, 1]``.""" + + +def normalize_size( + values: Iterable[float], + *, + lo: float = 6.0, + hi: float = 24.0, + power: float = 1.0, +) -> FloatArray: + """Scale a per-unit scalar array into a size range. + + The values are min-max normalised to ``[0, 1]``, raised to ``power`` (a + ``power`` above 1 compresses the low end so only the largest units stand + out, mirroring the PageRank-driven sizing in the Wittgenstein piece), then + mapped onto ``[lo, hi]``. + + Args: + values: One scalar per unit (e.g. sentence length or a centrality). + lo: The smallest output size. + hi: The largest output size. + power: Exponent applied to the normalised values before scaling. + + Returns: + A float array of sizes, one per input value. A constant or empty input + maps every unit to ``lo``. + + Raises: + ValueError: If ``hi < lo`` or ``power`` is not positive. + + Examples: + >>> normalize_size([0, 5, 10], lo=1.0, hi=3.0).tolist() + [1.0, 2.0, 3.0] + """ + if hi < lo: + msg = f"hi ({hi}) must be >= lo ({lo})" + raise ValueError(msg) + if power <= 0: + msg = f"power must be positive, got {power}" + raise ValueError(msg) + array = np.asarray(list(values), dtype=float) + if array.size == 0: + return array + vmin, vmax = float(array.min()), float(array.max()) + if vmax == vmin: + norm = np.zeros_like(array) + else: + norm = (array - vmin) / (vmax - vmin) + return lo + (hi - lo) * norm**power + + +def categorical_colors( + labels: Sequence[object], *, cmap: str | None = None +) -> list[RGBA]: + """Map category labels to a stable qualitative palette. + + Labels are coloured in order of first appearance, so the same label always + gets the same colour and adjacent distinct labels are visually separated. + The default palette follows the Wittgenstein piece: ``tab20`` for up to 20 + categories, evenly spaced ``hsv`` beyond that. + + Args: + labels: One hashable label per unit (e.g. a cluster or community id). + cmap: Override colormap name. ``None`` selects the default by count. + + Returns: + One RGBA colour per input label. + + Examples: + >>> colors = categorical_colors([0, 1, 0, 2]) + >>> len(colors) + 4 + >>> colors[0] == colors[2] + True + """ + order: dict[object, int] = {} + for label in labels: + if label not in order: + order[label] = len(order) + count = len(order) + if cmap is None: + cmap = "tab20" if count <= 20 else "hsv" + colormap = mpl.colormaps[cmap] + if count <= 20 and cmap == "tab20": + palette = [colormap(i) for i in range(count)] + else: + palette = [colormap(i / max(count, 1)) for i in range(count)] + return [palette[order[label]] for label in labels] + + +def continuous_colors(values: Iterable[float], *, cmap: str = "viridis") -> list[RGBA]: + """Map a per-unit numeric array through a continuous colormap. + + Args: + values: One scalar per unit. + cmap: A continuous matplotlib colormap name. + + Returns: + One RGBA colour per value. A constant or empty input maps every unit to + the colormap's midpoint. + + Examples: + >>> colors = continuous_colors([0.0, 0.5, 1.0]) + >>> len(colors) + 3 + """ + array = np.asarray(list(values), dtype=float) + colormap = mpl.colormaps[cmap] + if array.size == 0: + return [] + vmin, vmax = float(array.min()), float(array.max()) + if vmax == vmin: + return [colormap(0.5) for _ in array] + norm = Normalize(vmin=vmin, vmax=vmax) + return [colormap(float(norm(v))) for v in array] + + +@dataclass(frozen=True, slots=True) +class Channels: + """Resolved visual channels, one entry per unit. + + Attributes: + sizes: Per-unit size in points, or ``None`` for a uniform size. + colors: Per-unit RGBA colour, or ``None`` for a single default colour. + glyphs: Per-unit text to draw, or ``None`` to draw markers/segments. + """ + + sizes: FloatArray | None = None + colors: list[RGBA] | None = None + glyphs: list[str] | None = None diff --git a/src/lexograph/layout/__init__.py b/src/lexograph/layout/__init__.py index 232fadc..57c0add 100644 --- a/src/lexograph/layout/__init__.py +++ b/src/lexograph/layout/__init__.py @@ -1 +1,5 @@ """Layout: place ordered units in 2-D or 3-D space (walks, spirals, grids).""" + +from lexograph.layout.linear import linear_layout + +__all__ = ["linear_layout"] diff --git a/src/lexograph/layout/linear.py b/src/lexograph/layout/linear.py new file mode 100644 index 0000000..350e539 --- /dev/null +++ b/src/lexograph/layout/linear.py @@ -0,0 +1,65 @@ +"""A trivial reading-order layout: units flow left-to-right, top-to-bottom. + +This is the simplest point on the layout step of the spine — it just places the +``i``-th unit on a grid in reading order. It exists so the spine +(segment → layout → encode → render) is exercisable end to end before the +richer walk, spiral, and grid layouts arrive, and it is genuinely useful on its +own for "text as a field of tiles" figures. +""" + +from __future__ import annotations + +import numpy as np + +from lexograph._types import Coords + +__all__ = ["linear_layout"] + + +def linear_layout( + n: int, + *, + columns: int | None = None, + col_width: float = 1.0, + line_height: float = 1.0, +) -> Coords: + """Lay ``n`` units out on a reading-order grid. + + Unit ``i`` is placed at column ``i % columns`` and row ``i // columns``, + with rows running downward (decreasing ``y``) so the first unit is top-left. + + Args: + n: The number of units to place. + columns: Units per row. ``None`` puts them all on a single row. + col_width: Horizontal spacing between columns. + line_height: Vertical spacing between rows. + + Returns: + An ``(n, 2)`` float array of ``(x, y)`` coordinates. + + Raises: + ValueError: If ``n`` is negative or ``columns`` is not positive. + + Contract: + - Returns exactly ``n`` rows. + - Coordinates are deterministic in ``n`` and the spacing parameters. + + Examples: + >>> linear_layout(3).tolist() + [[0.0, 0.0], [1.0, 0.0], [2.0, 0.0]] + >>> linear_layout(3, columns=2).tolist() + [[0.0, 0.0], [1.0, 0.0], [0.0, -1.0]] + """ + if n < 0: + msg = f"n must be non-negative, got {n}" + raise ValueError(msg) + if columns is not None and columns <= 0: + msg = f"columns must be positive, got {columns}" + raise ValueError(msg) + if n == 0: + return np.zeros((0, 2), dtype=float) + width = n if columns is None else columns + index = np.arange(n) + x = (index % width) * col_width + y = -(index // width) * line_height + return np.column_stack([x, y]).astype(float) diff --git a/src/lexograph/render/__init__.py b/src/lexograph/render/__init__.py index c1ffeda..b42d023 100644 --- a/src/lexograph/render/__init__.py +++ b/src/lexograph/render/__init__.py @@ -1 +1,5 @@ """Render: draw a laid-out, encoded text as a matplotlib Figure.""" + +from lexograph.render.mpl import render_path, render_points + +__all__ = ["render_points", "render_path"] diff --git a/src/lexograph/render/mpl.py b/src/lexograph/render/mpl.py new file mode 100644 index 0000000..3d2d206 --- /dev/null +++ b/src/lexograph/render/mpl.py @@ -0,0 +1,215 @@ +"""Render laid-out, encoded units as a matplotlib ``Figure``. + +This is the final step of the spine. It takes coordinates from a layout and +optional visual channels (sizes, colours, glyphs) and draws them. Following the +family convention, every entry point constructs a :class:`matplotlib.figure.Figure` +directly (no global ``pyplot`` state) and never calls ``show()``, so figures +render inline in Jupyter and save cleanly with ``fig.savefig(...)``. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import numpy as np +from matplotlib.collections import LineCollection +from matplotlib.figure import Figure + +if TYPE_CHECKING: + from matplotlib.axes import Axes + + from lexograph._types import Coords + from lexograph.encode.channels import RGBA, Channels + +__all__ = ["render_points", "render_path"] + +_DEFAULT_POINT_COLOR = "#1f77b4" +_DEFAULT_LINE_COLOR = "#333333" + + +def _coords_2d(coords: Coords) -> np.ndarray: + """Validate and return a 2-D ``(N, 2)`` float coordinate array.""" + array = np.asarray(coords, dtype=float) + if array.ndim != 2 or array.shape[1] != 2: + msg = f"coords must have shape (N, 2), got {array.shape}" + raise ValueError(msg) + return array + + +def _frame(ax: Axes, array: np.ndarray, *, margin: float = 0.05) -> None: + """Equalise the aspect ratio, hide the axes, and fit the data with a margin.""" + ax.set_aspect("equal") + ax.axis("off") + if array.shape[0] == 0: + return + xmin, ymin = array.min(axis=0) + xmax, ymax = array.max(axis=0) + span_x = xmax - xmin or 1.0 + span_y = ymax - ymin or 1.0 + ax.set_xlim(xmin - margin * span_x, xmax + margin * span_x) + ax.set_ylim(ymin - margin * span_y, ymax + margin * span_y) + + +def render_points( + coords: Coords, + *, + channels: Channels | None = None, + sizes: np.ndarray | None = None, + colors: list[RGBA] | None = None, + glyphs: list[str] | None = None, + background: str = "white", + figsize: tuple[float, float] = (8.0, 8.0), +) -> Figure: + """Draw one mark per unit at its layout coordinate. + + If ``glyphs`` are given, each unit is drawn as text; otherwise each unit is a + scatter marker. Channel arrays may be passed individually or bundled in a + :class:`~lexograph.encode.channels.Channels`; individual arguments win. + + Args: + coords: An ``(N, 2)`` array of unit positions. + channels: Resolved channels to use as defaults for ``sizes``/``colors``/ + ``glyphs``. + sizes: Per-unit size in points (marker diameter, or glyph font size). + colors: Per-unit RGBA colour. + glyphs: Per-unit text; when given, units are drawn as text not markers. + background: Figure and axes background colour. + figsize: Figure size in inches. + + Returns: + A :class:`matplotlib.figure.Figure` with a single axes. Never calls + ``show()``. + + Raises: + ValueError: If ``coords`` is not ``(N, 2)``, or a channel length does + not match the number of units. + + Contract: + - Returns a Figure with exactly one axes. + - Inputs are never mutated. + + Examples: + >>> import numpy as np + >>> fig = render_points(np.array([[0.0, 0.0], [1.0, 1.0]])) + >>> type(fig).__name__ + 'Figure' + >>> len(fig.axes) + 1 + """ + array = _coords_2d(coords) + n = array.shape[0] + if channels is not None: + sizes = sizes if sizes is not None else channels.sizes + colors = colors if colors is not None else channels.colors + glyphs = glyphs if glyphs is not None else channels.glyphs + _check_lengths(n, sizes=sizes, colors=colors, glyphs=glyphs) + + fig = Figure(figsize=figsize, facecolor=background) + ax = fig.subplots() + ax.set_facecolor(background) + + if glyphs is not None: + for i in range(n): + ax.text( + array[i, 0], + array[i, 1], + glyphs[i], + fontsize=(float(sizes[i]) if sizes is not None else 12.0), + color=(colors[i] if colors is not None else _DEFAULT_POINT_COLOR), + ha="center", + va="center", + ) + elif n: + marker_area = ( + (np.asarray(sizes, dtype=float) ** 2) if sizes is not None else 36.0 + ) + ax.scatter( + array[:, 0], + array[:, 1], + s=marker_area, + c=(colors if colors is not None else _DEFAULT_POINT_COLOR), + ) + + _frame(ax, array) + fig.tight_layout() + return fig + + +def render_path( + coords: Coords, + *, + colors: list[RGBA] | None = None, + color: str = _DEFAULT_LINE_COLOR, + linewidth: float = 1.5, + background: str = "white", + figsize: tuple[float, float] = (8.0, 8.0), +) -> Figure: + """Draw the units as a connected path through their layout coordinates. + + This is the renderer behind the text walk: consecutive units are joined by + line segments. When ``colors`` is given (one colour per segment, i.e. one + per unit after the first), the path is drawn as a multi-coloured + :class:`~matplotlib.collections.LineCollection`. + + Args: + coords: An ``(N, 2)`` array of vertices in path order. + colors: Per-segment RGBA colours (length ``N - 1``). ``None`` draws a + single-colour path. + color: The path colour when ``colors`` is ``None``. + linewidth: Line width in points. + background: Figure and axes background colour. + figsize: Figure size in inches. + + Returns: + A :class:`matplotlib.figure.Figure` with a single axes. Never calls + ``show()``. + + Raises: + ValueError: If ``coords`` is not ``(N, 2)``, or ``colors`` has the wrong + length. + + Contract: + - Returns a Figure with exactly one axes. + - Inputs are never mutated. + + Examples: + >>> import numpy as np + >>> fig = render_path(np.array([[0.0, 0.0], [1.0, 0.0], [1.0, 1.0]])) + >>> type(fig).__name__ + 'Figure' + """ + array = _coords_2d(coords) + n = array.shape[0] + fig = Figure(figsize=figsize, facecolor=background) + ax = fig.subplots() + ax.set_facecolor(background) + + if n >= 2: + if colors is not None: + if len(colors) != n - 1: + msg = f"colors must have length N-1 ({n - 1}), got {len(colors)}" + raise ValueError(msg) + segments = list(np.stack([array[:-1], array[1:]], axis=1)) + ax.add_collection( + LineCollection(segments, colors=colors, linewidths=linewidth) + ) + else: + ax.plot(array[:, 0], array[:, 1], color=color, linewidth=linewidth) + + _frame(ax, array) + fig.tight_layout() + return fig + + +def _check_lengths( + n: int, + *, + sizes: np.ndarray | None, + colors: list[RGBA] | None, + glyphs: list[str] | None, +) -> None: + """Raise if any provided channel does not have one entry per unit.""" + for name, channel in (("sizes", sizes), ("colors", colors), ("glyphs", glyphs)): + if channel is not None and len(channel) != n: + msg = f"{name} must have one entry per unit ({n}), got {len(channel)}" + raise ValueError(msg) diff --git a/src/lexograph/segment/__init__.py b/src/lexograph/segment/__init__.py index aebf69c..0bcaa1a 100644 --- a/src/lexograph/segment/__init__.py +++ b/src/lexograph/segment/__init__.py @@ -1 +1,5 @@ """Segmentation: turn raw text into ordered units (chars, tokens, sentences).""" + +from lexograph.segment.units import characters, segment, sentences, tokens + +__all__ = ["segment", "characters", "tokens", "sentences"] diff --git a/src/lexograph/segment/units.py b/src/lexograph/segment/units.py new file mode 100644 index 0000000..b4120d3 --- /dev/null +++ b/src/lexograph/segment/units.py @@ -0,0 +1,188 @@ +"""Segment raw text into an ordered list of units. + +A *unit* is the atom lexograph lays out and draws: a single character, a token, +or a sentence. Segmentation is the first step of the spine; everything +downstream (layout, encode, render) consumes the ordered list this module +returns. + +Sentence splitting defaults to a self-contained, offline regex splitter that +guards common abbreviations (so ``Mr. Bennet`` is not cut in two). Pass +``punkt=True`` to use NLTK's Punkt model instead — higher quality, at the cost +of a one-time model download. +""" + +from __future__ import annotations + +import re + +from lexograph._types import Unit, UnitKind + +__all__ = ["segment", "characters", "tokens", "sentences"] + +# A token: a run of word characters, allowing internal apostrophes and hyphens +# (so ``good-humoured`` and ``don't`` stay whole). +_TOKEN_RE = re.compile(r"\w+(?:['’-]\w+)*") + +# A candidate sentence terminator: end punctuation, optional closing quotes or +# brackets, then whitespace. The whitespace requirement avoids splitting inside +# decimals and ellipses mid-token. +_SENTENCE_END_RE = re.compile(r'[.!?]+["\'”’)\]]*\s+') + +# The word immediately before a candidate terminator. +_TRAILING_WORD_RE = re.compile(r"(\w+)\W*$") + +# The first letter at or after a position (Unicode-aware, excludes digits). +_NEXT_LETTER_RE = re.compile(r"[^\W\d_]") + +# Abbreviations whose trailing period is not a sentence boundary. +_ABBREVIATIONS = frozenset( + { + "mr", "mrs", "ms", "dr", "prof", "sr", "jr", "st", "mt", "rev", "hon", + "gen", "col", "capt", "sgt", "lt", "messrs", + "vs", "etc", "al", "no", "vol", "fig", "pp", "inc", "ltd", "co", + "jan", "feb", "mar", "apr", "jun", "jul", "aug", "sep", "sept", + "oct", "nov", "dec", + } +) # fmt: skip + + +def characters(text: str) -> list[Unit]: + """Return every character of ``text`` in order. + + Args: + text: The source text. + + Returns: + One single-character string per character, including whitespace and + punctuation (the punctuation-spiral preset filters these itself). + + Examples: + >>> characters("Hi!") + ['H', 'i', '!'] + """ + return list(text) + + +def tokens(text: str) -> list[Unit]: + """Return the word tokens of ``text`` in order. + + A token is a run of word characters with optional internal apostrophes or + hyphens; punctuation and whitespace are dropped. + + Args: + text: The source text. + + Returns: + The ordered list of word tokens. + + Examples: + >>> tokens("It's a good-humoured day.") + ["It's", 'a', 'good-humoured', 'day'] + """ + return _TOKEN_RE.findall(text) + + +def sentences(text: str, *, punkt: bool = False) -> list[Unit]: + """Split ``text`` into sentences, in order. + + Args: + text: The source text. + punkt: If ``True``, use NLTK's Punkt sentence tokenizer (downloading the + model on first use). If ``False`` (the default), use the bundled + offline regex splitter, which guards common abbreviations. + + Returns: + The ordered list of sentences, each stripped of surrounding whitespace. + Empty or whitespace-only sentences are dropped. + + Examples: + >>> sentences("Mr. Bennet replied that he had not. He said no more.") + ['Mr. Bennet replied that he had not.', 'He said no more.'] + """ + if punkt: + return _punkt_sentences(text) + return _regex_sentences(text) + + +def _regex_sentences(text: str) -> list[Unit]: + """Split into sentences with the offline, abbreviation-aware regex splitter.""" + result: list[Unit] = [] + start = 0 + for match in _SENTENCE_END_RE.finditer(text): + prefix = text[start : match.start()] + trailing = _TRAILING_WORD_RE.search(prefix) + if trailing is not None: + word = trailing.group(1) + if word.lower() in _ABBREVIATIONS: + continue + # A single capital letter is an initial (e.g. "A. Bennet"), not an end. + if len(word) == 1 and word.isalpha() and word.isupper(): + continue + # If the next sentence would start with a lowercase letter, this + # terminator sits inside a larger sentence — e.g. dialogue followed by + # an attribution: '"Is it let?" she asked.' — so it is not a boundary. + following = _NEXT_LETTER_RE.search(text, match.end()) + if following is not None and following.group(0).islower(): + continue + sentence = text[start : match.end()].strip() + if sentence: + result.append(sentence) + start = match.end() + tail = text[start:].strip() + if tail: + result.append(tail) + return result + + +def _punkt_sentences(text: str) -> list[Unit]: + """Split into sentences with NLTK Punkt, fetching the model if needed.""" + import nltk + + try: + nltk.data.find("tokenizers/punkt_tab") + except LookupError: + nltk.download("punkt_tab", quiet=True) + from nltk.tokenize import sent_tokenize + + return [s.strip() for s in sent_tokenize(text) if s.strip()] + + +def segment( + text: str, unit: UnitKind = "sentences", *, punkt: bool = False +) -> list[Unit]: + """Segment ``text`` into ordered units of the requested kind. + + This is the single entry point to the segmentation step of the spine. + + Args: + text: The source text. + unit: ``"chars"``, ``"tokens"``, or ``"sentences"``. + punkt: Use NLTK Punkt for sentence splitting (ignored for other kinds). + + Returns: + The ordered list of units. + + Raises: + ValueError: If ``unit`` is not one of the three recognised kinds. + + Contract: + - The returned list preserves source order. + - ``"chars"`` keeps every character; ``"tokens"`` and ``"sentences"`` + drop pure-whitespace units. + + Examples: + >>> segment("One. Two.", unit="sentences") + ['One.', 'Two.'] + >>> segment("One two", unit="tokens") + ['One', 'two'] + >>> len(segment("abc", unit="chars")) + 3 + """ + if unit == "chars": + return characters(text) + if unit == "tokens": + return tokens(text) + if unit == "sentences": + return sentences(text, punkt=punkt) + msg = f"unit must be 'chars', 'tokens', or 'sentences', got {unit!r}" + raise ValueError(msg) diff --git a/tests/test_channels.py b/tests/test_channels.py new file mode 100644 index 0000000..9bcff03 --- /dev/null +++ b/tests/test_channels.py @@ -0,0 +1,76 @@ +import numpy as np +import pytest +from hypothesis import given +from hypothesis import strategies as st + +from lexograph.encode import ( + categorical_colors, + continuous_colors, + normalize_size, +) + + +class TestNormalizeSize: + """Sizes are min-max normalised, shaped, and scaled into [lo, hi].""" + + def test_endpoints_map_to_range(self) -> None: + sizes = normalize_size([0, 10], lo=2.0, hi=8.0) + assert sizes.tolist() == [2.0, 8.0] + + def test_constant_input_maps_to_lo(self) -> None: + sizes = normalize_size([5, 5, 5], lo=3.0, hi=9.0) + assert sizes.tolist() == [3.0, 3.0, 3.0] + + def test_empty(self) -> None: + assert normalize_size([]).tolist() == [] + + def test_bad_range_raises(self) -> None: + with pytest.raises(ValueError, match="must be >= lo"): + normalize_size([1, 2], lo=5.0, hi=1.0) + + def test_bad_power_raises(self) -> None: + with pytest.raises(ValueError, match="power must be positive"): + normalize_size([1, 2], power=0.0) + + @given( + st.lists(st.floats(min_value=-1e6, max_value=1e6), min_size=1, max_size=50), + st.floats(min_value=0.0, max_value=10.0), + st.floats(min_value=0.0, max_value=10.0), + ) + def test_stays_within_bounds(self, values: list[float], a: float, b: float) -> None: + lo, hi = min(a, b), max(a, b) + sizes = normalize_size(values, lo=lo, hi=hi) + assert np.all(sizes >= lo - 1e-9) + assert np.all(sizes <= hi + 1e-9) + + +class TestCategoricalColors: + """Labels colour by first appearance: same label, same colour.""" + + def test_same_label_same_color(self) -> None: + colors = categorical_colors([0, 1, 0, 2]) + assert colors[0] == colors[2] + assert colors[0] != colors[1] + + def test_one_per_label(self) -> None: + labels = ["a", "b", "c", "a"] + assert len(categorical_colors(labels)) == len(labels) + + def test_many_categories_use_hsv(self) -> None: + colors = categorical_colors(list(range(30))) + assert len(colors) == 30 + assert len(set(colors)) == 30 + + +class TestContinuousColors: + """Numeric values map through a continuous colormap.""" + + def test_one_per_value(self) -> None: + assert len(continuous_colors([0.0, 0.5, 1.0])) == 3 + + def test_constant_input_uses_midpoint(self) -> None: + colors = continuous_colors([2.0, 2.0]) + assert colors[0] == colors[1] + + def test_empty(self) -> None: + assert continuous_colors([]) == [] diff --git a/tests/test_linear.py b/tests/test_linear.py new file mode 100644 index 0000000..8cfc7d6 --- /dev/null +++ b/tests/test_linear.py @@ -0,0 +1,42 @@ +import numpy as np +import pytest +from hypothesis import given +from hypothesis import strategies as st + +from lexograph.layout import linear_layout + + +class TestLinearLayout: + """Units flow left-to-right, wrapping into downward rows.""" + + def test_single_row(self) -> None: + assert linear_layout(3).tolist() == [[0.0, 0.0], [1.0, 0.0], [2.0, 0.0]] + + def test_wraps_into_rows(self) -> None: + coords = linear_layout(4, columns=2) + assert coords.tolist() == [ + [0.0, 0.0], + [1.0, 0.0], + [0.0, -1.0], + [1.0, -1.0], + ] + + def test_empty(self) -> None: + assert linear_layout(0).shape == (0, 2) + + def test_negative_raises(self) -> None: + with pytest.raises(ValueError, match="non-negative"): + linear_layout(-1) + + def test_bad_columns_raises(self) -> None: + with pytest.raises(ValueError, match="columns must be positive"): + linear_layout(3, columns=0) + + @given( + st.integers(min_value=0, max_value=500), st.integers(min_value=1, max_value=40) + ) + def test_shape_matches_n(self, n: int, columns: int) -> None: + coords = linear_layout(n, columns=columns) + assert coords.shape == (n, 2) + if n: + assert np.all(coords[:, 1] <= 0) diff --git a/tests/test_render.py b/tests/test_render.py new file mode 100644 index 0000000..912ff87 --- /dev/null +++ b/tests/test_render.py @@ -0,0 +1,60 @@ +import numpy as np +import pytest +from matplotlib.figure import Figure + +from lexograph.encode import Channels, categorical_colors, normalize_size +from lexograph.render import render_path, render_points + + +class TestRenderPoints: + """render_points draws one mark or glyph per unit and returns a Figure.""" + + def test_returns_figure_with_one_axes(self) -> None: + fig = render_points(np.array([[0.0, 0.0], [1.0, 1.0]])) + assert isinstance(fig, Figure) + assert len(fig.axes) == 1 + + def test_accepts_channels(self) -> None: + coords = np.array([[0.0, 0.0], [1.0, 0.0], [2.0, 0.0]]) + channels = Channels( + sizes=normalize_size([1, 2, 3]), + colors=categorical_colors([0, 1, 0]), + ) + fig = render_points(coords, channels=channels) + assert isinstance(fig, Figure) + + def test_glyphs_render_as_text(self) -> None: + coords = np.array([[0.0, 0.0], [1.0, 0.0]]) + fig = render_points(coords, glyphs=["a", "b"]) + assert len(fig.axes[0].texts) == 2 + + def test_empty(self) -> None: + fig = render_points(np.zeros((0, 2))) + assert isinstance(fig, Figure) + + def test_bad_shape_raises(self) -> None: + with pytest.raises(ValueError, match=r"shape \(N, 2\)"): + render_points(np.zeros((3, 3))) + + def test_channel_length_mismatch_raises(self) -> None: + with pytest.raises(ValueError, match="one entry per unit"): + render_points(np.zeros((2, 2)), glyphs=["only-one"]) + + +class TestRenderPath: + """render_path joins the units into a connected (optionally coloured) path.""" + + def test_returns_figure(self) -> None: + fig = render_path(np.array([[0.0, 0.0], [1.0, 0.0], [1.0, 1.0]])) + assert isinstance(fig, Figure) + + def test_per_segment_colors(self) -> None: + coords = np.array([[0.0, 0.0], [1.0, 0.0], [1.0, 1.0]]) + colors = categorical_colors([0, 1]) # N-1 == 2 segments + fig = render_path(coords, colors=colors) + assert len(fig.axes[0].collections) == 1 + + def test_wrong_color_count_raises(self) -> None: + coords = np.array([[0.0, 0.0], [1.0, 0.0], [1.0, 1.0]]) + with pytest.raises(ValueError, match="length N-1"): + render_path(coords, colors=categorical_colors([0, 1, 2])) diff --git a/tests/test_segment.py b/tests/test_segment.py new file mode 100644 index 0000000..8acaf35 --- /dev/null +++ b/tests/test_segment.py @@ -0,0 +1,67 @@ +import pytest +from hypothesis import given +from hypothesis import strategies as st + +from lexograph.segment import characters, segment, sentences, tokens + + +class TestSegment: + """The unified segment() entry point dispatches on unit kind.""" + + def test_sentences_default(self) -> None: + assert segment("One. Two.") == ["One.", "Two."] + + def test_tokens(self) -> None: + assert segment("a b c", unit="tokens") == ["a", "b", "c"] + + def test_chars_keeps_everything(self) -> None: + assert segment("a b", unit="chars") == ["a", " ", "b"] + + def test_unknown_unit_raises(self) -> None: + with pytest.raises(ValueError, match="unit must be"): + segment("x", unit="paragraphs") # type: ignore[arg-type] + + +class TestSentences: + """The offline splitter ends on .!? but guards abbreviations and initials.""" + + def test_abbreviation_not_split(self) -> None: + assert sentences("Mr. Bennet went home. He slept.") == [ + "Mr. Bennet went home.", + "He slept.", + ] + + def test_question_and_quote(self) -> None: + assert sentences('"Is it let?" she asked. He nodded.') == [ + '"Is it let?" she asked.', + "He nodded.", + ] + + def test_initials_not_split(self) -> None: + assert sentences("A. B. Smith arrived. They waited.") == [ + "A. B. Smith arrived.", + "They waited.", + ] + + def test_no_terminal_punctuation(self) -> None: + assert sentences("just a fragment") == ["just a fragment"] + + def test_empty(self) -> None: + assert sentences("") == [] + + @given(st.text()) + def test_sentences_are_stripped_and_nonempty(self, text: str) -> None: + for sentence in sentences(text): + assert sentence == sentence.strip() + assert sentence != "" + + +class TestTokensAndChars: + """Tokens drop punctuation; chars keep every character in order.""" + + def test_internal_apostrophe_and_hyphen(self) -> None: + assert tokens("It's good-humoured.") == ["It's", "good-humoured"] + + @given(st.text()) + def test_chars_roundtrip(self, text: str) -> None: + assert "".join(characters(text)) == text diff --git a/tests/test_spine.py b/tests/test_spine.py new file mode 100644 index 0000000..246410b --- /dev/null +++ b/tests/test_spine.py @@ -0,0 +1,39 @@ +"""End-to-end: segment -> layout -> encode -> render on the bundled text.""" + +import numpy as np +from matplotlib.figure import Figure + +from lexograph import ( + Channels, + categorical_colors, + linear_layout, + normalize_size, + render_points, + segment, +) + + +class TestSpineEndToEnd: + """The four steps compose into a Figure from real text.""" + + def test_demo_text_to_figure(self, demo_text: str) -> None: + units = segment(demo_text, unit="sentences") + assert len(units) > 20 + + coords = linear_layout(len(units), columns=8) + channels = Channels( + sizes=normalize_size([len(u) for u in units], lo=4.0, hi=20.0), + colors=categorical_colors([i % 4 for i in range(len(units))]), + ) + fig = render_points(coords, channels=channels) + + assert isinstance(fig, Figure) + assert coords.shape == (len(units), 2) + assert len(channels.sizes) == len(units) + + def test_lengths_stay_aligned(self, demo_text: str) -> None: + units = segment(demo_text, unit="tokens") + sizes = normalize_size([len(u) for u in units]) + coords = linear_layout(len(units)) + assert len(units) == len(sizes) == coords.shape[0] + assert np.all(sizes >= 6.0 - 1e-9)