From a335942ead9c2b332ba30ce7e9f9773f19632088 Mon Sep 17 00:00:00 2001 From: Roberto Arista Date: Fri, 22 May 2026 10:38:11 +0200 Subject: [PATCH 1/2] add module docstrings for test modules --- tests/testAutomaticallyGeneratedCode.py | 2 ++ tests/testExamples.py | 9 +++++++++ tests/testExport.py | 8 ++++++++ tests/testImageObject.py | 8 ++++++-- tests/testMisc.py | 2 ++ tests/testPackage.py | 2 ++ tests/testScripts.py | 11 +++++++++++ tests/testSupport.py | 2 ++ 8 files changed, 42 insertions(+), 2 deletions(-) diff --git a/tests/testAutomaticallyGeneratedCode.py b/tests/testAutomaticallyGeneratedCode.py index f4941e37..9465a383 100644 --- a/tests/testAutomaticallyGeneratedCode.py +++ b/tests/testAutomaticallyGeneratedCode.py @@ -1,3 +1,5 @@ +"""Guards that committed auto-generated source files stay in sync with their generators.""" + import sys import unittest from pathlib import Path diff --git a/tests/testExamples.py b/tests/testExamples.py index b6f99a79..1beda1fb 100644 --- a/tests/testExamples.py +++ b/tests/testExamples.py @@ -1,3 +1,12 @@ +"""Tests for the ``.. downloadcode::`` examples embedded in DrawBot docstrings. + +Walks the public API of ``DrawBotDrawingTool``, ``FormattedString``, +``BezierPath``, and ``ImageObject``, extracts every example marked with a +``.. downloadcode:: name.py`` directive in a docstring, and generates a +``test_`` for each one. + +""" + import os import random import re diff --git a/tests/testExport.py b/tests/testExport.py index 8a85a1ea..5c4d3534 100644 --- a/tests/testExport.py +++ b/tests/testExport.py @@ -1,3 +1,11 @@ +"""Tests for the various ``saveImage`` output formats. + +Renders some drawings and saves them to a range of extensions +(``.png``, ``.jpg``, ``.gif``, ``.tif``, ``.svg``, ``.pdf``, ``.mp4``, +``.icns``, etc.), then asserts on file size, dimensions, frame count, or +content. +""" + import glob import os import random diff --git a/tests/testImageObject.py b/tests/testImageObject.py index 229e30e4..b1505eee 100644 --- a/tests/testImageObject.py +++ b/tests/testImageObject.py @@ -1,8 +1,12 @@ -import unittest +"""Smoke tests for every Core Image filter exposed on ``ImageObject``.""" + import sys -import drawBot +import unittest + from testSupport import DrawBotBaseTest +import drawBot + sourceImagePath = "tests/data/drawBot144.png" sampleImage = drawBot.ImageObject("tests/data/drawBot.png") sampleFormattedString = drawBot.FormattedString("Hello World") diff --git a/tests/testMisc.py b/tests/testMisc.py index 05ac7c47..9031407b 100644 --- a/tests/testMisc.py +++ b/tests/testMisc.py @@ -1,3 +1,5 @@ +"""Plain unit tests for miscellaneous DrawBot APIs. No image rendering involved.""" + import io import os import pathlib diff --git a/tests/testPackage.py b/tests/testPackage.py index beff908b..5bceec6e 100644 --- a/tests/testPackage.py +++ b/tests/testPackage.py @@ -1,3 +1,5 @@ +"""Tests for the ``.drawbot`` package format.""" + import os import sys import unittest diff --git a/tests/testScripts.py b/tests/testScripts.py index 1ccfd5ad..5418fd16 100644 --- a/tests/testScripts.py +++ b/tests/testScripts.py @@ -1,3 +1,14 @@ +"""End-to-end rendering tests driven by scripts in `tests/drawBotScripts/`. + +For every `.py` file in `drawBotScripts/`, a test is generated for each of +`svg`, `png`, and `pdf`: the script is executed, its captured stdout is +compared against `tests/data/expected_.txt` (when present), and the +rendered file is compared against `tests/data/expected_.. + +Fresh renders are written under `tests/tempTestData/` so failures can be +inspected locally. +""" + import glob import os import sys diff --git a/tests/testSupport.py b/tests/testSupport.py index ab64ec6d..202e72d0 100644 --- a/tests/testSupport.py +++ b/tests/testSupport.py @@ -1,3 +1,5 @@ +"""Shared helpers for the test suite. Not a test module itself.""" + import io import os import random From 5b51639f4b6a3ccca30256120e6518bfe0dacffd Mon Sep 17 00:00:00 2001 From: Roberto Arista Date: Fri, 22 May 2026 11:00:35 +0200 Subject: [PATCH 2/2] add a Development section in the README file --- README.md | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/README.md b/README.md index 4a809a5f..eed2ca0b 100644 --- a/README.md +++ b/README.md @@ -94,3 +94,51 @@ __Compile:__ ## [Release protocol](https://github.com/typemytype/drawbot/wiki/DrawBot-release-protocol) + +--- + +## Development + +### Setup + +From the repository root, create a virtual environment, activate it, and install both the package and test dependencies: + + python3 -m venv .venv + source .venv/bin/activate + pip install -r requirements.txt -r test-requirements.txt + +### Running the tests + +From the repository root (with the environment activated): + + python tests/runAllTests.py + +`runAllTests.py` discovers every `test*.py` module under `tests/`. Individual test modules can also be executed directly (e.g. `python tests/testExport.py`). + +The runtime dependencies for the test suite (`Pillow`, `attrs`, `packaging`, `mypy`, `ruff`, `ruff-api`) live in `test-requirements.txt`. + +### Benchmarks and tempTestData + +`tests/data/` is the benchmark folder. Filenames follow a prefix convention: + +- `expected_.` — reference output for `testScripts.py` and parts of `testExport.py`. +- `example_.png` — reference renders for `testExamples.py` (also used by the docs site). +- no prefix — assets consumed by the test scripts themselves (sample images, fonts, etc.). + +Freshly rendered output goes into `tests/tempTestData/` (gitignored). When a comparison fails, that's where you'll find the actual rendered file alongside the expected one. + +### Rendering drift across macOS versions and CI + +Quartz / Core Image rendering is not bit-identical across macOS versions, so the comparison helpers in `testSupport.py` fall back to a fuzzy image diff with a small tolerance whenever raw byte equality fails. Even so, CI on GitHub Actions might run on a different macOS image than your machine, and some tests may pass locally while failing in CI (or vice versa). To find the exact macOS version GitHub Actions is using, open the workflow webpage, expand the **Set up job** step, and read the `Image OS` / `Runner Image` lines in its log. + +To investigate further, use `tests/differenceBuilder.py`. It builds a PDF report visualizing the diff between a folder of freshly rendered images and the benchmarks in `tests/data/`. For each pair it lays out the new render, the expected image tracked in the benchmark folder, a pixel-difference image, and a histogram side by side. Pages where the fuzzy similarity exceeds the tolerance are drawn with a red border so failures are easier to spot. Run it as: + + python tests/differenceBuilder.py + +or invoke it with no arguments to pick the folders via a dialog. The resulting PDF is generated on the fly and not committed. + +When a test passes locally but fails in CI (or vice versa): + +1. Pull the failing artifacts from the Github Actions failed workflow and check the "Drawbot Test Differences" PDF. Keep the "DrawBot Temp Data Results" at hand as well. +2. If the difference between the images is small and clearly attributable to an OS version change (check the overlay and the histogram), the benchmark in `tests/data/` can be updated — but don't blindly copy the CI render over the local one, remember to add the correct prefix as needed. +3. If the difference is large, treat it as a real problem.