From 60450c68cdc64e2dce93722364716cae858927a2 Mon Sep 17 00:00:00 2001 From: Harry Mander Date: Tue, 24 Jun 2025 16:39:36 +1200 Subject: [PATCH 1/6] Widen type requirement for unpack to allow buffer protocol objects --- dataclasses_struct/_typing.py | 6 ++++++ dataclasses_struct/dataclass.py | 10 +++++----- docs/guide.md | 8 +++++--- pyproject.toml | 2 +- uv.lock | 4 ++-- 5 files changed, 19 insertions(+), 11 deletions(-) diff --git a/dataclasses_struct/_typing.py b/dataclasses_struct/_typing.py index 8f07262..28ddd17 100644 --- a/dataclasses_struct/_typing.py +++ b/dataclasses_struct/_typing.py @@ -10,8 +10,14 @@ else: from typing_extensions import Unpack, dataclass_transform +if sys.version_info >= (3, 12): + from collections.abc import Buffer +else: + from typing_extensions import Buffer + __all__ = [ + "Buffer", "TypeGuard", "Unpack", "dataclass_transform", diff --git a/dataclasses_struct/dataclass.py b/dataclasses_struct/dataclass.py index 3456859..988197e 100644 --- a/dataclasses_struct/dataclass.py +++ b/dataclasses_struct/dataclass.py @@ -19,7 +19,7 @@ overload, ) -from ._typing import TypeGuard, Unpack, dataclass_transform +from ._typing import Buffer, TypeGuard, Unpack, dataclass_transform from .field import Field, builtin_fields from .types import PadAfter, PadBefore @@ -192,7 +192,7 @@ def _init_from_args(self, args: Iterator) -> T: setattr(obj, name, arg) return obj - def _unpack(self, data: bytes) -> T: + def _unpack(self, data: Buffer) -> T: return self._init_from_args(iter(self.struct.unpack(data))) @@ -206,7 +206,7 @@ class DataclassStructProtocol(Protocol): """ @classmethod - def from_packed(cls: type[T], data: bytes) -> T: + def from_packed(cls: type[T], data: Buffer) -> T: """Return an instance of the class from its packed representation. Args: @@ -529,12 +529,12 @@ def pack(self) -> bytes: def _make_unpack_method(cls: type) -> classmethod: func = """ -def from_packed(cls, data: bytes) -> cls_type: +def from_packed(cls, data: Buffer) -> cls_type: '''Unpack from bytes.''' return cls.__dataclass_struct__._unpack(data) """ - scope: dict[str, Any] = {"cls_type": cls} + scope: dict[str, Any] = {"cls_type": cls, "Buffer": Buffer} exec(func, {}, scope) return classmethod(scope["from_packed"]) diff --git a/docs/guide.md b/docs/guide.md index 0c5d8c1..117a191 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -35,9 +35,11 @@ are added to the class: [`pack`][dataclasses_struct.DataclassStructProtocol.pack], a method for packing an instance of the class to `bytes`, and [`from_packed`][dataclasses_struct.DataclassStructProtocol.from_packed], a class -method that returns a new instance of the class from its packed `bytes` -representation. The additional `dataclass_kwargs` keyword arguments will be -passed through to the [stdlib `dataclass` +method that returns a new instance of the class from its packed representation +in an object that implements the [buffer +protococol](https://docs.python.org/3/c-api/buffer.html) (e.g. `bytes`). The +additional `dataclass_kwargs` keyword arguments will be passed through to the +[stdlib `dataclass` decorator](https://docs.python.org/3/library/dataclasses.html#dataclasses.dataclass): all standard keyword arguments are supported except for `slots` and `weakref_slot`. diff --git a/pyproject.toml b/pyproject.toml index 9643bc0..fe1ebf1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ classifiers = [ ] requires-python = ">=3.9.0" dependencies = [ - "typing-extensions>=4.12.2 ; python_full_version < '3.11'", + "typing-extensions>=4.12.2 ; python_full_version < '3.12'", ] [project.urls] diff --git a/uv.lock b/uv.lock index 7705fbd..b2e4161 100644 --- a/uv.lock +++ b/uv.lock @@ -238,7 +238,7 @@ name = "dataclasses-struct" version = "1.4.0" source = { editable = "." } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions", marker = "python_full_version < '3.12'" }, ] [package.dev-dependencies] @@ -256,7 +256,7 @@ docs = [ ] [package.metadata] -requires-dist = [{ name = "typing-extensions", marker = "python_full_version < '3.11'", specifier = ">=4.12.2" }] +requires-dist = [{ name = "typing-extensions", marker = "python_full_version < '3.12'", specifier = ">=4.12.2" }] [package.metadata.requires-dev] dev = [ From edd6ea317448966d036213df5375305aa652c3d2 Mon Sep 17 00:00:00 2001 From: Harry Mander Date: Tue, 24 Jun 2025 16:46:48 +1200 Subject: [PATCH 2/6] docs: restructure section on added methods and attributes --- docs/guide.md | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/docs/guide.md b/docs/guide.md index 117a191..2558d31 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -30,20 +30,30 @@ controls the size and alignment of fields: Decorated classes are transformed to a standard Python [dataclass](https://docs.python.org/3/library/dataclasses.html) with boilerplate -`__init__`, `__repr__`, `__eq__` etc. auto-generated. Additionally, two methods -are added to the class: -[`pack`][dataclasses_struct.DataclassStructProtocol.pack], a method for packing -an instance of the class to `bytes`, and -[`from_packed`][dataclasses_struct.DataclassStructProtocol.from_packed], a class -method that returns a new instance of the class from its packed representation -in an object that implements the [buffer -protococol](https://docs.python.org/3/c-api/buffer.html) (e.g. `bytes`). The -additional `dataclass_kwargs` keyword arguments will be passed through to the -[stdlib `dataclass` -decorator](https://docs.python.org/3/library/dataclasses.html#dataclasses.dataclass): +`__init__`, `__repr__`, `__eq__` etc. auto-generated. The additional +`dataclass_kwargs` keyword arguments will be passed through to the [stdlib +`dataclass` decorator](https://docs.python.org/3/library/dataclasses.html#dataclasses.dataclass): all standard keyword arguments are supported except for `slots` and `weakref_slot`. + +In addition to the standard `dataclass` methods, two methods +are added to the class: + +* [`pack`][dataclasses_struct.DataclassStructProtocol.pack], which packs an + instance of the class to `bytes`. +* [`from_packed`][dataclasses_struct.DataclassStructProtocol.from_packed], which + is a class method that returns a new instance of the class from its packed + representation in an object that implements the [buffer + protococol](https://docs.python.org/3/c-api/buffer.html) (`bytes`, + `bytearray`, [memory-mapped file + objects](https://docs.python.org/3/library/mmap.html) etc.). + +A class attribute named +[`__dataclass_struct__`][dataclasses_struct.DataclassStructProtocol.__dataclass_struct__] +is also added (see [Inspecting +dataclass-structs](#inspecting-dataclass-structs)). + ## Default value validation Default attribute values will be validated against their expected type and From cbd105635bc26c05a86cb33c88a230021dc702af Mon Sep 17 00:00:00 2001 From: Harry Mander Date: Tue, 24 Jun 2025 16:59:19 +1200 Subject: [PATCH 3/6] docs: use Buffer for from_packed arg type hint in pyright example usage --- docs/guide.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/guide.md b/docs/guide.md index 2558d31..d170365 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -504,6 +504,7 @@ can add stubs for the generated functions and attribute to the class: ```python from typing import ClassVar, TYPE_CHECKING +from collections.abc import Buffer # import from typing_extensions on Python <3.12 import dataclasses_struct as dcs @dcs.dataclass_struct() @@ -517,7 +518,7 @@ class Test: def pack(self) -> bytes: ... @classmethod - def from_packed(cls, data: bytes) -> "Test": ... + def from_packed(cls, data: Buffer) -> "Test": ... ``` The [`DataclassStructProtocol`][dataclasses_struct.DataclassStructProtocol] From edb186922758640ff67babc819d3551845f7052a Mon Sep 17 00:00:00 2001 From: Harry Mander Date: Tue, 24 Jun 2025 17:00:36 +1200 Subject: [PATCH 4/6] Test pack/unpack from buffer protocol objects --- test/test_pack_unpack.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/test/test_pack_unpack.py b/test/test_pack_unpack.py index 455a681..57a4ad9 100644 --- a/test/test_pack_unpack.py +++ b/test/test_pack_unpack.py @@ -1,6 +1,8 @@ import dataclasses import itertools +import mmap import struct +from pathlib import Path from typing import Annotated import pytest @@ -648,3 +650,37 @@ class T: t.x = -1 unpacked = T.from_packed(t.pack()) assert unpacked.x == -1 + + +def test_pack_unpack_bytearray() -> None: + @dcs.dataclass_struct() + class T: + x: dcs.Int = 100 + y: dcs.F32 = -1.5 + z: Annotated[bytes, 12] = b"hello, world" + + t = T() + packed_array = bytearray(t.pack()) + assert T.from_packed(packed_array) == t + + +def test_pack_unpack_mmap(tmp_path: Path) -> None: + @dcs.dataclass_struct() + class T: + x: dcs.Int = 100 + y: dcs.F32 = -1.5 + z: Annotated[bytes, 12] = b"hello, world" + + path = tmp_path / "data" + + t = T() + packed = t.pack() + path.write_bytes(packed) + + with path.open("rb+") as f, mmap.mmap(f.fileno(), 0) as mapped: + unpacked = T.from_packed(mapped) + + assert unpacked == t + + # Check that the file isn't changed + assert path.read_bytes() == packed From 025193712bb841e56fe227b38e163dcb6726d586 Mon Sep 17 00:00:00 2001 From: Harry Mander Date: Tue, 24 Jun 2025 17:08:09 +1200 Subject: [PATCH 5/6] Add mypy plugin tests for unpacking buffer protocol objects --- test/test-mypy-plugin.yml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/test/test-mypy-plugin.yml b/test/test-mypy-plugin.yml index efcd7af..5993e20 100644 --- a/test/test-mypy-plugin.yml +++ b/test/test-mypy-plugin.yml @@ -50,3 +50,37 @@ t = Test.from_packed(Test(1).pack()) reveal_type(t) # N: Revealed type is "main.Test" + +- case: test_from_packed_supports_bytearray_argument + mypy_config: 'plugins = dataclasses_struct.ext.mypy_plugin' + main: | + import dataclasses_struct as dcs + + @dcs.dataclass_struct() + class Test: + x: int + + t = Test.from_packed(bytearray(Test(1).pack())) + reveal_type(t) # N: Revealed type is "main.Test" + +- case: test_from_packed_supports_mmap_argument + mypy_config: 'plugins = dataclasses_struct.ext.mypy_plugin' + main: | + import mmap + import tempfile + from pathlib import Path + + import dataclasses_struct as dcs + + @dcs.dataclass_struct() + class Test: + x: int + + packed = Test(1).pack() + with tempfile.TemporaryDirectory() as tempdir: + path = Path(tempdir) / "data" + path.write_bytes(packed) + with path.open("rb+") as f, mmap.mmap(f.fileno(), 0) as mapped: + t = Test.from_packed(mapped) + + reveal_type(t) # N: Revealed type is "main.Test" From e1276e3828a02cdefad275b4cf02fe4b717afe60 Mon Sep 17 00:00:00 2001 From: Harry Mander Date: Tue, 24 Jun 2025 17:11:29 +1200 Subject: [PATCH 6/6] mypy: specify that from_packed takes a buffer protocol object --- dataclasses_struct/ext/mypy_plugin.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/dataclasses_struct/ext/mypy_plugin.py b/dataclasses_struct/ext/mypy_plugin.py index b57af20..cbfced7 100644 --- a/dataclasses_struct/ext/mypy_plugin.py +++ b/dataclasses_struct/ext/mypy_plugin.py @@ -11,6 +11,7 @@ def transform_dataclass_struct(ctx: ClassDefContext) -> bool: + buffer_type = ctx.api.named_type("dataclasses_struct._typing.Buffer") bytes_type = ctx.api.named_type("builtins.bytes") tvd = TypeVarType( "T", @@ -25,7 +26,14 @@ def transform_dataclass_struct(ctx: ClassDefContext) -> bool: ctx.api, ctx.cls, "from_packed", - [Argument(Var("data", bytes_type), bytes_type, None, ArgKind.ARG_POS)], + [ + Argument( + Var("data", buffer_type), + buffer_type, + None, + ArgKind.ARG_POS, + ) + ], tvd, self_type=TypeType(tvd), tvar_def=tvd,