From 6ba6e891d103b42d40849de229183fea26745231 Mon Sep 17 00:00:00 2001 From: Johnson K C Date: Wed, 12 Aug 2026 12:29:12 -0700 Subject: [PATCH 1/3] fix: allow `require_array` to accept a `ZDType` (#4189) * fix: allow `require_array` to accept a `ZDType` AsyncGroup.require_array normalised its dtype with np.dtype(), which cannot consume a ZDType, so requiring an existing array with one raised a TypeError. Every sibling creation method already accepts ZDTypeLike. Widen the annotation and normalise via parse_data_type().to_native_dtype(). parse_data_type(None) resolves to float64 just as np.dtype(None) did, so the default is unchanged. This leaves numpy.typing unused, so drop it. * chore: rename changelog fragment to the PR number * fix: keep the float64 default explicit for mypy parse_data_type does not accept None, so pass "float64" directly, which is what np.dtype(None) resolved to before. * test: parametrize require_array dtype cases over (input, expected) pairs Covers the `dtype=None` path, which resolves to float64 and was previously untested, and asserts on the resulting ZDType rather than the native dtype. --------- Co-authored-by: Davis Bennett --- changes/4189.bugfix.md | 1 + src/zarr/core/group.py | 15 ++++++++++----- tests/test_group.py | 25 +++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 5 deletions(-) create mode 100644 changes/4189.bugfix.md diff --git a/changes/4189.bugfix.md b/changes/4189.bugfix.md new file mode 100644 index 0000000000..76ef7e7a5e --- /dev/null +++ b/changes/4189.bugfix.md @@ -0,0 +1 @@ +Allow `Group.require_array` to accept a `ZDType` for `dtype`, matching the other array creation methods. Previously an existing array could only be required with a string or NumPy dtype. diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 65f7767a29..548f2141d2 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -10,7 +10,6 @@ from typing import TYPE_CHECKING, Literal, assert_never, cast, overload import numpy as np -import numpy.typing as npt import zarr.api.asynchronous as async_api from zarr.abc.metadata import Metadata @@ -46,6 +45,7 @@ parse_shapelike, ) from zarr.core.config import config +from zarr.core.dtype import parse_data_type from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata from zarr.core.metadata.io import save_metadata from zarr.core.sync import SyncMixin, sync @@ -1225,7 +1225,7 @@ async def require_array( name: str, *, shape: ShapeLike, - dtype: npt.DTypeLike | None = None, + dtype: ZDTypeLike | None = None, exact: bool = False, **kwargs: Any, ) -> AnyAsyncArray: @@ -1239,8 +1239,9 @@ async def require_array( Array name. shape : int or tuple of ints Array shape. - dtype : str or dtype, optional - NumPy dtype. + dtype : ZDTypeLike, optional + The data type of the array, given as a string, a NumPy dtype, or a + Zarr data type. exact : bool, optional If True, require `dtype` to match exactly. If false, require `dtype` can be cast from array dtype. @@ -1258,7 +1259,11 @@ async def require_array( if shape != ds.shape: raise TypeError(f"Incompatible shape ({ds.shape} vs {shape})") - dtype = np.dtype(dtype) + # `np.dtype(None)` used to resolve to float64 here; keep that default. + dtype = parse_data_type( + "float64" if dtype is None else dtype, + zarr_format=self.metadata.zarr_format, + ).to_native_dtype() if exact: if ds.dtype != dtype: raise TypeError(f"Incompatible dtype ({ds.dtype} vs {dtype})") diff --git a/tests/test_group.py b/tests/test_group.py index 29377a5392..f7f2333ef5 100644 --- a/tests/test_group.py +++ b/tests/test_group.py @@ -24,6 +24,7 @@ from zarr.core._info import GroupInfo from zarr.core.buffer import default_buffer_prototype from zarr.core.config import config as zarr_config +from zarr.core.dtype import Float64, Int32 from zarr.core.dtype.common import unpack_dtype_json from zarr.core.dtype.npy.int import UInt8 from zarr.core.group import ( @@ -61,6 +62,7 @@ from zarr.core.buffer.core import Buffer from zarr.core.common import JSON, ZarrFormat + from zarr.core.dtype import ZDType, ZDTypeLike @pytest.fixture(params=["local", "memory", "zip"]) @@ -1439,6 +1441,29 @@ async def test_require_array(store: Store, zarr_format: ZarrFormat) -> None: await root.require_array("bar", shape=(10,), dtype="int8") +@pytest.mark.parametrize( + ("dtype", "expected"), + [ + (Int32(), Int32()), + (np.dtype("int32"), Int32()), + ("int32", Int32()), + (None, Float64()), + ], + ids=["zdtype", "numpy", "str", "none"], +) +async def test_require_array_zdtype( + store: Store, zarr_format: ZarrFormat, dtype: ZDTypeLike | None, expected: ZDType[Any, Any] +) -> None: + """An existing array can be required with a ZDType, as well as a string, a NumPy dtype, + or None. See https://github.com/zarr-developers/zarr-python/issues/3377 + """ + root = await AsyncGroup.from_store(store=store, zarr_format=zarr_format) + await root.create_array("foo", shape=(10,), dtype=expected) + + foo = await root.require_array("foo", shape=(10,), dtype=dtype, exact=True) + assert foo._zdtype == expected + + @pytest.mark.parametrize("consolidate", [True, False]) async def test_members_name(store: Store, consolidate: bool, zarr_format: ZarrFormat): group = Group.from_store(store=store, zarr_format=zarr_format) From 788c787c8a547b521c41ce316332b4daf406c39a Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 12 Aug 2026 21:34:51 +0200 Subject: [PATCH 2/3] fix: accept numpy integers as chunk sizes `normalize_chunks_nd` dispatches the scalar convenience form on `numbers.Integral`, but `normalize_chunks_1d` narrowed on `int`. Numpy integer scalars satisfy the former and not the latter, so a per-dimension numpy integer passed the outer dispatch and then fell into the branch meant for explicit per-dimension chunk sequences, where `list(chunks)` raised `TypeError: 'numpy.int64' object is not iterable`. Numpy integers arise naturally whenever a chunk shape is computed rather than written as a literal, since numpy reductions and elementwise ops yield numpy scalars. Narrow on `numbers.Integral` and coerce with `int()`, matching the caller and the sequence branch, which already accepted `Integral` elements. Move the `-1` sentinel check inside that branch. It previously ran on the raw input, so a numpy array chunk specification made `chunks == -1` return an array and raise an ambiguous-truth-value error; rectilinear specs given as numpy arrays now work. A chunk specification that is neither an integer nor iterable now names the offending value and its type instead of surfacing an opaque "object is not iterable" from `list(chunks)`. Fixes #4255 Assisted-by: ClaudeCode:claude-opus-5 --- changes/4255.bugfix.md | 1 + src/zarr/core/chunk_grids.py | 27 ++++++++++++++++++--------- tests/test_api.py | 2 +- tests/test_chunk_grids.py | 24 ++++++++++++++++++++++++ 4 files changed, 44 insertions(+), 10 deletions(-) create mode 100644 changes/4255.bugfix.md diff --git a/changes/4255.bugfix.md b/changes/4255.bugfix.md new file mode 100644 index 0000000000..6f2740ccb4 --- /dev/null +++ b/changes/4255.bugfix.md @@ -0,0 +1 @@ +Numpy integers are accepted as chunk sizes again. Since 3.3.0 a per-dimension chunk size that was a numpy integer (e.g. `chunks=(np.int64(2), np.int64(2))`, as produced by any computed chunk shape) raised `TypeError: 'numpy.int64' object is not iterable`, because the scalar chunk path narrowed on `int` while its caller dispatched on `numbers.Integral`. Numpy arrays are now also accepted as chunk specifications, and a chunk specification that is neither an integer nor iterable now reports the offending value instead of failing with an opaque iteration error. diff --git a/src/zarr/core/chunk_grids.py b/src/zarr/core/chunk_grids.py index 2cb9762775..584829bc6c 100644 --- a/src/zarr/core/chunk_grids.py +++ b/src/zarr/core/chunk_grids.py @@ -729,17 +729,26 @@ def normalize_chunks_1d( overhang the span. The actual data extent of each chunk is determined by the chunk grid at runtime, not by this function. """ - if chunks == -1: - return np.array([span], dtype=np.int64) - if isinstance(chunks, int): - if chunks <= 0: - raise ValueError(f"Chunk size must be positive, got {chunks}") + # `numbers.Integral` rather than `int` so that numpy integer scalars (which are not + # `int` subclasses) take the uniform-chunk path instead of being treated as a sequence. + if isinstance(chunks, numbers.Integral): + chunk_size = int(chunks) + if chunk_size == -1: + return np.array([span], dtype=np.int64) + if chunk_size <= 0: + raise ValueError(f"Chunk size must be positive, got {chunk_size}") if span == 0: - return np.array([chunks], dtype=np.int64) - n = ceildiv(span, chunks) - return np.full(n, chunks, dtype=np.int64) + return np.array([chunk_size], dtype=np.int64) + n = ceildiv(span, chunk_size) + return np.full(n, chunk_size, dtype=np.int64) else: - chunk_list = list(chunks) + try: + chunk_list = list(chunks) # type: ignore[arg-type] + except TypeError: + raise TypeError( + f"Chunk specification must be an integer or an iterable of integers; got " + f"{chunks!r} of type {type(chunks).__name__}." + ) from None if not chunk_list: raise ValueError("Chunk specification must not be empty") non_int = [ diff --git a/tests/test_api.py b/tests/test_api.py index 2b831e942d..45d0c0dee4 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -79,7 +79,7 @@ def test_create(memory_store: Store) -> None: z = create(shape=(400.5, 100), store=store, overwrite=True) # type: ignore[arg-type] # create array with float chunk shape - with pytest.raises(TypeError, match="'float' object is not iterable"): + with pytest.raises(TypeError, match="Chunk specification must be an integer or an iterable"): z = create(shape=(400, 100), chunks=(16, 16.5), store=store, overwrite=True) # type: ignore[arg-type] diff --git a/tests/test_chunk_grids.py b/tests/test_chunk_grids.py index b730a43901..4640c43d1c 100644 --- a/tests/test_chunk_grids.py +++ b/tests/test_chunk_grids.py @@ -68,6 +68,15 @@ def test_guess_chunks(shape: tuple[int, ...], itemsize: int) -> None: (10, (0,), ((10,),)), ((5, 10), (0, 100), ((5,), (10,) * 10)), ((5, 10), (20, 0), ((5, 5, 5, 5), (10,))), + # numpy integers are accepted anywhere a python int is, whether as the scalar + # convenience form, as per-dimension entries, or as the `-1` sentinel. + (np.int64(10), (100,), ((10,) * 10,)), + ((np.int64(2), np.int64(2)), (4, 4), ((2, 2), (2, 2))), + ((1, 3, np.int64(16), np.int64(16)), (1, 3, 32, 32), ((1,), (3,), (16, 16), (16, 16))), + ((np.int32(30), np.int64(-1)), (100, 20), ((30, 30, 30, 30), (20,))), + (np.array([10, 10]), (100, 100), ((10,) * 10, (10,) * 10)), + # rectilinear chunks given as numpy arrays + ((np.array([60, 40]), np.array([50, 50])), (100, 100), ((60, 40), (50, 50))), ], ) def test_normalize_chunks( @@ -142,7 +151,22 @@ def test_chunk_layout_nested() -> None: id="negative-uniform", msg="Chunk size must be positive", ), + ExpectFail( + input=(np.int64(0), 100), + exception=ValueError, + id="zero-uniform-numpy", + msg="Chunk size must be positive", + ), ExpectFail(input=([], 100), exception=ValueError, id="empty-list", msg="must not be empty"), + # Scalars that are neither integers nor iterable name themselves in the error, + # rather than surfacing an opaque "object is not iterable" from `list(chunks)`. + ExpectFail( + input=(2.5, 100), + exception=TypeError, + id="non-iterable-scalar", + msg="must be an integer or an iterable of integers; got 2.5 of type float", + escape=True, + ), ExpectFail( input=([10, -1, 10], 100), exception=ValueError, From 09b06d7cdf7a62e7175aaa48ddb1f310f1b20827 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Wed, 12 Aug 2026 21:40:56 +0200 Subject: [PATCH 3/3] Rename 4255.bugfix.md to 4257.bugfix.md --- changes/{4255.bugfix.md => 4257.bugfix.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changes/{4255.bugfix.md => 4257.bugfix.md} (100%) diff --git a/changes/4255.bugfix.md b/changes/4257.bugfix.md similarity index 100% rename from changes/4255.bugfix.md rename to changes/4257.bugfix.md