From 3fd91c452596b8d8147a3f86ea70c3f37639b4e8 Mon Sep 17 00:00:00 2001 From: jhughes Date: Mon, 6 Jul 2026 14:31:34 -0400 Subject: [PATCH 1/7] Add claude.md --- .gitignore | 15 ++++++++++++- CLAUDE.md | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 CLAUDE.md diff --git a/.gitignore b/.gitignore index 3586d30..71834d3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,13 @@ # Folder for generating test Zarr files -test_files/ +test_files/ + +# Coverage artifacts produced by tools/run_coverage.m +test/coverageReport/ +test/cobertura.xml +test/coverage_results.mat + +# Leaked by tZarrRead/tooBigArray (runs outside a WorkingFolderFixture) +test/bigData/ # Windows default autosave extension *.asv @@ -10,3 +18,8 @@ test_files/ # Bytecode-compiled version of python code PythonModule/__pycache__ *.pyc +.venv-zarr/ + +# Local Claude Code profile (per-developer, not shared) +.claude/ +.claude-profiles diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..21b3f7e --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,63 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Overview + +MATLAB interface for reading and writing Zarr v2 arrays and metadata, from both local storage and Amazon S3. The MATLAB layer delegates the actual Zarr I/O to Google's [tensorstore](https://github.com/google/tensorstore) Python library, which it calls through MATLAB's `py.` Python bridge. + +## Architecture + +The codebase is a three-language stack. Data and type information flow across all three layers, so a change to the data path usually touches each: + +1. **User-facing MATLAB functions** (`zarrread.m`, `zarrwrite.m`, `zarrcreate.m`, `zarrinfo.m`, `zarrwriteatt.m`) — thin wrappers that do `arguments`-block input validation and construct a `Zarr` object. These are the documented public API. + +2. **`Zarr.m`** — the central gateway class (`classdef Zarr < handle`). It owns the connection between MATLAB and Python: bootstrapping the Python module path (`pySetup`/`ZarrPy`), building the tensorstore KVStore schema (local `file` driver vs. S3 `s3` driver), resolving/creating paths and Zarr groups, validating partial-read parameters, and converting between MATLAB and numpy arrays. Most non-trivial logic lives here as static helper methods. + +3. **`PythonModule/ZarrPy.py`** — a small wrapper over tensorstore. Exposes `createKVStore`, `createZarr`, `writeZarr`, `readZarr`. This is the only code that talks to tensorstore directly. `Zarr.m` imports it via `py.importlib.import_module('ZarrPy')` after inserting `PythonModule/` onto `py.sys.path`. + +### Key cross-cutting concerns + +- **Datatype mapping** (`ZarrDatatype.m`): a single class holds three parallel arrays mapping MATLAB types ↔ tensorstore types ↔ Zarr dtype strings (e.g. `"double"` ↔ `"float64"` ↔ `".m` in `test/`. They inherit shared fixtures from `SharedZarrTestSetup.m`, which adds the parent source folder to the path and copies `test/dataFiles/` into a `WorkingFolderFixture` so write tests don't pollute the repo. Read-test fixtures live in `test/dataFiles/grp_v2` (and `grp_v3`); expected results are stored in `expZarrArrData.mat` / `expZarrArrInfo.mat`. + +## Conventions + +- All error messages use `error("MATLAB::", ...)` identifiers — match the existing namespacing when adding new ones. +- Function help text is the block comment directly under the signature; keep it current since the README points users to `help `. +- Spelling is checked in CI by codespell (`.codespellrc`); add false positives to `ignore-words-list`. From 410df3e3fa2919cdc4c4d1d6d9ba73afa1a6ad4d Mon Sep 17 00:00:00 2001 From: jhughes Date: Mon, 6 Jul 2026 14:49:37 -0400 Subject: [PATCH 2/7] Use test fixtures instead of manual setup/teardown Replace hand-rolled directory and permission juggling with the matlab.unittest fixtures built for the purpose: - tZarrRead, tZarrInfo: inherit SharedZarrTestSetup (PathFixture + WorkingFolderFixture) instead of re-declaring addSrcCodePath. This also stops tZarrRead/tooBigArray from leaking a bigData/ folder into the repo, since writes now land in the temp working folder. Fixture paths move from dataFiles/grp_v2/... to grp_v2/... to match the copied-contents layout. - Relative-path tests (tZarrRead, tZarrInfo): use CurrentFolderFixture instead of a hardcoded ../test/... path that only resolved from the original test folder. - tZarrCreate/createArrayRelativePath: use WorkingFolderFixture instead of manual mkdir + pwd capture + addTeardown(cd). - tZarrAttributes/noWritePermissions: isolate the read-only array in a TemporaryFolderFixture. Drops the '+w' teardown, which forced a specific permission state rather than restoring the original and mutated shared fixture data. Co-Authored-By: Claude Opus 4.8 --- test/tZarrAttributes.m | 18 +++++++++++++----- test/tZarrCreate.m | 10 ++++------ test/tZarrInfo.m | 31 +++++++++++++++---------------- test/tZarrRead.m | 38 ++++++++++++++++++-------------------- 4 files changed, 50 insertions(+), 47 deletions(-) diff --git a/test/tZarrAttributes.m b/test/tZarrAttributes.m index 262a86a..afeb5c6 100644 --- a/test/tZarrAttributes.m +++ b/test/tZarrAttributes.m @@ -111,13 +111,21 @@ function notZarrObject(testcase) function noWritePermissions(testcase) % Verify error if there are no write permissions to the Zarr array. - - % Make the folder read-only. - fileattrib(testcase.ArrPathWrite,'-w','','s'); - testcase.addTeardown(@()fileattrib(testcase.ArrPathWrite,'+w','','s')); + % Create the array inside an isolated temporary folder fixture so + % no shared fixture data is modified and cleanup is automatic + % (removal succeeds via the writable parent, so no permission + % restore is needed). + import matlab.unittest.fixtures.TemporaryFolderFixture + tempFixture = testcase.applyFixture(TemporaryFolderFixture); + + arrPath = fullfile(tempFixture.Folder, "roArr"); + zarrcreate(arrPath, testcase.ArrSize); + + % Make the array folder read-only. + fileattrib(arrPath,'-w','','s'); errID = 'MATLAB:zarrwriteatt:fileOpenFailure'; - testcase.verifyError(@()zarrwriteatt(testcase.ArrPathWrite,'myAttr','attrVal'), ... + testcase.verifyError(@()zarrwriteatt(arrPath,'myAttr','attrVal'), ... errID); end diff --git a/test/tZarrCreate.m b/test/tZarrCreate.m index a82e84e..a2f46ef 100644 --- a/test/tZarrCreate.m +++ b/test/tZarrCreate.m @@ -48,13 +48,11 @@ function createIntermediateZgroups(testcase) function createArrayRelativePath(testcase) % Verify that the array is successfully created if a relative - % path is used. - newDir = 'myFolder'; - currDir = pwd; - mkdir(newDir); - testcase.addTeardown(@()cd(currDir)); + % path is used. Work from a fresh temporary folder (which the + % fixture enters and cleans up) so the "../" path resolves. + import matlab.unittest.fixtures.WorkingFolderFixture + testcase.applyFixture(WorkingFolderFixture); - cd(newDir); inpPath = fullfile('..','myGrp','myArr'); zarrcreate(inpPath,[10 10]); arrInfo = zarrinfo(inpPath); diff --git a/test/tZarrInfo.m b/test/tZarrInfo.m index ad7b494..ddfd344 100644 --- a/test/tZarrInfo.m +++ b/test/tZarrInfo.m @@ -1,22 +1,18 @@ -classdef tZarrInfo < matlab.unittest.TestCase +classdef tZarrInfo < SharedZarrTestSetup % Tests for zarrinfo function to get info of the Zarr file in MATLAB. % Copyright 2025 The MathWorks, Inc. properties(Constant) - GrpPathV2 = "dataFiles/grp_v2" - ArrPathV2 = "dataFiles/grp_v2/arr_v2" - GrpPathV3 = "dataFiles/grp_v3" - ArrPathV3 = "dataFiles/grp_v3/arr_v3" - ExpInfo = load(fullfile(pwd,"dataFiles","expZarrArrInfo.mat")) - end + % SharedZarrTestSetup copies the contents of dataFiles/ into the + % working folder, so fixtures are at the working folder root. + GrpPathV2 = "grp_v2" + ArrPathV2 = "grp_v2/arr_v2" + GrpPathV3 = "grp_v3" + ArrPathV3 = "grp_v3/arr_v3" - methods(TestClassSetup) - function addSrcCodePath(testcase) - % Add source code path before running the tests - import matlab.unittest.fixtures.PathFixture - testcase.applyFixture(PathFixture(fullfile('..'),'IncludeSubfolders',true)) - end + % Loaded at class-load time, while pwd is still the test folder. + ExpInfo = load(fullfile(pwd,"dataFiles","expZarrArrInfo.mat")) end methods(Test) @@ -35,9 +31,12 @@ function verifyGroupInfoV2(testcase) end function getArrayInfoRelativePath(testcase) - % Verify array info if the input is using relative path to the - % array. - inpPath = fullfile('..','test',testcase.ArrPathV2); + % Verify array info if the input is using a relative path to the + % array. Read from a subfolder using a "../" prefixed path. + import matlab.unittest.fixtures.CurrentFolderFixture + testcase.applyFixture(CurrentFolderFixture("grp_v2")); + + inpPath = fullfile('..', testcase.ArrPathV2); actInfo = zarrinfo(inpPath); expInfo = testcase.ExpInfo.zarrV2ArrInfo; testcase.verifyEqual(actInfo, expInfo, ['Failed to verify array info ' ... diff --git a/test/tZarrRead.m b/test/tZarrRead.m index a815231..7f70bf0 100644 --- a/test/tZarrRead.m +++ b/test/tZarrRead.m @@ -1,28 +1,23 @@ -classdef tZarrRead < matlab.unittest.TestCase +classdef tZarrRead < SharedZarrTestSetup % Tests for zarrread function to read data from Zarr files in MATLAB. % Copyright 2025 The MathWorks, Inc. properties(Constant) - % Path for read functions - GrpPathRead = "dataFiles/grp_v2" - ArrPathRead = "dataFiles/grp_v2/arr_v2" - ArrPathReadSmall = "dataFiles/grp_v2/smallArr" - ArrPathReadVector = "dataFiles/grp_v2/vectorData" - ArrPathReadScalar = "dataFiles/grp_v2/scalarData" - ArrPathReadV3 = "dataFiles/grp_v3/arr_v3" - + % Paths for read functions. SharedZarrTestSetup copies the contents + % of dataFiles/ into the working folder, so fixtures are at the + % working folder root (grp_v2/..., grp_v3/...). + GrpPathRead = "grp_v2" + ArrPathRead = "grp_v2/arr_v2" + ArrPathReadSmall = "grp_v2/smallArr" + ArrPathReadVector = "grp_v2/vectorData" + ArrPathReadScalar = "grp_v2/scalarData" + ArrPathReadV3 = "grp_v3/arr_v3" + + % Loaded at class-load time, while pwd is still the test folder. ExpData = load(fullfile(pwd,"dataFiles","expZarrArrData.mat")) end - methods(TestClassSetup) - function addSrcCodePath(testcase) - % Add source code path before running the tests - import matlab.unittest.fixtures.PathFixture - testcase.applyFixture(PathFixture(fullfile('..'),'IncludeSubfolders',true)) - end - end - methods(Test) function verifyArrayData(testcase) % Verify array data using zarrread function. @@ -88,9 +83,12 @@ function verifyReadScalarData(testcase) end function verifyArrayDataRelativePath(testcase) - % Verify array data if the input is using relative path to the - % array. - inpPath = fullfile('..','test',testcase.ArrPathRead); + % Verify array data if the input is using a relative path to the + % array. Read from a subfolder using a "../" prefixed path. + import matlab.unittest.fixtures.CurrentFolderFixture + testcase.applyFixture(CurrentFolderFixture("grp_v2")); + + inpPath = fullfile('..', testcase.ArrPathRead); actArrData = zarrread(inpPath); expArrData = testcase.ExpData.arr_v2; testcase.verifyEqual(actArrData,expArrData,['Failed to verify array ' ... From 8ad3b9cbdeb1cea9aab97cb30c2baa1765e0a16e Mon Sep 17 00:00:00 2001 From: jhughes Date: Mon, 6 Jul 2026 14:50:18 -0400 Subject: [PATCH 3/7] Remove empty remote write test stubs createArrayRemoteDefaultSyntax and createArrayRemoteUserDefinedSyntax were empty method bodies (marked "Move to a separate file") added when the suite was first created and never implemented. An empty MATLAB test method passes unconditionally, so they silently reported green while covering nothing. Implementing real remote (S3) write tests requires a CI-friendly mock S3 endpoint (no live AWS dependency); that work is tracked in separate GitHub issues. Co-Authored-By: Claude Opus 4.8 --- test/tZarrWrite.m | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/test/tZarrWrite.m b/test/tZarrWrite.m index bc598db..96360a9 100644 --- a/test/tZarrWrite.m +++ b/test/tZarrWrite.m @@ -28,13 +28,6 @@ function createArrayLocalDefaultSyntax(testcase,ArrSizeWrite) testcase.verifyEqual(actData,expData,'Failed to verify array data'); end - function createArrayRemoteDefaultSyntax(testcase) - % Verify data when creating and writing to arrays of different - % dimensions using zarrcreate and zarrwrite to a remote location. - - % Move to a separate file - end - function createArrayLocalUserDefinedSyntax(testcase,DataType,CompId) % Verify the data when creating and writing to arrays with % user-defined properties using zarrcreate and zarrwrite locally. @@ -51,14 +44,6 @@ function createArrayLocalUserDefinedSyntax(testcase,DataType,CompId) ' with ' CompId ' compression.']); end - function createArrayRemoteUserDefinedSyntax(testcase) - % Verify data when creating and writing data to arrays with - % user-defined properties using zarrcreate and zarrwrite to a - % remote location. - - % Move to a separate file - end - function createArrayWithDefaultBloscConfig(testcase) % Verify data when creating and writing to a Zarr array using % a default blosc compression configuration. From 98018b6fbe154e6ce05a607e5a7f2a668db1a3fa Mon Sep 17 00:00:00 2001 From: jhughes Date: Mon, 6 Jul 2026 14:51:19 -0400 Subject: [PATCH 4/7] Add tests covering previously untested Zarr class methods Cover public methods the suite never exercised, raising overall line coverage from ~94.7% to ~98.1%: - Zarr.isZarrGroup and Zarr.isZarrArray - ZarrDatatype.fromTensorstoreType (round-trip and invalid-type error) - Zarr.createGroup: folder creation, and the read-only .zgroup open failure (via a TemporaryFolderFixture, so no real data is modified) - Zarr.write scalar-shape branch, via the scalarData fixture whose stored shape is a true scalar [1] (zarrcreate expands scalars to [1 N], so this branch is otherwise unreachable from the public API) Co-Authored-By: Claude Opus 4.8 --- test/tZarr.m | 93 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 92 insertions(+), 1 deletion(-) diff --git a/test/tZarr.m b/test/tZarr.m index 8bdad5c..0274f99 100644 --- a/test/tZarr.m +++ b/test/tZarr.m @@ -35,6 +35,97 @@ function verifyReload(testcase) end - + function verifyIsZarrArrayAndGroup(testcase) + % Verify that isZarrArray and isZarrGroup correctly identify a + % Zarr array (has .zarray) versus a Zarr group (has .zgroup). + % SharedZarrTestSetup copies the *contents* of dataFiles into + % the working folder, so fixtures live at grp_v2/... directly. + arrPath = "grp_v2/arr_v2"; + grpPath = "grp_v2"; + + testcase.verifyTrue(Zarr.isZarrArray(arrPath),... + "Expected an array path to be a Zarr array."); + testcase.verifyFalse(Zarr.isZarrArray(grpPath),... + "Did not expect a group path to be a Zarr array."); + + testcase.verifyTrue(Zarr.isZarrGroup(grpPath),... + "Expected a group path to be a Zarr group."); + testcase.verifyFalse(Zarr.isZarrGroup(arrPath),... + "Did not expect an array path to be a Zarr group."); + end + + function verifyDatatypeRoundTrip(testcase) + % Verify that ZarrDatatype maps consistently across MATLAB, + % Tensorstore, and Zarr type names, regardless of which static + % constructor is used to create it. + mlType = "double"; + tsType = "float64"; + zType = " Date: Mon, 6 Jul 2026 14:51:27 -0400 Subject: [PATCH 5/7] Add local coverage runner tools/run_coverage.m runs the full test suite with code coverage over the source folder (excluding test/, matching codecov.yml) and produces an HTML report plus a Cobertura XML under test/. Mirrors what CI collects, for reproducing coverage locally. Generated artifacts are gitignored. Co-Authored-By: Claude Opus 4.8 --- tools/run_coverage.m | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 tools/run_coverage.m diff --git a/tools/run_coverage.m b/tools/run_coverage.m new file mode 100644 index 0000000..e817630 --- /dev/null +++ b/tools/run_coverage.m @@ -0,0 +1,39 @@ +function results = run_coverage() +% RUN_COVERAGE Run the full Zarr test suite with code coverage. +% Measures line coverage over the source folder (repo root, excluding +% the test/ folder), mirroring what CI collects. Produces an HTML report +% and a Cobertura XML under test/, and prints a per-test pass/fail table. +% +% Run from anywhere; paths are resolved relative to this file. + +% Copyright 2026, The MathWorks, inc. + +import matlab.unittest.TestRunner +import matlab.unittest.Verbosity +import matlab.unittest.plugins.CodeCoveragePlugin +import matlab.unittest.plugins.codecoverage.CoverageReport +import matlab.unittest.plugins.codecoverage.CoberturaFormat + +repoRoot = fileparts(fileparts(mfilename('fullpath'))); +testFolder = fullfile(repoRoot, 'test'); + +suite = testsuite(testFolder, 'IncludeSubfolders', true); +runner = TestRunner.withTextOutput('OutputDetail', Verbosity.Terse); + +reportDir = fullfile(testFolder, 'coverageReport'); +if ~isfolder(reportDir) + mkdir(reportDir); +end + +% Cover the source folder but not the tests themselves (matches codecov.yml). +runner.addPlugin(CodeCoveragePlugin.forFolder(repoRoot, ... + 'Producing', [ ... + CoverageReport(reportDir), ... + CoberturaFormat(fullfile(testFolder, 'cobertura.xml'))])); + +results = runner.run(suite); + +disp(table([results.Passed]', [results.Failed]', [results.Incomplete]', ... + 'VariableNames', {'Passed','Failed','Incomplete'}, ... + 'RowNames', {results.Name})); +end From 5a8c3bb22f3de952465dcb785eb92027eea29cbe Mon Sep 17 00:00:00 2001 From: jhughes Date: Mon, 6 Jul 2026 15:00:52 -0400 Subject: [PATCH 6/7] Remove tools from coverage metric --- codecov.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/codecov.yml b/codecov.yml index 23a0e90..ed2418a 100644 --- a/codecov.yml +++ b/codecov.yml @@ -5,4 +5,5 @@ coverage: target: 85% threshold: 5% ignore: - - "test/*" \ No newline at end of file + - "test/*" + - "tools/*" \ No newline at end of file From 36a142a0f4bd2f341edba4765b59e2d686c4fa8d Mon Sep 17 00:00:00 2001 From: jhughes Date: Mon, 6 Jul 2026 15:37:59 -0400 Subject: [PATCH 7/7] Make read-only permission tests work on Windows The read-only-folder approach does not fail on Windows: the directory read-only attribute is ignored by the OS for file creation, so the .zgroup / .zattrs write succeeded and no error was raised. (The original noWritePermissions test passed on Windows only incidentally, because it marked a pre-existing .zattrs *file* read-only via recursion.) Make the specific target file read-only instead of its folder. A read-only file is honored on both Windows and Unix, so the open-for- write failure fires on all platforms. Write permission is restored on teardown so the fixture can clean up. Co-Authored-By: Claude Opus 4.8 --- test/tZarr.m | 21 ++++++++++++++------- test/tZarrAttributes.m | 25 +++++++++++++++++-------- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/test/tZarr.m b/test/tZarr.m index 0274f99..37500cb 100644 --- a/test/tZarr.m +++ b/test/tZarr.m @@ -99,17 +99,24 @@ function verifyCreateGroupMakesFolder(testcase) function verifyCreateGroupOpenFailure(testcase) % Verify error when the .zgroup file cannot be opened for - % writing (e.g. the target folder is read-only). The read-only - % folder lives inside an isolated temporary folder fixture, so - % no real data is modified and cleanup is automatic (removal - % succeeds via the writable parent, so no permission restore is - % needed). + % writing. Everything lives inside an isolated temporary folder + % fixture, so no real data is modified. + % + % We make the existing .zgroup *file* read-only rather than its + % folder: a read-only folder does not prevent file creation on + % Windows (the directory read-only attribute is ignored there), + % whereas a read-only file is honored on both Windows and Unix. import matlab.unittest.fixtures.TemporaryFolderFixture tempFixture = testcase.applyFixture(TemporaryFolderFixture); groupPath = fullfile(tempFixture.Folder, "readOnlyGroup"); - mkdir(groupPath); - fileattrib(groupPath, '-w', '', 's'); + Zarr.createGroup(groupPath); % writes .zgroup + zgroupFile = fullfile(groupPath, ".zgroup"); + + fileattrib(zgroupFile, '-w'); + % Restore write permission before the fixture is torn down so its + % contents can be removed (runs before the fixture's rmdir). + testcase.addTeardown(@()fileattrib(zgroupFile, '+w')); testcase.verifyError(@()Zarr.createGroup(groupPath),... "MATLAB:Zarr:fileOpenFailure"); diff --git a/test/tZarrAttributes.m b/test/tZarrAttributes.m index afeb5c6..8750f02 100644 --- a/test/tZarrAttributes.m +++ b/test/tZarrAttributes.m @@ -110,19 +110,28 @@ function notZarrObject(testcase) end function noWritePermissions(testcase) - % Verify error if there are no write permissions to the Zarr array. - % Create the array inside an isolated temporary folder fixture so - % no shared fixture data is modified and cleanup is automatic - % (removal succeeds via the writable parent, so no permission - % restore is needed). + % Verify error if the .zattrs file cannot be opened for writing. + % Everything lives inside an isolated temporary folder fixture so + % no shared fixture data is modified. + % + % We make the existing .zattrs *file* read-only rather than its + % folder: a read-only folder does not prevent file creation on + % Windows (the directory read-only attribute is ignored there), + % whereas a read-only file is honored on both Windows and Unix. import matlab.unittest.fixtures.TemporaryFolderFixture tempFixture = testcase.applyFixture(TemporaryFolderFixture); arrPath = fullfile(tempFixture.Folder, "roArr"); zarrcreate(arrPath, testcase.ArrSize); - - % Make the array folder read-only. - fileattrib(arrPath,'-w','','s'); + % Write one attribute so the .zattrs file exists, then make it + % read-only so the next write cannot open it. + zarrwriteatt(arrPath, 'existingAttr', 1); + zattrsFile = fullfile(arrPath, '.zattrs'); + + fileattrib(zattrsFile, '-w'); + % Restore write permission before the fixture is torn down so its + % contents can be removed (runs before the fixture's rmdir). + testcase.addTeardown(@()fileattrib(zattrsFile, '+w')); errID = 'MATLAB:zarrwriteatt:fileOpenFailure'; testcase.verifyError(@()zarrwriteatt(arrPath,'myAttr','attrVal'), ...