Skip to content

add big endian dtypes to v2 #60

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
brokkoli71 wants to merge 2 commits into
base: main
Choose a base branch
from
Open

add big endian dtypes to v2 #60

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
663cd58
add big endian dtypes to v2
brokkoli71 Feb 13, 2026
aa8931a
add endian tests
brokkoli71 Feb 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,11 @@ public static Object parseFillValue(Object fillValue, @Nonnull DataType dataType
}
boolean dataTypeIsBool = dataType == dev.zarr.zarrjava.v3.DataType.BOOL || dataType == dev.zarr.zarrjava.v2.DataType.BOOL;
boolean dataTypeIsByte = dataType == dev.zarr.zarrjava.v3.DataType.INT8 || dataType == dev.zarr.zarrjava.v2.DataType.INT8 || dataType == dev.zarr.zarrjava.v3.DataType.UINT8 || dataType == dev.zarr.zarrjava.v2.DataType.UINT8;
boolean dataTypeIsShort = dataType == dev.zarr.zarrjava.v3.DataType.INT16 || dataType == dev.zarr.zarrjava.v2.DataType.INT16 || dataType == dev.zarr.zarrjava.v3.DataType.UINT16 || dataType == dev.zarr.zarrjava.v2.DataType.UINT16;
boolean dataTypeIsInt = dataType == dev.zarr.zarrjava.v3.DataType.INT32 || dataType == dev.zarr.zarrjava.v2.DataType.INT32 || dataType == dev.zarr.zarrjava.v3.DataType.UINT32 || dataType == dev.zarr.zarrjava.v2.DataType.UINT32;
boolean dataTypeIsLong = dataType == dev.zarr.zarrjava.v3.DataType.INT64 || dataType == dev.zarr.zarrjava.v2.DataType.INT64 || dataType == dev.zarr.zarrjava.v3.DataType.UINT64 || dataType == dev.zarr.zarrjava.v2.DataType.UINT64;
boolean dataTypeIsFloat = dataType == dev.zarr.zarrjava.v3.DataType.FLOAT32 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT32;
boolean dataTypeIsDouble = dataType == dev.zarr.zarrjava.v3.DataType.FLOAT64 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT64;
boolean dataTypeIsShort = dataType == dev.zarr.zarrjava.v3.DataType.INT16 || dataType == dev.zarr.zarrjava.v2.DataType.INT16 || dataType == dev.zarr.zarrjava.v2.DataType.INT16_BE || dataType == dev.zarr.zarrjava.v3.DataType.UINT16 || dataType == dev.zarr.zarrjava.v2.DataType.UINT16|| dataType == dev.zarr.zarrjava.v2.DataType.UINT16_BE;
boolean dataTypeIsInt = dataType == dev.zarr.zarrjava.v3.DataType.INT32 || dataType == dev.zarr.zarrjava.v2.DataType.INT32 || dataType == dev.zarr.zarrjava.v2.DataType.INT32_BE || dataType == dev.zarr.zarrjava.v3.DataType.UINT32 || dataType == dev.zarr.zarrjava.v2.DataType.UINT32 || dataType == dev.zarr.zarrjava.v2.DataType.UINT32_BE;
boolean dataTypeIsLong = dataType == dev.zarr.zarrjava.v3.DataType.INT64 || dataType == dev.zarr.zarrjava.v2.DataType.INT64 || dataType == dev.zarr.zarrjava.v2.DataType.INT64_BE || dataType == dev.zarr.zarrjava.v3.DataType.UINT64 || dataType == dev.zarr.zarrjava.v2.DataType.UINT64 || dataType == dev.zarr.zarrjava.v2.DataType.UINT64_BE;
boolean dataTypeIsFloat = dataType == dev.zarr.zarrjava.v3.DataType.FLOAT32 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT32 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT32_BE;
boolean dataTypeIsDouble = dataType == dev.zarr.zarrjava.v3.DataType.FLOAT64 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT64 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT64_BE;

if (fillValue instanceof Boolean) {
Boolean fillValueBool = (Boolean) fillValue;
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,12 +150,12 @@ public ArrayMetadata build() throws ZarrException {
if (dataType == null) {
throw new IllegalStateException("Please call `withDataType` first.");
}

// If chunks are not specified, calculate default chunks
if (chunks == null) {
chunks = Utils.calculateDefaultChunks(shape);
}

return new ArrayMetadata(
2,
shape,
Expand Down
18 changes: 17 additions & 1 deletion src/main/java/dev/zarr/zarrjava/v2/DataType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,15 @@ public enum DataType implements dev.zarr.zarrjava.core.DataType {
UINT32("u4", Endianness.LITTLE),
UINT64("u8", Endianness.LITTLE),
FLOAT32("f4", Endianness.LITTLE),
FLOAT64("f8", Endianness.LITTLE);
FLOAT64("f8", Endianness.LITTLE),
INT16_BE("i2", Endianness.BIG),
INT32_BE("i4", Endianness.BIG),
INT64_BE("i8", Endianness.BIG),
UINT16_BE("u2", Endianness.BIG),
UINT32_BE("u4", Endianness.BIG),
UINT64_BE("u8", Endianness.BIG),
FLOAT32_BE("f4", Endianness.BIG),
FLOAT64_BE("f8", Endianness.BIG);

private final String dtype;
private final Endianness endianness;
Expand Down Expand Up @@ -41,22 +49,30 @@ public ucar.ma2.DataType getMA2DataType() {
case INT8:
return ucar.ma2.DataType.BYTE;
case INT16:
case INT16_BE:
return ucar.ma2.DataType.SHORT;
case INT32:
case INT32_BE:
return ucar.ma2.DataType.INT;
case INT64:
case INT64_BE:
return ucar.ma2.DataType.LONG;
case UINT8:
return ucar.ma2.DataType.UBYTE;
case UINT16:
case UINT16_BE:
return ucar.ma2.DataType.USHORT;
case UINT32:
case UINT32_BE:
return ucar.ma2.DataType.UINT;
case UINT64:
case UINT64_BE:
return ucar.ma2.DataType.ULONG;
case FLOAT32:
case FLOAT32_BE:
return ucar.ma2.DataType.FLOAT;
case FLOAT64:
case FLOAT64_BE:
return ucar.ma2.DataType.DOUBLE;
default:
throw new RuntimeException("Unreachable");
Expand Down
109 changes: 4 additions & 105 deletions src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,97 +66,8 @@ public static void setupUV() {
}
}

static ucar.ma2.Array testdata(dev.zarr.zarrjava.core.DataType dt) {
ucar.ma2.DataType ma2Type = dt.getMA2DataType();
ucar.ma2.Array array = ucar.ma2.Array.factory(ma2Type, new int[]{16, 16, 16});
for (int i = 0; i < array.getSize(); i++) {
switch (ma2Type) {
case BOOLEAN:
array.setBoolean(i, i % 2 == 0);
break;
case BYTE:
case UBYTE:
array.setByte(i, (byte) i);
break;
case SHORT:
case USHORT:
array.setShort(i, (short) i);
break;
case INT:
array.setInt(i, i);
break;
case UINT:
array.setLong(i, i & 0xFFFFFFFFL);
break;
case LONG:
case ULONG:
array.setLong(i, i);
break;
case FLOAT:
array.setFloat(i, (float) i);
break;
case DOUBLE:
array.setDouble(i, i);
break;
default:
throw new IllegalArgumentException("Invalid DataType: " + dt);
}
}
return array;
}

static void assertIsTestdata(ucar.ma2.Array result, dev.zarr.zarrjava.core.DataType dt) {
// expected values are i for index i
ucar.ma2.DataType ma2Type = dt.getMA2DataType();
for (int i = 0; i < result.getSize(); i++) {
switch (ma2Type) {
case BOOLEAN:
Assertions.assertEquals(i % 2 == 0, result.getBoolean(i));
break;
case BYTE:
case UBYTE:
Assertions.assertEquals((byte) i, result.getByte(i));
break;
case SHORT:
case USHORT:
Assertions.assertEquals((short) i, result.getShort(i));
break;
case INT:
Assertions.assertEquals(i, result.getInt(i));
break;
case UINT:
Assertions.assertEquals(i & 0xFFFFFFFFL, result.getLong(i));
break;
case LONG:
case ULONG:
Assertions.assertEquals(i, result.getLong(i));
break;
case FLOAT:
Assertions.assertEquals((float) i, result.getFloat(i), 1e-6);
break;
case DOUBLE:
Assertions.assertEquals(i, result.getDouble(i), 1e-12);
break;
default:
throw new IllegalArgumentException("Invalid DataType: " + dt);
}
}
}

static Stream<Object[]> compressorAndDataTypeProviderV3() {
Stream<Object[]> datatypeTests = Stream.of(
DataType.BOOL,
DataType.INT8,
DataType.UINT8,
DataType.INT16,
DataType.UINT16,
DataType.INT32,
DataType.UINT32,
DataType.INT64,
DataType.UINT64,
DataType.FLOAT32,
DataType.FLOAT64
).flatMap(dt -> Stream.of(
Stream<Object[]> datatypeTests = dataTypeProviderV3().flatMap(dt -> Stream.of(
new Object[]{"sharding", "end", dt},
new Object[]{"blosc", "blosclz_shuffle_3", dt}
));
Expand Down Expand Up @@ -185,19 +96,7 @@ static Stream<Object[]> compressorAndDataTypeProviderV3() {
}

static Stream<Object[]> compressorAndDataTypeProviderV2() {
Stream<Object[]> datatypeTests = Stream.of(
dev.zarr.zarrjava.v2.DataType.BOOL,
dev.zarr.zarrjava.v2.DataType.INT8,
dev.zarr.zarrjava.v2.DataType.UINT8,
dev.zarr.zarrjava.v2.DataType.INT16,
dev.zarr.zarrjava.v2.DataType.UINT16,
dev.zarr.zarrjava.v2.DataType.INT32,
dev.zarr.zarrjava.v2.DataType.UINT32,
dev.zarr.zarrjava.v2.DataType.INT64,
dev.zarr.zarrjava.v2.DataType.UINT64,
dev.zarr.zarrjava.v2.DataType.FLOAT32,
dev.zarr.zarrjava.v2.DataType.FLOAT64
).flatMap(dt -> Stream.of(
Stream<Object[]> datatypeTests = dataTypeProviderV2().flatMap(dt -> Stream.of(
new Object[]{"zlib", "0", dt},
new Object[]{"blosc", "blosclz_shuffle_3", dt}
));
Expand Down Expand Up @@ -305,7 +204,7 @@ public void testWriteV3(String codec, String codecParam, DataType dataType) thro
@MethodSource("compressorAndDataTypeProviderV2")
public void testReadV2(String compressor, String compressorParam, dev.zarr.zarrjava.v2.DataType dt) throws IOException, ZarrException, InterruptedException {
StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testReadV2", compressor, compressorParam, dt.name());
run_python_script("zarr_python_write_v2.py", compressor, compressorParam, dt.name().toLowerCase(), storeHandle.toPath().toString());
run_python_script("zarr_python_write_v2.py", compressor, compressorParam, dt.getValue(), storeHandle.toPath().toString());

dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.open(storeHandle);
ucar.ma2.Array result = array.read();
Expand Down Expand Up @@ -361,7 +260,7 @@ public void testWriteV2(String compressor, String compressorParam, dev.zarr.zarr
assertIsTestdata(result, dt);

//read in zarr_python
run_python_script("zarr_python_read_v2.py", compressor, compressorParam, dt.name().toLowerCase(), storeHandle.toPath().toString());
run_python_script("zarr_python_read_v2.py", compressor, compressorParam, dt.getValue(), storeHandle.toPath().toString());
}

@CsvSource({"0,true", "0,false", "5, true", "10, false"})
Expand Down
120 changes: 120 additions & 0 deletions src/test/java/dev/zarr/zarrjava/ZarrTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package dev.zarr.zarrjava;

import dev.zarr.zarrjava.core.Attributes;
import dev.zarr.zarrjava.v3.DataType;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;

Expand Down Expand Up @@ -34,6 +35,46 @@ public static void clearTestoutputFolder() throws IOException {
Files.createDirectory(TESTOUTPUT);
}

static Stream<DataType> dataTypeProviderV3() {
return Stream.of(
DataType.BOOL,
DataType.INT8,
DataType.UINT8,
DataType.INT16,
DataType.UINT16,
DataType.INT32,
DataType.UINT32,
DataType.INT64,
DataType.UINT64,
DataType.FLOAT32,
DataType.FLOAT64
);
}

static Stream<dev.zarr.zarrjava.v2.DataType> dataTypeProviderV2() {
return Stream.of(
dev.zarr.zarrjava.v2.DataType.BOOL,
dev.zarr.zarrjava.v2.DataType.INT8,
dev.zarr.zarrjava.v2.DataType.UINT8,
dev.zarr.zarrjava.v2.DataType.INT16,
dev.zarr.zarrjava.v2.DataType.UINT16,
dev.zarr.zarrjava.v2.DataType.INT32,
dev.zarr.zarrjava.v2.DataType.UINT32,
dev.zarr.zarrjava.v2.DataType.INT64,
dev.zarr.zarrjava.v2.DataType.UINT64,
dev.zarr.zarrjava.v2.DataType.FLOAT32,
dev.zarr.zarrjava.v2.DataType.FLOAT64,
dev.zarr.zarrjava.v2.DataType.UINT16_BE,
dev.zarr.zarrjava.v2.DataType.UINT32_BE,
dev.zarr.zarrjava.v2.DataType.UINT64_BE,
dev.zarr.zarrjava.v2.DataType.INT16_BE,
dev.zarr.zarrjava.v2.DataType.INT32_BE,
dev.zarr.zarrjava.v2.DataType.INT64_BE,
dev.zarr.zarrjava.v2.DataType.FLOAT32_BE,
dev.zarr.zarrjava.v2.DataType.FLOAT64_BE
);
}

protected void assertListEquals(List<Object> a, List<Object> b) {
Assertions.assertEquals(a.size(), b.size());
for (int i = 0; i < a.size(); i++) {
Expand Down Expand Up @@ -112,4 +153,83 @@ protected void assertContainsTestAttributes(Attributes attributes) throws ZarrEx
);
}


protected ucar.ma2.Array testdata(dev.zarr.zarrjava.core.DataType dt) {
ucar.ma2.DataType ma2Type = dt.getMA2DataType();
ucar.ma2.Array array = ucar.ma2.Array.factory(ma2Type, new int[]{16, 16, 16});
for (int i = 0; i < array.getSize(); i++) {
switch (ma2Type) {
case BOOLEAN:
array.setBoolean(i, i % 2 == 0);
break;
case BYTE:
case UBYTE:
array.setByte(i, (byte) i);
break;
case SHORT:
case USHORT:
array.setShort(i, (short) i);
break;
case INT:
array.setInt(i, i);
break;
case UINT:
array.setLong(i, i & 0xFFFFFFFFL);
break;
case LONG:
case ULONG:
array.setLong(i, i);
break;
case FLOAT:
array.setFloat(i, (float) i);
break;
case DOUBLE:
array.setDouble(i, i);
break;
default:
throw new IllegalArgumentException("Invalid DataType: " + dt);
}
}
return array;
}

protected void assertIsTestdata(ucar.ma2.Array result, dev.zarr.zarrjava.core.DataType dt) {
// expected values are i for index i
ucar.ma2.DataType ma2Type = dt.getMA2DataType();
for (int i = 0; i < result.getSize(); i++) {
switch (ma2Type) {
case BOOLEAN:
Assertions.assertEquals(i % 2 == 0, result.getBoolean(i));
break;
case BYTE:
case UBYTE:
Assertions.assertEquals((byte) i, result.getByte(i));
break;
case SHORT:
case USHORT:
Assertions.assertEquals((short) i, result.getShort(i));
break;
case INT:
Assertions.assertEquals(i, result.getInt(i));
break;
case UINT:
Assertions.assertEquals(i & 0xFFFFFFFFL, result.getLong(i));
break;
case LONG:
case ULONG:
Assertions.assertEquals(i, result.getLong(i));
break;
case FLOAT:
Assertions.assertEquals((float) i, result.getFloat(i), 1e-6);
break;
case DOUBLE:
Assertions.assertEquals(i, result.getDouble(i), 1e-12);
break;
default:
throw new IllegalArgumentException("Invalid DataType: " + dt);
}
}
}


}
Loading
Loading