Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import io.github.dfa1.vortex.writer.WriteOptions;
import io.github.dfa1.vortex.writer.encode.VariantData;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.Schema;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
Expand Down Expand Up @@ -66,7 +67,7 @@ void javaWriter_jniReader_constantVariantColumn(@TempDir Path tmp) throws IOExce
assertThat(count).hasValue(rows);

Schema schema = ds.arrowSchema(ALLOCATOR);
assertThat(schema.getFields()).extracting(f -> f.getName()).contains("v");
assertThat(schema.getFields()).extracting(Field::getName).contains("v");
}

@Test
Expand All @@ -87,7 +88,7 @@ void javaWriter_jniReader_varyingVariantColumn(@TempDir Path tmp) throws IOExcep
// Then — the Rust reader parses the chunked variant layout and agrees on row count + schema.
DataSource ds = DataSource.open(SESSION, file.toAbsolutePath().toUri().toString());
assertThat(ds.rowCount().asOptional()).hasValue(values.size());
assertThat(ds.arrowSchema(ALLOCATOR).getFields()).extracting(f -> f.getName()).contains("v");
assertThat(ds.arrowSchema(ALLOCATOR).getFields()).extracting(Field::getName).contains("v");
}

@Test
Expand All @@ -108,7 +109,7 @@ void javaWriter_jniReader_shreddedVariantColumn(@TempDir Path tmp) throws IOExce
// Then — the Rust reader parses the shredded variant layout and agrees on row count + schema.
DataSource ds = DataSource.open(SESSION, file.toAbsolutePath().toUri().toString());
assertThat(ds.rowCount().asOptional()).hasValue(values.size());
assertThat(ds.arrowSchema(ALLOCATOR).getFields()).extracting(f -> f.getName()).contains("v");
assertThat(ds.arrowSchema(ALLOCATOR).getFields()).extracting(Field::getName).contains("v");
}

private static Scalar i32Variant(long value) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,8 @@ private static long estimateEncodedSizeF64(double[] sample, int expE, int expF,
private static AlpF64Data computeF64(double[] values) {
int n = values.length;
int[] exps = findExponentsF64(values);
int expE = exps[0], expF = exps[1];
int expE = exps[0];
int expF = exps[1];
double ef = F10_F64[expE];
double iff = IF10_F64[expF];
double df = F10_F64[expF];
Expand All @@ -166,7 +167,8 @@ private static AlpF64Data computeF64(double[] values) {
var patchIndices = new ArrayList<Integer>();
var patchValues = new ArrayList<Double>();

double min = Double.MAX_VALUE, max = -Double.MAX_VALUE;
double min = Double.MAX_VALUE;
double max = -Double.MAX_VALUE;
for (int i = 0; i < n; i++) {
double v = values[i];
double enc = v * ef * iff;
Expand Down Expand Up @@ -329,7 +331,8 @@ private static long estimateEncodedSizeF32(float[] sample, int expE, int expF, i
private static EncodeResult encodeF32(float[] values, EncodeContext ctx) {
int n = values.length;
int[] exps = findExponentsF32(values);
int expE = exps[0], expF = exps[1];
int expE = exps[0];
int expF = exps[1];
float ef = F10_F32[expE];
float iff = IF10_F32[expF];
float df = F10_F32[expF];
Expand All @@ -339,7 +342,8 @@ private static EncodeResult encodeF32(float[] values, EncodeContext ctx) {
var patchIndices = new ArrayList<Integer>();
var patchValues = new ArrayList<Float>();

float min = Float.MAX_VALUE, max = -Float.MAX_VALUE;
float min = Float.MAX_VALUE;
float max = -Float.MAX_VALUE;
for (int i = 0; i < n; i++) {
float v = values[i];
float enc = v * ef * iff;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,8 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) {
long mask = typeMask(ptype);
boolean unsign = isUnsigned(ptype);

long minVal = 0L, maxVal = 0L;
long minVal = 0L;
long maxVal = 0L;
if (n > 0) {
minVal = longs[0];
maxVal = longs[0];
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,8 @@ private static byte[][] computeStats(PType ptype, Object data) {
if (arr.length == 0) {
yield null;
}
long min = arr[0], max = arr[0];
long min = arr[0];
long max = arr[0];
for (byte v : arr) {
if (v < min) {
min = v;
Expand All @@ -109,7 +110,8 @@ private static byte[][] computeStats(PType ptype, Object data) {
if (arr.length == 0) {
yield null;
}
long min = arr[0], max = arr[0];
long min = arr[0];
long max = arr[0];
for (short v : arr) {
if (v < min) {
min = v;
Expand All @@ -125,7 +127,8 @@ private static byte[][] computeStats(PType ptype, Object data) {
if (arr.length == 0) {
yield null;
}
long min = arr[0], max = arr[0];
long min = arr[0];
long max = arr[0];
for (int v : arr) {
if (v < min) {
min = v;
Expand All @@ -141,7 +144,8 @@ private static byte[][] computeStats(PType ptype, Object data) {
if (arr.length == 0) {
yield null;
}
long min = arr[0], max = arr[0];
long min = arr[0];
long max = arr[0];
for (long v : arr) {
if (v < min) {
min = v;
Expand All @@ -157,7 +161,8 @@ private static byte[][] computeStats(PType ptype, Object data) {
if (arr.length == 0) {
yield null;
}
long min = Byte.toUnsignedInt(arr[0]), max = Byte.toUnsignedInt(arr[0]);
long min = Byte.toUnsignedInt(arr[0]);
long max = Byte.toUnsignedInt(arr[0]);
for (byte v : arr) {
long uv = Byte.toUnsignedInt(v);
if (uv < min) {
Expand All @@ -174,7 +179,8 @@ private static byte[][] computeStats(PType ptype, Object data) {
if (arr.length == 0) {
yield null;
}
long min = Short.toUnsignedInt(arr[0]), max = Short.toUnsignedInt(arr[0]);
long min = Short.toUnsignedInt(arr[0]);
long max = Short.toUnsignedInt(arr[0]);
for (short v : arr) {
long uv = Short.toUnsignedInt(v);
if (uv < min) {
Expand All @@ -191,7 +197,8 @@ private static byte[][] computeStats(PType ptype, Object data) {
if (arr.length == 0) {
yield null;
}
long min = Integer.toUnsignedLong(arr[0]), max = Integer.toUnsignedLong(arr[0]);
long min = Integer.toUnsignedLong(arr[0]);
long max = Integer.toUnsignedLong(arr[0]);
for (int v : arr) {
long uv = Integer.toUnsignedLong(v);
if (uv < min) {
Expand All @@ -208,7 +215,8 @@ private static byte[][] computeStats(PType ptype, Object data) {
if (arr.length == 0) {
yield null;
}
long min = arr[0], max = arr[0];
long min = arr[0];
long max = arr[0];
for (long v : arr) {
if (Long.compareUnsigned(v, min) < 0) {
min = v;
Expand All @@ -224,7 +232,8 @@ private static byte[][] computeStats(PType ptype, Object data) {
if (arr.length == 0) {
yield null;
}
float min = arr[0], max = arr[0];
float min = arr[0];
float max = arr[0];
for (float v : arr) {
if (v < min) {
min = v;
Expand All @@ -240,7 +249,8 @@ private static byte[][] computeStats(PType ptype, Object data) {
if (arr.length == 0) {
yield null;
}
double min = arr[0], max = arr[0];
double min = arr[0];
double max = arr[0];
for (double v : arr) {
if (v < min) {
min = v;
Expand All @@ -256,7 +266,8 @@ private static byte[][] computeStats(PType ptype, Object data) {
if (arr.length == 0) {
yield null;
}
float min = Float.float16ToFloat(arr[0]), max = Float.float16ToFloat(arr[0]);
float min = Float.float16ToFloat(arr[0]);
float max = Float.float16ToFloat(arr[0]);
for (short v : arr) {
float fv = Float.float16ToFloat(v);
if (fv < min) {
Expand Down