From 8fc975d94621a30e3393e63ad19a4a079586b9b6 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 17 Jun 2026 12:40:04 +0200 Subject: [PATCH 1/4] fix(qwp): empty arrays are distinct from NULL across the columnar codec QuestDB stores a non-null empty array (cardinality 0) as a value distinct from a NULL array, and emits it on QWP egress inline with a 0-length dimension. The egress decoder rejected dl < 1, so reading an empty array back failed ("ARRAY dim N must be >= 1"); the 1D/2D/3D/ND array setters also collapsed a nil slice into a (malformed) empty array instead of NULL. - parseArray: accept dl >= 0, keeping nDims >= 1 and a per-dimension cap so a 0 in one dimension can't slip an arbitrary value past the element-count cap. Matches the server's QwpArrayColumnCursor. - Array setters (Float64/Int64 1D/2D/3D + ND): a nil slice is a NULL array (null bitmap); a non-nil empty slice is a distinct empty array (inline, 0 elements). - Tests: encode->decode round-trip for empty vs NULL and nil/empty setter semantics (unit), plus a real-server ingest->egress round-trip (TestQwpIntegrationEmptyAndNullArray). Flip the stale H27b zero-dim hardening case to assert the empty array now decodes. Co-Authored-By: Claude Opus 4.8 (1M context) --- qwp_query_decoder.go | 27 +++++--- qwp_query_decoder_test.go | 117 ++++++++++++++++++++++++++++++++-- qwp_query_integration_test.go | 96 ++++++++++++++++++++++++++++ qwp_sender.go | 54 +++++++++++++++- qwp_sender_test.go | 45 +++++++++++++ 5 files changed, 319 insertions(+), 20 deletions(-) diff --git a/qwp_query_decoder.go b/qwp_query_decoder.go index b7bff43..c0d84ab 100644 --- a/qwp_query_decoder.go +++ b/qwp_query_decoder.go @@ -889,9 +889,12 @@ func (d *qwpQueryDecoder) parseGeohash(l *qwpColumnLayout) error { // region of the payload so accessors can address elements by // (row-start + offset). // -// The server encodes a NULL array via the null bitmap, never inline, -// so a non-null row must carry nDims >= 1. An inline nDims of 0 is -// rejected as a malformed frame. +// A NULL array is signaled by the null bitmap, never inline, so a +// non-null row must carry nDims >= 1; an inline nDims of 0 is rejected +// as a malformed frame. Individual dimensions may be 0, though: +// QuestDB stores empty arrays (cardinality 0) of various shapes as +// distinct, non-null values, so a 0-length dimension is valid and +// yields a 0-element row. func (d *qwpQueryDecoder) parseArray(l *qwpColumnLayout, rowCount int) error { base := d.br.pos if cap(l.arrayRowStart) < rowCount { @@ -928,14 +931,18 @@ func (d *qwpQueryDecoder) parseArray(l *qwpColumnLayout, rowCount int) error { elements := int64(1) for dim := 0; dim < nDims; dim++ { dl := int32(binary.LittleEndian.Uint32(shapeBytes[dim*4:])) - // Require dl >= 1 in every dimension. A dl of 0 would zero out - // elements and short-circuit the qwpMaxArrayElements cap for - // the remaining dimensions, letting them hold arbitrary values - // unchecked; the encoder never emits dl == 0. Matches - // QwpResultBatchDecoder.java. - if dl < 1 { + // A 0-length dimension is a valid empty array (cardinality 0), + // distinct from a NULL array (which the null bitmap carries). + // Reject only a negative length, and bound each dimension + // independently: a single 0 collapses the element-count product + // to 0, so without a per-dimension cap a 0 in one dimension + // would let another hold an arbitrary value the product check + // below could no longer catch. Matches the server's + // QwpArrayColumnCursor. + if dl < 0 || int64(dl) > qwpMaxArrayElements { return newQwpDecodeError(fmt.Sprintf( - "ARRAY dim %d must be >= 1: %d", dim, dl)) + "ARRAY dim %d out of range [0, %d]: %d", + dim, qwpMaxArrayElements, dl)) } elements *= int64(dl) if elements > qwpMaxArrayElements { diff --git a/qwp_query_decoder_test.go b/qwp_query_decoder_test.go index 1547b8f..8dd6b18 100644 --- a/qwp_query_decoder_test.go +++ b/qwp_query_decoder_test.go @@ -885,6 +885,95 @@ func TestQwpDecoderRoundTripFloat64Array(t *testing.T) { } } +// TestQwpDecoderRoundTripEmptyVsNullArray pins the empty-vs-NULL array +// distinction across the wire. QuestDB stores a non-null empty array +// (cardinality 0) of a given shape as a value distinct from a NULL +// array: the encoder writes an empty array inline (nDims >= 1 with a +// 0-length dimension) and a NULL via the null bitmap, and the decoder +// must round-trip both without conflating them. Before the +// per-dimension guard in parseArray was relaxed from "dl >= 1" to +// "dl >= 0", decoding the empty-array rows failed outright. +func TestQwpDecoderRoundTripEmptyVsNullArray(t *testing.T) { + tb := newQwpTableBuffer("t") + mk := func() *qwpColumnBuffer { + col, err := tb.getOrCreateColumn("a", qwpTypeDoubleArray, true) + if err != nil { + t.Fatalf("getOrCreateColumn: %v", err) + } + return col + } + // Row 0: regular 1x3 array. + mk().addDoubleArray(1, []int32{3}, []float64{1, 2, 3}) + tb.commitRow() + // Row 1: empty 1D array, shape {0}. + mk().addDoubleArray(1, []int32{0}, nil) + tb.commitRow() + // Row 2: NULL array. + mk().addNull() + tb.commitRow() + // Row 3: empty 2D array, shape {2, 0} — still cardinality 0. + mk().addDoubleArray(2, []int32{2, 0}, nil) + tb.commitRow() + + var enc qwpEncoder + frame := wrapAsResultBatch(enc.encodeTable(tb), 1, 0) + dec := newTestQueryDecoder() + var batch QwpColumnBatch + if err := dec.decode(frame, &batch); err != nil { + t.Fatalf("decode: %v", err) + } + + // Row 0: regular, non-null, [1 2 3]. + if batch.IsNull(0, 0) { + t.Fatal("row 0 (regular array) must NOT be null") + } + got0, want0 := batch.Float64Array(0, 0), []float64{1, 2, 3} + if len(got0) != len(want0) { + t.Fatalf("row 0 len = %d, want %d", len(got0), len(want0)) + } + for i := range want0 { + if got0[i] != want0[i] { + t.Fatalf("row 0[%d] = %v, want %v", i, got0[i], want0[i]) + } + } + + // Row 1: empty 1D — non-null, nDims 1, dim 0 == 0, non-nil empty slice. + if batch.IsNull(0, 1) { + t.Fatal("row 1 (empty 1D array) must NOT be null") + } + if nd := batch.ArrayNDims(0, 1); nd != 1 { + t.Fatalf("row 1 ArrayNDims = %d, want 1", nd) + } + if d0 := batch.ArrayDim(0, 1, 0); d0 != 0 { + t.Fatalf("row 1 ArrayDim(0) = %d, want 0", d0) + } + if got := batch.Float64Array(0, 1); got == nil || len(got) != 0 { + t.Fatalf("row 1 Float64Array = %v (nil=%t), want non-nil empty", got, got == nil) + } + + // Row 2: NULL — IsNull true, Float64Array nil. + if !batch.IsNull(0, 2) { + t.Fatal("row 2 (NULL array) must be null") + } + if got := batch.Float64Array(0, 2); got != nil { + t.Fatalf("row 2 Float64Array = %v, want nil (NULL)", got) + } + + // Row 3: empty 2D {2, 0} — non-null, nDims 2, cardinality 0. + if batch.IsNull(0, 3) { + t.Fatal("row 3 (empty 2D array) must NOT be null") + } + if nd := batch.ArrayNDims(0, 3); nd != 2 { + t.Fatalf("row 3 ArrayNDims = %d, want 2", nd) + } + if d0, d1 := batch.ArrayDim(0, 3, 0), batch.ArrayDim(0, 3, 1); d0 != 2 || d1 != 0 { + t.Fatalf("row 3 ArrayDim = %dx%d, want 2x0", d0, d1) + } + if got := batch.Float64Array(0, 3); got == nil || len(got) != 0 { + t.Fatalf("row 3 Float64Array = %v (nil=%t), want non-nil empty", got, got == nil) + } +} + func TestQwpDecoderRoundTripSymbolDelta(t *testing.T) { // Batch 1 introduces three symbols; Batch 2 adds one more via a // delta section. The decoder's connection-scoped dict must grow @@ -1681,16 +1770,30 @@ func TestQwpDecoderHardening(t *testing.T) { assertDecodeErrContains(t, err, "ARRAY dim") }) - t.Run("H27b_ArrayZeroDim", func(t *testing.T) { - // shape[0] = 0. A zero-extent dimension would zero out the - // element count and short-circuit the qwpMaxArrayElements cap - // for any remaining dimensions. The encoder never emits dl == 0; - // the decoder must reject it (matches Java's dl < 1 guard). + t.Run("H27b_ArrayZeroDimIsEmptyArray", func(t *testing.T) { + // shape[0] = 0 is a valid empty 1D array (cardinality 0), distinct + // from a NULL array (which the null bitmap carries). The decoder + // must accept it and report a non-null, zero-element row. The + // per-dimension guard still collapses the element-count product to + // 0, so the trailing padding bytes are simply left unconsumed. frame := buildArrayHardeningFrame(t, 1, []int32{0}) dec := newTestQueryDecoder() var b QwpColumnBatch - err := dec.decode(frame, &b) - assertDecodeErrContains(t, err, "ARRAY dim") + if err := dec.decode(frame, &b); err != nil { + t.Fatalf("decode: %v", err) + } + if b.IsNull(0, 0) { + t.Fatal("empty array must NOT be null") + } + if nd := b.ArrayNDims(0, 0); nd != 1 { + t.Fatalf("ArrayNDims = %d, want 1", nd) + } + if d0 := b.ArrayDim(0, 0, 0); d0 != 0 { + t.Fatalf("ArrayDim(0) = %d, want 0", d0) + } + if got := b.Float64Array(0, 0); got == nil || len(got) != 0 { + t.Fatalf("Float64Array = %v (nil=%t), want non-nil empty", got, got == nil) + } }) t.Run("H28_ArrayElementCountExceeded", func(t *testing.T) { diff --git a/qwp_query_integration_test.go b/qwp_query_integration_test.go index d8ca78a..8756d6d 100644 --- a/qwp_query_integration_test.go +++ b/qwp_query_integration_test.go @@ -116,6 +116,102 @@ func TestQwpIntegrationQuerySimpleSelect(t *testing.T) { } } +// TestQwpIntegrationEmptyAndNullArray ingests a regular, an empty, and a +// NULL DOUBLE[] via the QWP sender, then reads them back via the QWP +// query client — a full Go round-trip against a real QuestDB. It pins the +// empty-vs-NULL distinction end-to-end: the sender's nil->NULL / +// empty->empty-array encoding, the server's ingest + storage + egress of +// an empty array (emitted inline with a 0-length dimension), and the +// decoder's acceptance of that form. Before parseArray was relaxed from +// dl>=1 to dl>=0, the egress decode of the empty-array row failed with +// "ARRAY dim 0 must be >= 1: 0". +func TestQwpIntegrationEmptyAndNullArray(t *testing.T) { + const tableName = "qwp_integ_empty_null_array" + qwpEnsureServer(t) + qwpDropTable(t, tableName) + defer qwpDropTable(t, tableName) + + ctx := context.Background() + s, err := newQwpLineSender(ctx, "ws://"+qwpTestAddr, + qwpTransportOpts{endpointPath: qwpWritePath}, 0, 0, nil) + if err != nil { + t.Fatalf("newQwpLineSender: %v", err) + } + base := time.Date(2024, 6, 15, 12, 0, 0, 0, time.UTC) + // Row 0 establishes DOUBLE[] (1-D) with a regular array; row 1 is an + // empty array; row 2 is NULL. `i` is an order key so the read side does + // not depend on WAL scan order. + if err := s.Table(tableName).Int64Column("i", 0). + Float64Array1DColumn("d", []float64{1, 2}). + At(ctx, base); err != nil { + t.Fatalf("At row 0: %v", err) + } + if err := s.Table(tableName).Int64Column("i", 1). + Float64Array1DColumn("d", []float64{}). + At(ctx, base.Add(time.Second)); err != nil { + t.Fatalf("At row 1 (empty): %v", err) + } + if err := s.Table(tableName).Int64Column("i", 2). + Float64Array1DColumn("d", nil). + At(ctx, base.Add(2*time.Second)); err != nil { + t.Fatalf("At row 2 (nil): %v", err) + } + if err := s.Flush(ctx); err != nil { + t.Fatalf("Flush: %v", err) + } + _ = s.Close(ctx) + qwpWaitForRows(t, tableName, 3) + + c := newTestQueryClient(t) + qctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + defer c.Close(qctx) + + type rowResult struct { + seen bool + isNull bool + nDims int + elems []float64 + } + got := map[int64]rowResult{} + // Column 0 = i (order key), column 1 = d (the array). + q := c.Query(qctx, fmt.Sprintf("SELECT i, d FROM '%s'", tableName)) + defer q.Close() + for batch, err := range q.Batches() { + if err != nil { + t.Fatalf("iter err: %v", err) + } + for r := 0; r < batch.RowCount(); r++ { + idx := batch.Int64(0, r) + rr := rowResult{seen: true, isNull: batch.IsNull(1, r)} + if !rr.isNull { + rr.nDims = batch.ArrayNDims(1, r) + rr.elems = batch.Float64Array(1, r) + } + got[idx] = rr + } + } + if len(got) != 3 { + t.Fatalf("got %d distinct rows, want 3: %+v", len(got), got) + } + + // Row 0: regular 1-D array [1, 2]. + if r0 := got[0]; r0.isNull || r0.nDims != 1 || len(r0.elems) != 2 || + r0.elems[0] != 1 || r0.elems[1] != 2 { + t.Fatalf("row 0 (regular) = %+v, want non-null 1-D [1 2]", r0) + } + // Row 1: empty array -> non-null, 1-D, zero elements (distinct from NULL). + if r1 := got[1]; r1.isNull { + t.Fatalf("row 1 (empty array) must NOT be null: %+v", r1) + } else if r1.nDims != 1 || len(r1.elems) != 0 { + t.Fatalf("row 1 (empty) nDims=%d len=%d, want 1 and 0", r1.nDims, len(r1.elems)) + } + // Row 2: NULL array. + if r2 := got[2]; !r2.isNull { + t.Fatalf("row 2 (NULL array) must be null: %+v", r2) + } +} + // TestQwpIntegrationQueryError runs a SELECT against a nonexistent // table and verifies the server's QUERY_ERROR surfaces as a // *QwpQueryError with a useful message. diff --git a/qwp_sender.go b/qwp_sender.go index 2aacd93..4e43576 100644 --- a/qwp_sender.go +++ b/qwp_sender.go @@ -801,6 +801,13 @@ func (s *qwpLineSender) Float64Array1DColumn(name string, values []float64) Line s.lastErr = err return s } + // A nil slice is a NULL array (carried in the null bitmap); a non-nil + // empty slice is a zero-length 1D array (shape {0}), which QuestDB + // stores as a distinct, non-null value. + if values == nil { + col.addNull() + return s + } col.addDoubleArray(1, []int32{int32(len(values))}, values) return s } @@ -823,6 +830,12 @@ func (s *qwpLineSender) Float64Array2DColumn(name string, values [][]float64) Li return s } + // A nil slice is a NULL array; a non-nil empty slice is an empty 2D + // array (shape {0, 0}), stored distinct from NULL. + if values == nil { + col.addNull() + return s + } if len(values) == 0 { col.addDoubleArray(2, []int32{0, 0}, nil) return s @@ -863,6 +876,12 @@ func (s *qwpLineSender) Float64Array3DColumn(name string, values [][][]float64) return s } + // A nil slice is a NULL array; a non-nil empty slice is an empty 3D + // array (shape {0, 0, 0}), stored distinct from NULL. + if values == nil { + col.addNull() + return s + } if len(values) == 0 { col.addDoubleArray(3, []int32{0, 0, 0}, nil) return s @@ -903,13 +922,23 @@ func (s *qwpLineSender) Float64ArrayNDColumn(name string, values *NdArray[float6 s.lastErr = fmt.Errorf("qwp: Float64ArrayNDColumn() called without Table()") return s } - if values == nil { - return s - } if err := qwpValidateColumnName(name, s.fileNameLimit); err != nil { s.lastErr = err return s } + // A nil NdArray is a NULL array, consistent with the typed 1D/2D/3D + // setters: ensure the column exists and mark this row NULL via the + // null bitmap. A non-nil NdArray with a zero-length dimension is a + // distinct, non-null empty array and flows through the normal path. + if values == nil { + col, err := s.currentTable.getOrCreateColumn(name, qwpTypeDoubleArray, true) + if err != nil { + s.lastErr = err + return s + } + col.addNull() + return s + } shape := values.Shape() shapeInt := make([]int, len(shape)) for i, d := range shape { @@ -1535,6 +1564,13 @@ func (s *qwpLineSender) Int64Array1DColumn(name string, values []int64) QwpSende s.lastErr = err return s } + // A nil slice is a NULL array (carried in the null bitmap); a non-nil + // empty slice is a zero-length 1D array (shape {0}), which QuestDB + // stores as a distinct, non-null value. + if values == nil { + col.addNull() + return s + } col.addLongArray(1, []int32{int32(len(values))}, values) return s } @@ -1557,6 +1593,12 @@ func (s *qwpLineSender) Int64Array2DColumn(name string, values [][]int64) QwpSen return s } + // A nil slice is a NULL array; a non-nil empty slice is an empty 2D + // array (shape {0, 0}), stored distinct from NULL. + if values == nil { + col.addNull() + return s + } if len(values) == 0 { col.addLongArray(2, []int32{0, 0}, nil) return s @@ -1597,6 +1639,12 @@ func (s *qwpLineSender) Int64Array3DColumn(name string, values [][][]int64) QwpS return s } + // A nil slice is a NULL array; a non-nil empty slice is an empty 3D + // array (shape {0, 0, 0}), stored distinct from NULL. + if values == nil { + col.addNull() + return s + } if len(values) == 0 { col.addLongArray(3, []int32{0, 0, 0}, nil) return s diff --git a/qwp_sender_test.go b/qwp_sender_test.go index d8e5ebd..70aeb7d 100644 --- a/qwp_sender_test.go +++ b/qwp_sender_test.go @@ -1016,6 +1016,51 @@ func TestQwpSenderFloat64ArrayEmpty(t *testing.T) { } } +// TestQwpSenderArrayNilIsNullEmptyIsEmpty verifies the typed array +// setters map a nil slice to a NULL array (null bitmap, no inline data) +// and a non-nil empty slice to a distinct, non-null empty array (inline +// nDims + shape header, zero elements). QuestDB treats these as +// different values, so the wire encoding must too. +func TestQwpSenderArrayNilIsNullEmptyIsEmpty(t *testing.T) { + srv := newQwpTestServer(t) + defer srv.Close() + s := newQwpSenderForTest(t, srv.URL) + defer s.Close(context.Background()) + + s.Table("t") + s.Float64Array1DColumn("dnull", nil) + s.Float64Array1DColumn("dempty", []float64{}) + s.Int64Array1DColumn("lnull", nil) + s.Int64Array1DColumn("lempty", []int64{}) + if s.lastErr != nil { + t.Fatalf("unexpected lastErr: %v", s.lastErr) + } + + // nDims byte (1) + one int32 shape dimension (4) = 5 inline bytes for + // an empty 1D array; a NULL writes no inline array data. + const emptyArrayInlineLen = 5 + check := func(name string, wantNullCount, wantArrayDataLen int) { + idx, ok := s.currentTable.columnIndex[name] + if !ok { + t.Fatalf("column %q not found", name) + } + col := s.currentTable.columns[idx] + if col.rowCount != 1 { + t.Fatalf("%s: rowCount = %d, want 1", name, col.rowCount) + } + if col.nullCount != wantNullCount { + t.Fatalf("%s: nullCount = %d, want %d", name, col.nullCount, wantNullCount) + } + if len(col.arrayData) != wantArrayDataLen { + t.Fatalf("%s: len(arrayData) = %d, want %d", name, len(col.arrayData), wantArrayDataLen) + } + } + check("dnull", 1, 0) + check("dempty", 0, emptyArrayInlineLen) + check("lnull", 1, 0) + check("lempty", 0, emptyArrayInlineLen) +} + func TestParseDecimalFromString(t *testing.T) { tests := []struct { input string From b3c8d69f3ea65f6bfd5e6c1fd12ed92aba4b5ea2 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 17 Jun 2026 14:42:21 +0200 Subject: [PATCH 2/4] Export jdk.internal.vm to QuestDB in fuzz fixture The qwp-fuzz CI job builds QuestDB master and launches the SNAPSHOT jar as the named module io.questdb. Recent master adds io.questdb.mp.continuation.WorkerContinuation, which reaches the JDK-internal jdk.internal.vm.ContinuationScope. java.base does not export jdk.internal.vm to io.questdb by default, so every worker thread dies at class-init with IllegalAccessError / NoClassDefFoundError and the HTTP service never comes up. Pass --add-exports=java.base/jdk.internal.vm=io.questdb, the flag QuestDB's own launcher (questdb.sh, docker-entrypoint.sh) and the canonical c-questdb-client fixture.py both use. The fixture's launch args were ported from fixture.py before WorkerContinuation existed, so they predate this requirement. Co-Authored-By: Claude Opus 4.8 (1M context) --- qwp_fuzz_fixture_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/qwp_fuzz_fixture_test.go b/qwp_fuzz_fixture_test.go index 66db1a0..da99e26 100644 --- a/qwp_fuzz_fixture_test.go +++ b/qwp_fuzz_fixture_test.go @@ -475,6 +475,13 @@ func (s *qwpFuzzServer) start() error { "-Dnoebug", "-XX:+UnlockExperimentalVMOptions", "-XX:+AlwaysPreTouch", + // QuestDB's worker pools reach jdk.internal.vm.ContinuationScope + // through io.questdb.mp.continuation.WorkerContinuation. Launched as + // the named module io.questdb (-m below), the server needs java.base + // to export jdk.internal.vm to it; without this flag every worker + // thread dies with IllegalAccessError and the HTTP service never + // comes up. QuestDB's own launcher passes it. + "--add-exports=java.base/jdk.internal.vm=io.questdb", "-p", s.jarPath, "-m", "io.questdb/io.questdb.ServerMain", "-d", s.dataDir, From f96cf037312f6d0ab9de7953be8881be85f1df4a Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 17 Jun 2026 15:55:34 +0200 Subject: [PATCH 3/4] Cover long-array empty/NULL; document nil arrays The empty-vs-NULL array fix landed with decode coverage only for DOUBLE_ARRAY and with the nil/empty contract recorded only in implementation comments. This fills both gaps. - TestQwpDecoderRoundTripEmptyVsNullArray now carries a second LONG_ARRAY column alongside the existing DOUBLE_ARRAY one, decoding the same regular/empty-1D/NULL/empty-2D shapes via Int64Array. The double-array column precedes the long one in the frame, so the long column decoding correctly also pins that the empty and NULL rows of the first column consumed exactly their wire bytes (inter-column alignment), since parseArray reads columns sequentially with no length prefix. - TestQwpSenderArrayNilIsNullEmptyIsEmpty now exercises Float64ArrayNDColumn(nil), pinning that a nil NdArray creates the column and marks the row NULL instead of being silently dropped. - The LineSender Float64Array{1,2,3}D/ND and QwpSender Int64Array{1,2,3}D doc comments now state the nil=NULL versus non-nil-empty=empty-array contract. No production logic changes; the doc comments are the only non-test edits. Co-Authored-By: Claude Opus 4.8 (1M context) --- qwp_query_decoder_test.go | 85 ++++++++++++++++++++++++++++++++++----- qwp_sender.go | 9 +++++ qwp_sender_test.go | 6 ++- sender.go | 12 ++++++ 4 files changed, 102 insertions(+), 10 deletions(-) diff --git a/qwp_query_decoder_test.go b/qwp_query_decoder_test.go index 8dd6b18..4369f94 100644 --- a/qwp_query_decoder_test.go +++ b/qwp_query_decoder_test.go @@ -890,29 +890,45 @@ func TestQwpDecoderRoundTripFloat64Array(t *testing.T) { // (cardinality 0) of a given shape as a value distinct from a NULL // array: the encoder writes an empty array inline (nDims >= 1 with a // 0-length dimension) and a NULL via the null bitmap, and the decoder -// must round-trip both without conflating them. Before the -// per-dimension guard in parseArray was relaxed from "dl >= 1" to -// "dl >= 0", decoding the empty-array rows failed outright. +// round-trips both without conflating them. A 0-length dimension is the +// boundary the per-dimension guard in parseArray must admit. +// +// Two columns carry the same four row shapes: "a" is a DOUBLE_ARRAY and +// "b" a LONG_ARRAY, exercising the shared parseArray path for both +// element types. "a" precedes "b" in the frame, so "b" decoding +// correctly also pins that the empty and NULL rows of "a" consumed +// exactly their wire bytes (inter-column alignment). func TestQwpDecoderRoundTripEmptyVsNullArray(t *testing.T) { tb := newQwpTableBuffer("t") - mk := func() *qwpColumnBuffer { + mkDouble := func() *qwpColumnBuffer { col, err := tb.getOrCreateColumn("a", qwpTypeDoubleArray, true) if err != nil { - t.Fatalf("getOrCreateColumn: %v", err) + t.Fatalf("getOrCreateColumn(a): %v", err) + } + return col + } + mkLong := func() *qwpColumnBuffer { + col, err := tb.getOrCreateColumn("b", qwpTypeLongArray, true) + if err != nil { + t.Fatalf("getOrCreateColumn(b): %v", err) } return col } // Row 0: regular 1x3 array. - mk().addDoubleArray(1, []int32{3}, []float64{1, 2, 3}) + mkDouble().addDoubleArray(1, []int32{3}, []float64{1, 2, 3}) + mkLong().addLongArray(1, []int32{3}, []int64{1, 2, 3}) tb.commitRow() // Row 1: empty 1D array, shape {0}. - mk().addDoubleArray(1, []int32{0}, nil) + mkDouble().addDoubleArray(1, []int32{0}, nil) + mkLong().addLongArray(1, []int32{0}, nil) tb.commitRow() // Row 2: NULL array. - mk().addNull() + mkDouble().addNull() + mkLong().addNull() tb.commitRow() // Row 3: empty 2D array, shape {2, 0} — still cardinality 0. - mk().addDoubleArray(2, []int32{2, 0}, nil) + mkDouble().addDoubleArray(2, []int32{2, 0}, nil) + mkLong().addLongArray(2, []int32{2, 0}, nil) tb.commitRow() var enc qwpEncoder @@ -972,6 +988,57 @@ func TestQwpDecoderRoundTripEmptyVsNullArray(t *testing.T) { if got := batch.Float64Array(0, 3); got == nil || len(got) != 0 { t.Fatalf("row 3 Float64Array = %v (nil=%t), want non-nil empty", got, got == nil) } + + // Column 1 = "b" (LONG_ARRAY): same four shapes, read via Int64Array. + // Row 0: regular, non-null, [1 2 3]. + if batch.IsNull(1, 0) { + t.Fatal("col b row 0 (regular array) must NOT be null") + } + gotL0, wantL0 := batch.Int64Array(1, 0), []int64{1, 2, 3} + if len(gotL0) != len(wantL0) { + t.Fatalf("col b row 0 len = %d, want %d", len(gotL0), len(wantL0)) + } + for i := range wantL0 { + if gotL0[i] != wantL0[i] { + t.Fatalf("col b row 0[%d] = %v, want %v", i, gotL0[i], wantL0[i]) + } + } + + // Row 1: empty 1D — non-null, nDims 1, dim 0 == 0, non-nil empty slice. + if batch.IsNull(1, 1) { + t.Fatal("col b row 1 (empty 1D array) must NOT be null") + } + if nd := batch.ArrayNDims(1, 1); nd != 1 { + t.Fatalf("col b row 1 ArrayNDims = %d, want 1", nd) + } + if d0 := batch.ArrayDim(1, 1, 0); d0 != 0 { + t.Fatalf("col b row 1 ArrayDim(0) = %d, want 0", d0) + } + if got := batch.Int64Array(1, 1); got == nil || len(got) != 0 { + t.Fatalf("col b row 1 Int64Array = %v (nil=%t), want non-nil empty", got, got == nil) + } + + // Row 2: NULL — IsNull true, Int64Array nil. + if !batch.IsNull(1, 2) { + t.Fatal("col b row 2 (NULL array) must be null") + } + if got := batch.Int64Array(1, 2); got != nil { + t.Fatalf("col b row 2 Int64Array = %v, want nil (NULL)", got) + } + + // Row 3: empty 2D {2, 0} — non-null, nDims 2, cardinality 0. + if batch.IsNull(1, 3) { + t.Fatal("col b row 3 (empty 2D array) must NOT be null") + } + if nd := batch.ArrayNDims(1, 3); nd != 2 { + t.Fatalf("col b row 3 ArrayNDims = %d, want 2", nd) + } + if d0, d1 := batch.ArrayDim(1, 3, 0), batch.ArrayDim(1, 3, 1); d0 != 2 || d1 != 0 { + t.Fatalf("col b row 3 ArrayDim = %dx%d, want 2x0", d0, d1) + } + if got := batch.Int64Array(1, 3); got == nil || len(got) != 0 { + t.Fatalf("col b row 3 Int64Array = %v (nil=%t), want non-nil empty", got, got == nil) + } } func TestQwpDecoderRoundTripSymbolDelta(t *testing.T) { diff --git a/qwp_sender.go b/qwp_sender.go index 4e43576..1f73cb8 100644 --- a/qwp_sender.go +++ b/qwp_sender.go @@ -70,12 +70,21 @@ type QwpSender interface { GeohashColumn(name string, hash uint64, precision int) QwpSender // Int64Array1DColumn adds a 1-dimensional LONG array column. + // + // A nil values slice yields a NULL array; a non-nil empty slice yields + // a distinct, non-null empty array (cardinality 0). Int64Array1DColumn(name string, values []int64) QwpSender // Int64Array2DColumn adds a 2-dimensional LONG array column. + // + // A nil values slice yields a NULL array; a non-nil empty slice yields + // a distinct, non-null empty array (cardinality 0). Int64Array2DColumn(name string, values [][]int64) QwpSender // Int64Array3DColumn adds a 3-dimensional LONG array column. + // + // A nil values slice yields a NULL array; a non-nil empty slice yields + // a distinct, non-null empty array (cardinality 0). Int64Array3DColumn(name string, values [][][]int64) QwpSender // Decimal64Column adds a DECIMAL64 column value (8 bytes on the wire, diff --git a/qwp_sender_test.go b/qwp_sender_test.go index 70aeb7d..46b9079 100644 --- a/qwp_sender_test.go +++ b/qwp_sender_test.go @@ -1020,7 +1020,9 @@ func TestQwpSenderFloat64ArrayEmpty(t *testing.T) { // setters map a nil slice to a NULL array (null bitmap, no inline data) // and a non-nil empty slice to a distinct, non-null empty array (inline // nDims + shape header, zero elements). QuestDB treats these as -// different values, so the wire encoding must too. +// different values, so the wire encoding must too. Float64ArrayNDColumn +// is covered too: a nil NdArray creates the column and marks the row +// NULL (rather than being silently dropped). func TestQwpSenderArrayNilIsNullEmptyIsEmpty(t *testing.T) { srv := newQwpTestServer(t) defer srv.Close() @@ -1032,6 +1034,7 @@ func TestQwpSenderArrayNilIsNullEmptyIsEmpty(t *testing.T) { s.Float64Array1DColumn("dempty", []float64{}) s.Int64Array1DColumn("lnull", nil) s.Int64Array1DColumn("lempty", []int64{}) + s.Float64ArrayNDColumn("ndnull", nil) if s.lastErr != nil { t.Fatalf("unexpected lastErr: %v", s.lastErr) } @@ -1059,6 +1062,7 @@ func TestQwpSenderArrayNilIsNullEmptyIsEmpty(t *testing.T) { check("dempty", 0, emptyArrayInlineLen) check("lnull", 1, 0) check("lempty", 0, emptyArrayInlineLen) + check("ndnull", 1, 0) } func TestParseDecimalFromString(t *testing.T) { diff --git a/sender.go b/sender.go index cd8dc46..466e002 100644 --- a/sender.go +++ b/sender.go @@ -152,6 +152,9 @@ type LineSender interface { // Float64Array1DColumn adds an array of 64-bit floats (double array) to the ILP message. // + // A nil values slice yields a NULL array; a non-nil empty slice yields + // a distinct, non-null empty array (cardinality 0). + // // Column name cannot contain any of the following characters: // '\n', '\r', '?', '.', ',', "', '"', '\', '/', ':', ')', '(', '+', // '-', '*' '%%', '~', or a non-printable char. @@ -159,6 +162,9 @@ type LineSender interface { // Float64Array2DColumn adds a 2D array of 64-bit floats (double 2D array) to the ILP message. // + // A nil values slice yields a NULL array; a non-nil empty slice yields + // a distinct, non-null empty array (cardinality 0). + // // The values parameter must have a regular (rectangular) shape - all rows must have // exactly the same length. If the array has irregular shape, this method returns an error. // @@ -175,6 +181,9 @@ type LineSender interface { // Float64Array3DColumn adds a 3D array of 64-bit floats (double 3D array) to the ILP message. // + // A nil values slice yields a NULL array; a non-nil empty slice yields + // a distinct, non-null empty array (cardinality 0). + // // The values parameter must have a regular (cuboid) shape - all dimensions must have // consistent sizes throughout. If the array has irregular shape, this method returns an error. // @@ -197,6 +206,9 @@ type LineSender interface { // Float64ArrayNDColumn adds an n-dimensional array of 64-bit floats (double n-D array) to the ILP message. // + // A nil value yields a NULL array; a non-nil array with a zero-length + // dimension yields a distinct, non-null empty array (cardinality 0). + // // Example usage: // // Create a 2x3x4 array // arr, _ := questdb.NewNDArray[float64](2, 3, 4) From 97284d3a7c3ab83f6e179e97a605f51781066394 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 17 Jun 2026 16:00:28 +0200 Subject: [PATCH 4/4] Fix stale empty-array comments in tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The empty-vs-NULL array fix left three test comments describing the old decoder behavior, where a 0-length dimension was rejected. They are now wrong or misleading; this updates them to present-tense facts. - qwp_cursor_bounds_check_fuzz_test.go: the array seed comment no longer claims the decoder rejects a 0-length dim. It now explains the real reason for 1-3 elements — the seed carries element-data bytes for the corruption and truncation passes to mutate, which still reach the 0-length-dim form. - buildArrayHardeningFrame: the padding comment no longer asserts the decoder always rejects before reading element bytes (untrue for the flipped H27b case). It now covers all callers: each shape is either rejected at the shape/nDims check or accepted as a 0-element array, so no element bytes are consumed. - TestQwpIntegrationEmptyAndNullArray: dropped the changelog-style framing for a present-tense property — an empty array reads back as a non-null, zero-element row distinct from NULL. Comment-only; no test logic changes. Co-Authored-By: Claude Opus 4.8 (1M context) --- qwp_cursor_bounds_check_fuzz_test.go | 6 +++--- qwp_query_decoder_test.go | 9 +++++---- qwp_query_integration_test.go | 5 ++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/qwp_cursor_bounds_check_fuzz_test.go b/qwp_cursor_bounds_check_fuzz_test.go index c9fc4d6..c835ac7 100644 --- a/qwp_cursor_bounds_check_fuzz_test.go +++ b/qwp_cursor_bounds_check_fuzz_test.go @@ -135,9 +135,9 @@ func boundsAdderFor(t *testing.T, code qwpTypeCode, r *rand.Rand) func(*qwpColum } case qwpTypeDoubleArray: return func(c *qwpColumnBuffer, r *rand.Rand) { - // 1-3 elements: the Go decoder rejects a 0-length dim - // ("ARRAY dim 0 must be >= 1"), so a valid seed needs >= 1; - // the corruption/truncation passes still explore dim 0. + // 1-3 elements so the seed carries real element-data bytes; + // the corruption and truncation passes mutate those (and the + // shape) and still explore the 0-length-dim form. n := 1 + r.Intn(3) flat := make([]float64, n) for i := range flat { diff --git a/qwp_query_decoder_test.go b/qwp_query_decoder_test.go index 4369f94..19b8602 100644 --- a/qwp_query_decoder_test.go +++ b/qwp_query_decoder_test.go @@ -1982,10 +1982,11 @@ func buildArrayHardeningFrame(t *testing.T, nDims int, shape []int32) []byte { for _, d := range shape { _ = binary.Write(&buf, binary.LittleEndian, d) } - // The decoder rejects on the shape/nDims check before reading any - // element bytes, so we don't need to append them for those paths. - // Append zero padding just to avoid a truncated-frame error - // masking the real one. + // Each caller's shape is either rejected at the shape/nDims check or + // accepted as a 0-element (empty) array, so the decoder consumes no + // element bytes. The 8 trailing bytes are slack so a short frame + // can't raise a truncated-frame error that masks the behavior under + // test. buf.Write(make([]byte, 8)) out := buf.Bytes() binary.LittleEndian.PutUint32(out[qwpHeaderOffsetPayloadLen:], uint32(len(out)-qwpHeaderSize)) diff --git a/qwp_query_integration_test.go b/qwp_query_integration_test.go index 8756d6d..fc33293 100644 --- a/qwp_query_integration_test.go +++ b/qwp_query_integration_test.go @@ -122,9 +122,8 @@ func TestQwpIntegrationQuerySimpleSelect(t *testing.T) { // empty-vs-NULL distinction end-to-end: the sender's nil->NULL / // empty->empty-array encoding, the server's ingest + storage + egress of // an empty array (emitted inline with a 0-length dimension), and the -// decoder's acceptance of that form. Before parseArray was relaxed from -// dl>=1 to dl>=0, the egress decode of the empty-array row failed with -// "ARRAY dim 0 must be >= 1: 0". +// decoder's acceptance of that form, which reads back as a non-null, +// zero-element row distinct from NULL. func TestQwpIntegrationEmptyAndNullArray(t *testing.T) { const tableName = "qwp_integ_empty_null_array" qwpEnsureServer(t)