fix(qwp): empty arrays are distinct from NULL in the columnar codec

qwp_cursor_bounds_check_fuzz_test.go

@@ -135,9 +135,9 @@ func boundsAdderFor(t *testing.T, code qwpTypeCode, r *rand.Rand) func(*qwpColum
 		}
 	case qwpTypeDoubleArray:
 		return func(c *qwpColumnBuffer, r *rand.Rand) {
-			// 1-3 elements: the Go decoder rejects a 0-length dim
-			// ("ARRAY dim 0 must be >= 1"), so a valid seed needs >= 1;
-			// the corruption/truncation passes still explore dim 0.
+			// 1-3 elements so the seed carries real element-data bytes;
+			// the corruption and truncation passes mutate those (and the
+			// shape) and still explore the 0-length-dim form.
 			n := 1 + r.Intn(3)
 			flat := make([]float64, n)
 			for i := range flat {

qwp_fuzz_fixture_test.go

@@ -475,6 +475,13 @@ func (s *qwpFuzzServer) start() error {
 		"-Dnoebug",
 		"-XX:+UnlockExperimentalVMOptions",
 		"-XX:+AlwaysPreTouch",
+		// QuestDB's worker pools reach jdk.internal.vm.ContinuationScope
+		// through io.questdb.mp.continuation.WorkerContinuation. Launched as
+		// the named module io.questdb (-m below), the server needs java.base
+		// to export jdk.internal.vm to it; without this flag every worker
+		// thread dies with IllegalAccessError and the HTTP service never
+		// comes up. QuestDB's own launcher passes it.
+		"--add-exports=java.base/jdk.internal.vm=io.questdb",
 		"-p", s.jarPath,
 		"-m", "io.questdb/io.questdb.ServerMain",
 		"-d", s.dataDir,

qwp_query_decoder.go

@@ -889,9 +889,12 @@ func (d *qwpQueryDecoder) parseGeohash(l *qwpColumnLayout) error {
 // region of the payload so accessors can address elements by
 // (row-start + offset).
 //
-// The server encodes a NULL array via the null bitmap, never inline,
-// so a non-null row must carry nDims >= 1. An inline nDims of 0 is
-// rejected as a malformed frame.
+// A NULL array is signaled by the null bitmap, never inline, so a
+// non-null row must carry nDims >= 1; an inline nDims of 0 is rejected
+// as a malformed frame. Individual dimensions may be 0, though:
+// QuestDB stores empty arrays (cardinality 0) of various shapes as
+// distinct, non-null values, so a 0-length dimension is valid and
+// yields a 0-element row.
 func (d *qwpQueryDecoder) parseArray(l *qwpColumnLayout, rowCount int) error {
 	base := d.br.pos
 	if cap(l.arrayRowStart) < rowCount {
@@ -928,14 +931,18 @@ func (d *qwpQueryDecoder) parseArray(l *qwpColumnLayout, rowCount int) error {
 		elements := int64(1)
 		for dim := 0; dim < nDims; dim++ {
 			dl := int32(binary.LittleEndian.Uint32(shapeBytes[dim*4:]))
-			// Require dl >= 1 in every dimension. A dl of 0 would zero out
-			// elements and short-circuit the qwpMaxArrayElements cap for
-			// the remaining dimensions, letting them hold arbitrary values
-			// unchecked; the encoder never emits dl == 0. Matches
-			// QwpResultBatchDecoder.java.
-			if dl < 1 {
+			// A 0-length dimension is a valid empty array (cardinality 0),
+			// distinct from a NULL array (which the null bitmap carries).
+			// Reject only a negative length, and bound each dimension
+			// independently: a single 0 collapses the element-count product
+			// to 0, so without a per-dimension cap a 0 in one dimension
+			// would let another hold an arbitrary value the product check
+			// below could no longer catch. Matches the server's
+			// QwpArrayColumnCursor.
+			if dl < 0 || int64(dl) > qwpMaxArrayElements {
 				return newQwpDecodeError(fmt.Sprintf(
-					"ARRAY dim %d must be >= 1: %d", dim, dl))
+					"ARRAY dim %d out of range [0, %d]: %d",
+					dim, qwpMaxArrayElements, dl))
 			}
 			elements *= int64(dl)
 			if elements > qwpMaxArrayElements {

qwp_query_decoder_test.go

@@ -885,6 +885,162 @@ func TestQwpDecoderRoundTripFloat64Array(t *testing.T) {
 	}
 }
 
+// TestQwpDecoderRoundTripEmptyVsNullArray pins the empty-vs-NULL array
+// distinction across the wire. QuestDB stores a non-null empty array
+// (cardinality 0) of a given shape as a value distinct from a NULL
+// array: the encoder writes an empty array inline (nDims >= 1 with a
+// 0-length dimension) and a NULL via the null bitmap, and the decoder
+// round-trips both without conflating them. A 0-length dimension is the
+// boundary the per-dimension guard in parseArray must admit.
+//
+// Two columns carry the same four row shapes: "a" is a DOUBLE_ARRAY and
+// "b" a LONG_ARRAY, exercising the shared parseArray path for both
+// element types. "a" precedes "b" in the frame, so "b" decoding
+// correctly also pins that the empty and NULL rows of "a" consumed
+// exactly their wire bytes (inter-column alignment).
+func TestQwpDecoderRoundTripEmptyVsNullArray(t *testing.T) {
+	tb := newQwpTableBuffer("t")
+	mkDouble := func() *qwpColumnBuffer {
+		col, err := tb.getOrCreateColumn("a", qwpTypeDoubleArray, true)
+		if err != nil {
+			t.Fatalf("getOrCreateColumn(a): %v", err)
+		}
+		return col
+	}
+	mkLong := func() *qwpColumnBuffer {
+		col, err := tb.getOrCreateColumn("b", qwpTypeLongArray, true)
+		if err != nil {
+			t.Fatalf("getOrCreateColumn(b): %v", err)
+		}
+		return col
+	}
+	// Row 0: regular 1x3 array.
+	mkDouble().addDoubleArray(1, []int32{3}, []float64{1, 2, 3})
+	mkLong().addLongArray(1, []int32{3}, []int64{1, 2, 3})
+	tb.commitRow()
+	// Row 1: empty 1D array, shape {0}.
+	mkDouble().addDoubleArray(1, []int32{0}, nil)
+	mkLong().addLongArray(1, []int32{0}, nil)
+	tb.commitRow()
+	// Row 2: NULL array.
+	mkDouble().addNull()
+	mkLong().addNull()
+	tb.commitRow()
+	// Row 3: empty 2D array, shape {2, 0} — still cardinality 0.
+	mkDouble().addDoubleArray(2, []int32{2, 0}, nil)
+	mkLong().addLongArray(2, []int32{2, 0}, nil)
+	tb.commitRow()
+
+	var enc qwpEncoder
+	frame := wrapAsResultBatch(enc.encodeTable(tb), 1, 0)
+	dec := newTestQueryDecoder()
+	var batch QwpColumnBatch
+	if err := dec.decode(frame, &batch); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+
+	// Row 0: regular, non-null, [1 2 3].
+	if batch.IsNull(0, 0) {
+		t.Fatal("row 0 (regular array) must NOT be null")
+	}
+	got0, want0 := batch.Float64Array(0, 0), []float64{1, 2, 3}
+	if len(got0) != len(want0) {
+		t.Fatalf("row 0 len = %d, want %d", len(got0), len(want0))
+	}
+	for i := range want0 {
+		if got0[i] != want0[i] {
+			t.Fatalf("row 0[%d] = %v, want %v", i, got0[i], want0[i])
+		}
+	}
+
+	// Row 1: empty 1D — non-null, nDims 1, dim 0 == 0, non-nil empty slice.
+	if batch.IsNull(0, 1) {
+		t.Fatal("row 1 (empty 1D array) must NOT be null")
+	}
+	if nd := batch.ArrayNDims(0, 1); nd != 1 {
+		t.Fatalf("row 1 ArrayNDims = %d, want 1", nd)
+	}
+	if d0 := batch.ArrayDim(0, 1, 0); d0 != 0 {
+		t.Fatalf("row 1 ArrayDim(0) = %d, want 0", d0)
+	}
+	if got := batch.Float64Array(0, 1); got == nil || len(got) != 0 {
+		t.Fatalf("row 1 Float64Array = %v (nil=%t), want non-nil empty", got, got == nil)
+	}
+
+	// Row 2: NULL — IsNull true, Float64Array nil.
+	if !batch.IsNull(0, 2) {
+		t.Fatal("row 2 (NULL array) must be null")
+	}
+	if got := batch.Float64Array(0, 2); got != nil {
+		t.Fatalf("row 2 Float64Array = %v, want nil (NULL)", got)
+	}
+
+	// Row 3: empty 2D {2, 0} — non-null, nDims 2, cardinality 0.
+	if batch.IsNull(0, 3) {
+		t.Fatal("row 3 (empty 2D array) must NOT be null")
+	}
+	if nd := batch.ArrayNDims(0, 3); nd != 2 {
+		t.Fatalf("row 3 ArrayNDims = %d, want 2", nd)
+	}
+	if d0, d1 := batch.ArrayDim(0, 3, 0), batch.ArrayDim(0, 3, 1); d0 != 2 || d1 != 0 {
+		t.Fatalf("row 3 ArrayDim = %dx%d, want 2x0", d0, d1)
+	}
+	if got := batch.Float64Array(0, 3); got == nil || len(got) != 0 {
+		t.Fatalf("row 3 Float64Array = %v (nil=%t), want non-nil empty", got, got == nil)
+	}
+
+	// Column 1 = "b" (LONG_ARRAY): same four shapes, read via Int64Array.
+	// Row 0: regular, non-null, [1 2 3].
+	if batch.IsNull(1, 0) {
+		t.Fatal("col b row 0 (regular array) must NOT be null")
+	}
+	gotL0, wantL0 := batch.Int64Array(1, 0), []int64{1, 2, 3}
+	if len(gotL0) != len(wantL0) {
+		t.Fatalf("col b row 0 len = %d, want %d", len(gotL0), len(wantL0))
+	}
+	for i := range wantL0 {
+		if gotL0[i] != wantL0[i] {
+			t.Fatalf("col b row 0[%d] = %v, want %v", i, gotL0[i], wantL0[i])
+		}
+	}
+
+	// Row 1: empty 1D — non-null, nDims 1, dim 0 == 0, non-nil empty slice.
+	if batch.IsNull(1, 1) {
+		t.Fatal("col b row 1 (empty 1D array) must NOT be null")
+	}
+	if nd := batch.ArrayNDims(1, 1); nd != 1 {
+		t.Fatalf("col b row 1 ArrayNDims = %d, want 1", nd)
+	}
+	if d0 := batch.ArrayDim(1, 1, 0); d0 != 0 {
+		t.Fatalf("col b row 1 ArrayDim(0) = %d, want 0", d0)
+	}
+	if got := batch.Int64Array(1, 1); got == nil || len(got) != 0 {
+		t.Fatalf("col b row 1 Int64Array = %v (nil=%t), want non-nil empty", got, got == nil)
+	}
+
+	// Row 2: NULL — IsNull true, Int64Array nil.
+	if !batch.IsNull(1, 2) {
+		t.Fatal("col b row 2 (NULL array) must be null")
+	}
+	if got := batch.Int64Array(1, 2); got != nil {
+		t.Fatalf("col b row 2 Int64Array = %v, want nil (NULL)", got)
+	}
+
+	// Row 3: empty 2D {2, 0} — non-null, nDims 2, cardinality 0.
+	if batch.IsNull(1, 3) {
+		t.Fatal("col b row 3 (empty 2D array) must NOT be null")
+	}
+	if nd := batch.ArrayNDims(1, 3); nd != 2 {
+		t.Fatalf("col b row 3 ArrayNDims = %d, want 2", nd)
+	}
+	if d0, d1 := batch.ArrayDim(1, 3, 0), batch.ArrayDim(1, 3, 1); d0 != 2 || d1 != 0 {
+		t.Fatalf("col b row 3 ArrayDim = %dx%d, want 2x0", d0, d1)
+	}
+	if got := batch.Int64Array(1, 3); got == nil || len(got) != 0 {
+		t.Fatalf("col b row 3 Int64Array = %v (nil=%t), want non-nil empty", got, got == nil)
+	}
+}
+
 func TestQwpDecoderRoundTripSymbolDelta(t *testing.T) {
 	// Batch 1 introduces three symbols; Batch 2 adds one more via a
 	// delta section. The decoder's connection-scoped dict must grow
@@ -1681,16 +1837,30 @@ func TestQwpDecoderHardening(t *testing.T) {
 		assertDecodeErrContains(t, err, "ARRAY dim")
 	})
 
-	t.Run("H27b_ArrayZeroDim", func(t *testing.T) {
-		// shape[0] = 0. A zero-extent dimension would zero out the
-		// element count and short-circuit the qwpMaxArrayElements cap
-		// for any remaining dimensions. The encoder never emits dl == 0;
-		// the decoder must reject it (matches Java's dl < 1 guard).
+	t.Run("H27b_ArrayZeroDimIsEmptyArray", func(t *testing.T) {
+		// shape[0] = 0 is a valid empty 1D array (cardinality 0), distinct
+		// from a NULL array (which the null bitmap carries). The decoder
+		// must accept it and report a non-null, zero-element row. The
+		// per-dimension guard still collapses the element-count product to
+		// 0, so the trailing padding bytes are simply left unconsumed.
 		frame := buildArrayHardeningFrame(t, 1, []int32{0})
 		dec := newTestQueryDecoder()
 		var b QwpColumnBatch
-		err := dec.decode(frame, &b)
-		assertDecodeErrContains(t, err, "ARRAY dim")
+		if err := dec.decode(frame, &b); err != nil {
+			t.Fatalf("decode: %v", err)
+		}
+		if b.IsNull(0, 0) {
+			t.Fatal("empty array must NOT be null")
+		}
+		if nd := b.ArrayNDims(0, 0); nd != 1 {
+			t.Fatalf("ArrayNDims = %d, want 1", nd)
+		}
+		if d0 := b.ArrayDim(0, 0, 0); d0 != 0 {
+			t.Fatalf("ArrayDim(0) = %d, want 0", d0)
+		}
+		if got := b.Float64Array(0, 0); got == nil || len(got) != 0 {
+			t.Fatalf("Float64Array = %v (nil=%t), want non-nil empty", got, got == nil)
+		}
 	})
 
 	t.Run("H28_ArrayElementCountExceeded", func(t *testing.T) {
@@ -1812,10 +1982,11 @@ func buildArrayHardeningFrame(t *testing.T, nDims int, shape []int32) []byte {
 	for _, d := range shape {
 		_ = binary.Write(&buf, binary.LittleEndian, d)
 	}
-	// The decoder rejects on the shape/nDims check before reading any
-	// element bytes, so we don't need to append them for those paths.
-	// Append zero padding just to avoid a truncated-frame error
-	// masking the real one.
+	// Each caller's shape is either rejected at the shape/nDims check or
+	// accepted as a 0-element (empty) array, so the decoder consumes no
+	// element bytes. The 8 trailing bytes are slack so a short frame
+	// can't raise a truncated-frame error that masks the behavior under
+	// test.
 	buf.Write(make([]byte, 8))
 	out := buf.Bytes()
 	binary.LittleEndian.PutUint32(out[qwpHeaderOffsetPayloadLen:], uint32(len(out)-qwpHeaderSize))

qwp_query_integration_test.go

@@ -116,6 +116,101 @@ func TestQwpIntegrationQuerySimpleSelect(t *testing.T) {
 	}
 }
 
+// TestQwpIntegrationEmptyAndNullArray ingests a regular, an empty, and a
+// NULL DOUBLE[] via the QWP sender, then reads them back via the QWP
+// query client — a full Go round-trip against a real QuestDB. It pins the
+// empty-vs-NULL distinction end-to-end: the sender's nil->NULL /
+// empty->empty-array encoding, the server's ingest + storage + egress of
+// an empty array (emitted inline with a 0-length dimension), and the
+// decoder's acceptance of that form, which reads back as a non-null,
+// zero-element row distinct from NULL.
+func TestQwpIntegrationEmptyAndNullArray(t *testing.T) {
+	const tableName = "qwp_integ_empty_null_array"
+	qwpEnsureServer(t)
+	qwpDropTable(t, tableName)
+	defer qwpDropTable(t, tableName)
+
+	ctx := context.Background()
+	s, err := newQwpLineSender(ctx, "ws://"+qwpTestAddr,
+		qwpTransportOpts{endpointPath: qwpWritePath}, 0, 0, nil)
+	if err != nil {
+		t.Fatalf("newQwpLineSender: %v", err)
+	}
+	base := time.Date(2024, 6, 15, 12, 0, 0, 0, time.UTC)
+	// Row 0 establishes DOUBLE[] (1-D) with a regular array; row 1 is an
+	// empty array; row 2 is NULL. `i` is an order key so the read side does
+	// not depend on WAL scan order.
+	if err := s.Table(tableName).Int64Column("i", 0).
+		Float64Array1DColumn("d", []float64{1, 2}).
+		At(ctx, base); err != nil {
+		t.Fatalf("At row 0: %v", err)
+	}
+	if err := s.Table(tableName).Int64Column("i", 1).
+		Float64Array1DColumn("d", []float64{}).
+		At(ctx, base.Add(time.Second)); err != nil {
+		t.Fatalf("At row 1 (empty): %v", err)
+	}
+	if err := s.Table(tableName).Int64Column("i", 2).
+		Float64Array1DColumn("d", nil).
+		At(ctx, base.Add(2*time.Second)); err != nil {
+		t.Fatalf("At row 2 (nil): %v", err)
+	}
+	if err := s.Flush(ctx); err != nil {
+		t.Fatalf("Flush: %v", err)
+	}
+	_ = s.Close(ctx)
+	qwpWaitForRows(t, tableName, 3)
+
+	c := newTestQueryClient(t)
+	qctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	defer c.Close(qctx)
+
+	type rowResult struct {
+		seen   bool
+		isNull bool
+		nDims  int
+		elems  []float64
+	}
+	got := map[int64]rowResult{}
+	// Column 0 = i (order key), column 1 = d (the array).
+	q := c.Query(qctx, fmt.Sprintf("SELECT i, d FROM '%s'", tableName))
+	defer q.Close()
+	for batch, err := range q.Batches() {
+		if err != nil {
+			t.Fatalf("iter err: %v", err)
+		}
+		for r := 0; r < batch.RowCount(); r++ {
+			idx := batch.Int64(0, r)
+			rr := rowResult{seen: true, isNull: batch.IsNull(1, r)}
+			if !rr.isNull {
+				rr.nDims = batch.ArrayNDims(1, r)
+				rr.elems = batch.Float64Array(1, r)
+			}
+			got[idx] = rr
+		}
+	}
+	if len(got) != 3 {
+		t.Fatalf("got %d distinct rows, want 3: %+v", len(got), got)
+	}
+
+	// Row 0: regular 1-D array [1, 2].
+	if r0 := got[0]; r0.isNull || r0.nDims != 1 || len(r0.elems) != 2 ||
+		r0.elems[0] != 1 || r0.elems[1] != 2 {
+		t.Fatalf("row 0 (regular) = %+v, want non-null 1-D [1 2]", r0)
+	}
+	// Row 1: empty array -> non-null, 1-D, zero elements (distinct from NULL).
+	if r1 := got[1]; r1.isNull {
+		t.Fatalf("row 1 (empty array) must NOT be null: %+v", r1)
+	} else if r1.nDims != 1 || len(r1.elems) != 0 {
+		t.Fatalf("row 1 (empty) nDims=%d len=%d, want 1 and 0", r1.nDims, len(r1.elems))
+	}
+	// Row 2: NULL array.
+	if r2 := got[2]; !r2.isNull {
+		t.Fatalf("row 2 (NULL array) must be null: %+v", r2)
+	}
+}
+
 // TestQwpIntegrationQueryError runs a SELECT against a nonexistent
 // table and verifies the server's QUERY_ERROR surfaces as a
 // *QwpQueryError with a useful message.