diff --git a/docs/sphinx/source/reference/Functions/scalar_functions.rst b/docs/sphinx/source/reference/Functions/scalar_functions.rst index 9da2d248b9..9436a40c27 100644 --- a/docs/sphinx/source/reference/Functions/scalar_functions.rst +++ b/docs/sphinx/source/reference/Functions/scalar_functions.rst @@ -13,4 +13,5 @@ List of functions (by sub-category) scalar_functions/bitmap_bit_position scalar_functions/bitmap_bucket_number scalar_functions/bitmap_bucket_offset + scalar_functions/cardinality scalar_functions/get_versionstamp_incarnation diff --git a/docs/sphinx/source/reference/Functions/scalar_functions/cardinality.rst b/docs/sphinx/source/reference/Functions/scalar_functions/cardinality.rst new file mode 100644 index 0000000000..51c76534b1 --- /dev/null +++ b/docs/sphinx/source/reference/Functions/scalar_functions/cardinality.rst @@ -0,0 +1,33 @@ +=========== +CARDINALITY +=========== + +The CARDINALITY function returns the cardinality of an array. + +Syntax +====== + +.. code-block:: sql + + CARDINALITY( ) + +Parameters +========== + +``expr`` + An array expression. + +Returns +======= + +Given an array value, returns a value of type INTEGER representing the number of elements in the array, or 0 if it is +empty. If the argument is NULL, the result is the NULL value. + +Example +======= + +.. code-block:: sql + + CREATE TABLE t1 (id INTEGER, arr INTEGER ARRAY); + INSERT INTO t1 VALUES (1, [1, 2]); + SELECT CARDINALITY(arr) FROM t1; -- yields 2 diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/FunctionNames.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/FunctionNames.java index cfdcd128ff..09f77060d1 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/FunctionNames.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/FunctionNames.java @@ -74,6 +74,9 @@ public class FunctionNames { public static final String BITMAP_BIT_POSITION = "bitmap_bit_position"; public static final String BITMAP_BUCKET_OFFSET = "bitmap_bucket_offset"; + /* Assorted functions */ + public static final String CARDINALITY = "cardinality"; + private FunctionNames() { } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/CardinalityValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/CardinalityValue.java new file mode 100644 index 0000000000..d8beb11beb --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/CardinalityValue.java @@ -0,0 +1,180 @@ +/* + * CardinalityValue.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2023-2030 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.cascades.values; + +import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.annotation.SpotBugsSuppressWarnings; +import com.apple.foundationdb.record.EvaluationContext; +import com.apple.foundationdb.record.FunctionNames; +import com.apple.foundationdb.record.ObjectPlanHash; +import com.apple.foundationdb.record.PlanDeserializer; +import com.apple.foundationdb.record.PlanHashable; +import com.apple.foundationdb.record.PlanSerializationContext; +import com.apple.foundationdb.record.planprotos.PCardinalityValue; +import com.apple.foundationdb.record.planprotos.PValue; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; +import com.apple.foundationdb.record.query.plan.cascades.AliasMap; +import com.apple.foundationdb.record.query.plan.cascades.BuiltInFunction; +import com.apple.foundationdb.record.query.plan.cascades.SemanticException; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.record.query.plan.cascades.typing.Typed; +import com.apple.foundationdb.record.query.plan.explain.ExplainTokens; +import com.apple.foundationdb.record.query.plan.explain.ExplainTokensWithPrecedence; +import com.google.auto.service.AutoService; +import com.google.common.base.Verify; +import com.google.common.collect.ImmutableList; +import com.google.protobuf.Message; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.List; +import java.util.Objects; +import java.util.function.Supplier; + +/** + * A value representing the {@code CARDINALITY()} function. + */ +@API(API.Status.EXPERIMENTAL) +public class CardinalityValue extends AbstractValue { + private static final ObjectPlanHash BASE_HASH = new ObjectPlanHash("Cardinality-Value"); + + @Nonnull + private final Value childValue; + + public CardinalityValue(@Nonnull final Value childValue) { + SemanticException.check(childValue.getResultType().isArray(), SemanticException.ErrorCode.INCOMPATIBLE_TYPE, "The argument of CARDINALITY() must be an array expression."); + + this.childValue = childValue; + } + + @Nonnull + @Override + public List computeChildren() { + return List.of(childValue); + } + + @Nonnull + @Override + public Value withChildren(final Iterable newChildren) { + final var newChildrenList = ImmutableList.copyOf(newChildren); + Verify.verify(newChildrenList.size() == 1); + return new CardinalityValue(newChildrenList.get(0)); + } + + @Nonnull + @Override + public Type getResultType() { + // Array indexes and sizes are 32-bit integers. + return Type.primitiveType(Type.TypeCode.INT); + } + + @Override + public Object eval(@Nullable final FDBRecordStoreBase store, @Nonnull final EvaluationContext context) { + final Object childResult = childValue.eval(store, context); + if (childResult == null) { + return null; + } + return ((List)childResult).size(); + } + + @Override + public int hashCodeWithoutChildren() { + return PlanHashable.objectsPlanHash(PlanHashable.CURRENT_FOR_CONTINUATION, BASE_HASH); + } + + @Override + public int planHash(@Nonnull final PlanHashMode mode) { + return PlanHashable.objectsPlanHash(mode, BASE_HASH, childValue); + } + + @Nonnull + @Override + public ExplainTokensWithPrecedence explain(@Nonnull final Iterable> explainSuppliers) { + return ExplainTokensWithPrecedence.of(new ExplainTokens().addFunctionCall(FunctionNames.CARDINALITY, + Value.explainFunctionArguments(explainSuppliers))); + } + + @Override + public int hashCode() { + return semanticHashCode(); + } + + @SuppressWarnings("EqualsWhichDoesntCheckParameterClass") + @SpotBugsSuppressWarnings("EQ_UNUSUAL") + @Override + public boolean equals(final Object other) { + return semanticEquals(other, AliasMap.emptyMap()); + } + + @Nonnull + @Override + public PCardinalityValue toProto(@Nonnull final PlanSerializationContext serializationContext) { + return PCardinalityValue.newBuilder() + .setChildValue(childValue.toValueProto(serializationContext)) + .build(); + } + + @Nonnull + @Override + public PValue toValueProto(@Nonnull PlanSerializationContext serializationContext) { + return PValue.newBuilder().setCardinalityValue(toProto(serializationContext)).build(); + } + + @Nonnull + public static CardinalityValue fromProto(@Nonnull final PlanSerializationContext serializationContext, @Nonnull final PCardinalityValue cardinalityValueProto) { + return new CardinalityValue(Value.fromValueProto(serializationContext, Objects.requireNonNull(cardinalityValueProto.getChildValue()))); + } + + /** + * The {@code CARDINALITY()} function. + */ + @AutoService(BuiltInFunction.class) + public static class CardinalityFn extends BuiltInFunction { + public CardinalityFn() { + super(FunctionNames.CARDINALITY, + List.of(Type.any()), (builtInFunction, arguments) -> encapsulateInternal(arguments)); + } + + private static Value encapsulateInternal(@Nonnull final List arguments) { + Verify.verify(arguments.size() == 1); + return new CardinalityValue((Value)arguments.get(0)); + } + } + + /** + * Deserializer. + */ + @AutoService(PlanDeserializer.class) + public static class Deserializer implements PlanDeserializer { + @Nonnull + @Override + public Class getProtoMessageClass() { + return PCardinalityValue.class; + } + + @Nonnull + @Override + public CardinalityValue fromProto(@Nonnull final PlanSerializationContext serializationContext, + @Nonnull final PCardinalityValue cardinalityValueProto) { + return CardinalityValue.fromProto(serializationContext, cardinalityValueProto); + } + } +} diff --git a/fdb-record-layer-core/src/main/proto/record_query_plan.proto b/fdb-record-layer-core/src/main/proto/record_query_plan.proto index fcb2ea2525..d8db62b591 100644 --- a/fdb-record-layer-core/src/main/proto/record_query_plan.proto +++ b/fdb-record-layer-core/src/main/proto/record_query_plan.proto @@ -286,6 +286,7 @@ message PValue { PEuclideanSquareDistanceRowNumberValue euclidean_square_distance_row_number_value = 60; PDotProductDistanceRowNumberValue dot_product_distance_row_number_value = 61; PIncarnationValue incarnation_value = 62; + PCardinalityValue cardinality_value = 63; } } @@ -1374,6 +1375,10 @@ message PIncarnationValue { // No fields needed - this function takes no arguments } +message PCardinalityValue { + optional PValue child_value = 1; +} + message PWindowedValue { repeated PValue partitioning_values = 1; repeated PValue argument_values = 2; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/CardinalityValueTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/CardinalityValueTest.java new file mode 100644 index 0000000000..bbfe3a593c --- /dev/null +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/CardinalityValueTest.java @@ -0,0 +1,63 @@ +/* + * CardinalityValueTest.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2023-2030 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.cascades.values; + +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; + +class CardinalityValueTest { + + private static final Type.Array INT_ARRAY_TYPE = new Type.Array(Type.primitiveType(Type.TypeCode.INT)); + private static final Type.Array STRING_ARRAY_TYPE = new Type.Array(Type.primitiveType(Type.TypeCode.STRING)); + + private static final LiteralValue> INT_ARRAY_1 = + new LiteralValue<>(INT_ARRAY_TYPE, List.of(1, 2, 3)); + private static final LiteralValue> INT_ARRAY_2 = + new LiteralValue<>(INT_ARRAY_TYPE, List.of(4, 5)); + private static final LiteralValue> STRING_ARRAY = + new LiteralValue<>(STRING_ARRAY_TYPE, List.of("a", "b")); + + @Test + void testEqualsAndHashCode() { + final var c1 = new CardinalityValue(INT_ARRAY_1); + final var c2 = new CardinalityValue(INT_ARRAY_1); + final var cDifferentChild = new CardinalityValue(INT_ARRAY_2); + final var cDifferentElementType = new CardinalityValue(STRING_ARRAY); + + // Reflexive + Assertions.assertEquals(c1, c1); + + // Two instances with the same child are equal and have the same hash code + Assertions.assertEquals(c1, c2); + Assertions.assertEquals(c1.hashCode(), c2.hashCode()); + + // Instances with different children are not equal + Assertions.assertNotEquals(cDifferentChild, c1); + Assertions.assertNotEquals(cDifferentElementType, c1); + + // Not equal to null or an unrelated type + Assertions.assertNotEquals(null, c1); + Assertions.assertNotEquals("not a Value", c1); + } +} diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalogImpl.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalogImpl.java index f4f7030ec5..8bc23f0ea0 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalogImpl.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalogImpl.java @@ -156,6 +156,7 @@ private static ImmutableMap BuiltInFunctionCatalog.resolve("array", argumentsCount)) .put("__pick_value", argumentsCount -> BuiltInFunctionCatalog.resolve("pick", argumentsCount)) .put("get_versionstamp_incarnation", argumentsCount -> BuiltInFunctionCatalog.resolve("get_versionstamp_incarnation", argumentsCount)) + .put("cardinality", argumentsCount -> BuiltInFunctionCatalog.resolve("cardinality", argumentsCount)) .build(); } diff --git a/yaml-tests/src/test/java/YamlIntegrationTests.java b/yaml-tests/src/test/java/YamlIntegrationTests.java index 81cfb680a2..200789d9b1 100644 --- a/yaml-tests/src/test/java/YamlIntegrationTests.java +++ b/yaml-tests/src/test/java/YamlIntegrationTests.java @@ -70,6 +70,11 @@ void arrays(YamlTest.Runner runner) throws Exception { runner.runYamsql("arrays.yamsql"); } + @TestTemplate + void arrays_cardinality(YamlTest.Runner runner) throws Exception { + runner.runYamsql("arrays_cardinality.yamsql"); + } + @TestTemplate void arraysOperators(YamlTest.Runner runner) throws Exception { runner.runYamsql("arrays-operators.yamsql"); diff --git a/yaml-tests/src/test/resources/arrays_cardinality.metrics.binpb b/yaml-tests/src/test/resources/arrays_cardinality.metrics.binpb new file mode 100644 index 0000000000..bf4cbb959f --- /dev/null +++ b/yaml-tests/src/test/resources/arrays_cardinality.metrics.binpb @@ -0,0 +1,27 @@ + +P +arrays-cardinality-tests4EXPLAIN SELECT CARDINALITY("int_arr") FROM "tab1_nn" +"( ( 08 @7SCAN([IS tab1_nn]) | MAP (cardinality(_.int_arr) AS _0) +digraph G { + fontname=courier; + rankdir=BT; + splines=line; + 1 [ label=<
Value Computation
MAP (cardinality(q2.int_arr) AS _0)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(INT AS _0)" ]; + 2 [ label=<
Scan
comparisons: [IS tab1_nn]
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(INT AS id, ARRAY(INT) AS int_arr)" ]; + 3 [ label=<
Primary Storage
record types: [tab1_nn, tab1]
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(INT AS id, ARRAY(INT) AS int_arr)" ]; + 3 -> 2 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 2 -> 1 [ label=< q2> label="q2" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; +} +M +arrays-cardinality-tests1EXPLAIN SELECT CARDINALITY("int_arr") FROM "tab1" +ɵ( ȩ( 0!8 @4SCAN([IS tab1]) | MAP (cardinality(_.int_arr) AS _0) +digraph G { + fontname=courier; + rankdir=BT; + splines=line; + 1 [ label=<
Value Computation
MAP (cardinality(q2.int_arr) AS _0)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(INT AS _0)" ]; + 2 [ label=<
Scan
comparisons: [IS tab1]
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(INT AS id, ARRAY(INT) AS int_arr)" ]; + 3 [ label=<
Primary Storage
record types: [dummy, tab1_nn, tab1]
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(INT AS id, ARRAY(INT) AS int_arr)" ]; + 3 -> 2 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 2 -> 1 [ label=< q2> label="q2" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; +} \ No newline at end of file diff --git a/yaml-tests/src/test/resources/arrays_cardinality.metrics.yaml b/yaml-tests/src/test/resources/arrays_cardinality.metrics.yaml new file mode 100644 index 0000000000..f637edd61e --- /dev/null +++ b/yaml-tests/src/test/resources/arrays_cardinality.metrics.yaml @@ -0,0 +1,23 @@ +arrays-cardinality-tests: +- query: EXPLAIN SELECT CARDINALITY("int_arr") FROM "tab1_nn" + ref: arrays_cardinality.yamsql:57 + explain: SCAN([IS tab1_nn]) | MAP (cardinality(_.int_arr) AS _0) + task_count: 146 + task_total_time_ms: 71 + transform_count: 40 + transform_time_ms: 55 + transform_yield_count: 13 + insert_time_ms: 2 + insert_new_count: 12 + insert_reused_count: 2 +- query: EXPLAIN SELECT CARDINALITY("int_arr") FROM "tab1" + ref: arrays_cardinality.yamsql:61 + explain: SCAN([IS tab1]) | MAP (cardinality(_.int_arr) AS _0) + task_count: 146 + task_total_time_ms: 9 + transform_count: 40 + transform_time_ms: 4 + transform_yield_count: 13 + insert_time_ms: 0 + insert_new_count: 12 + insert_reused_count: 2 diff --git a/yaml-tests/src/test/resources/arrays_cardinality.yamsql b/yaml-tests/src/test/resources/arrays_cardinality.yamsql new file mode 100644 index 0000000000..2b0ce993ed --- /dev/null +++ b/yaml-tests/src/test/resources/arrays_cardinality.yamsql @@ -0,0 +1,63 @@ +# +# arrays_cardinality.yamsql +# +# This source file is part of the FoundationDB open source project +# +# Copyright 2023-2030 Apple Inc. and the FoundationDB project authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +options: + supported_version: !current_version +--- +schema_template: + CREATE TABLE "dummy" ("id" INTEGER, PRIMARY KEY ("id")) + CREATE TABLE "tab1_nn" ("id" INTEGER, "int_arr" INTEGER ARRAY NOT NULL, PRIMARY KEY ("id")) + CREATE TABLE "tab1" ("id" INTEGER, "int_arr" INTEGER ARRAY NULL, PRIMARY KEY ("id")) +--- +setup: + steps: + - query: INSERT INTO "dummy" ("id") VALUES (1) + - query: INSERT INTO "tab1_nn" ("id", "int_arr") VALUES (1, []), (2, [1]), (3, [1, 2]) + - query: INSERT INTO "tab1" ("id", "int_arr") VALUES (0, NULL), (1, []), (2, [1]), (3, [1, 2]) +--- +test_block: + name: arrays-cardinality-tests + # Basic tests for CARDINALITY(). + preset: single_repetition_ordered + tests: + - # Incorrect argument type: Passing a non-array (here: INTEGER) raises error 22000. + - query: SELECT CARDINALITY("id") FROM "dummy" + - error: "22000" + - # Incorrect argument type (constant case) + - query: SELECT CARDINALITY(1) FROM "dummy" + - error: "22000" + - # NULL argument array (constant case): NULL gets mapped to NULL. + - query: SELECT CARDINALITY(CAST(NULL AS INTEGER ARRAY)) FROM "tab1" WHERE "id" = 1 + - result: [{!null _}] + - # Non-NULL constant cases using array constructor literals. + - query: SELECT CARDINALITY(CAST([] AS INTEGER ARRAY)), + CARDINALITY(CAST([1] AS INTEGER ARRAY)), + CARDINALITY(CAST([1, 2] AS INTEGER ARRAY)) + FROM "dummy" + - result: [{0, 1, 2}] + - # Not-nullable array column containing arrays of size 0, 1, >1. + - query: SELECT CARDINALITY("int_arr") FROM "tab1_nn" + - explain: "SCAN([IS tab1_nn]) | MAP (cardinality(_.int_arr) AS _0)" + - unorderedResult: [{0}, {1}, {2}] + - # Nullable array column containing arrays of size 0, 1, >1 as well as a NULL array. + - query: SELECT CARDINALITY("int_arr") FROM "tab1" + - explain: "SCAN([IS tab1]) | MAP (cardinality(_.int_arr) AS _0)" + - unorderedResult: [{!null _}, {0}, {1}, {2}] +...