From e930a7a4341bafa5296012e5339e14257d2e5dc3 Mon Sep 17 00:00:00 2001 From: Carlo Curino Date: Tue, 28 Jun 2022 21:43:03 +0200 Subject: [PATCH 01/16] feat: add initial support for DDL and data modification operator (#128) --- proto/substrait/algebra.proto | 242 ++++++++++++++++++++-------------- 1 file changed, 142 insertions(+), 100 deletions(-) diff --git a/proto/substrait/algebra.proto b/proto/substrait/algebra.proto index 8ea867eba..1e3f0cc72 100644 --- a/proto/substrait/algebra.proto +++ b/proto/substrait/algebra.proto @@ -361,6 +361,47 @@ message Rel { ExtensionMultiRel extension_multi = 10; ExtensionLeafRel extension_leaf = 11; CrossRel cross = 12; + WriteRel write = 13; + } +} + +// The operator that modifies the schema/content of a database +message WriteRel { + // The table to be modified + NamedTable table = 1; + // The columns that will be modified (representing after-image of a schema change) + NamedStruct base_schema = 2; + // The default values for the columns (representing after-image of a schema change) + repeated Literal defaults = 3; + + //TODO add constraints/indexes/etc..? + + // The type of operation to perform + WriteOp op = 4; + + // The relation that determines the tuples to add/remove/modify + // remaining columns are left as-is or to default (for INSERT). + Rel query = 5; + + // The rel that specifies the output of the operator. It allows references + // to DELETED. or INSERTED.. It can also compute simply count of + // affected tuples (per common behavior of most RDBMS). Defaults to no output. + Rel output = 6; + + enum WriteOp { + CREATE_TABLE = 0; + DROP_TABLE = 1; + ALTER_TABLE = 2; + TRUNCATE_TABLE = 3; + + CREATE_VIEW = 4; + CREATE_OR_REPLACE = 5; + DROP_VIEW = 6; + + INSERT = 7; + INSERT_OR_REPLACE = 8; + DELETE = 9; + UPDATE = 10; } } @@ -380,6 +421,107 @@ message FunctionArgument { } } +message Literal { + oneof literal_type { + bool boolean = 1; + int32 i8 = 2; + int32 i16 = 3; + int32 i32 = 5; + int64 i64 = 7; + float fp32 = 10; + double fp64 = 11; + string string = 12; + bytes binary = 13; + // Timestamp in units of microseconds since the UNIX epoch. + int64 timestamp = 14; + // Date in units of days since the UNIX epoch. + int32 date = 16; + // Time in units of microseconds past midnight + int64 time = 17; + IntervalYearToMonth interval_year_to_month = 19; + IntervalDayToSecond interval_day_to_second = 20; + string fixed_char = 21; + VarChar var_char = 22; + bytes fixed_binary = 23; + Decimal decimal = 24; + Struct struct = 25; + Map map = 26; + // Timestamp in units of microseconds since the UNIX epoch. + int64 timestamp_tz = 27; + bytes uuid = 28; + Type null = 29; // a typed null literal + List list = 30; + Type.List empty_list = 31; + Type.Map empty_map = 32; + UserDefined user_defined = 33; + } + + // whether the literal type should be treated as a nullable type. Applies to + // all members of union other than the Typed null (which should directly + // declare nullability). + bool nullable = 50; + + // optionally points to a type_variation_anchor defined in this plan. + // Applies to all members of union other than the Typed null (which should + // directly declare the type variation). + uint32 type_variation_reference = 51; + + message VarChar { + string value = 1; + uint32 length = 2; + } + + message Decimal { + // little-endian twos-complement integer representation of complete value + // (ignoring precision) Always 16 bytes in length + bytes value = 1; + // The maximum number of digits allowed in the value. + // the maximum precision is 38. + int32 precision = 2; + // declared scale of decimal literal + int32 scale = 3; + } + + message Map { + message KeyValue { + Literal key = 1; + Literal value = 2; + } + + repeated KeyValue key_values = 1; + } + + message IntervalYearToMonth { + int32 years = 1; + int32 months = 2; + } + + message IntervalDayToSecond { + int32 days = 1; + int32 seconds = 2; + int32 microseconds = 3; + } + + message Struct { + // A possibly heterogeneously typed list of literals + repeated Literal fields = 1; + } + + message List { + // A homogeneously typed list of literals + repeated Literal values = 1; + } + + message UserDefined { + // points to a type_anchor defined in this plan + uint32 type_reference = 1; + + // the value of the literal, serialized using some type-specific + // protobuf message + google.protobuf.Any value = 2; + } +} + message Expression { oneof rex_type { Literal literal = 1; @@ -412,106 +554,6 @@ message Expression { } } - message Literal { - oneof literal_type { - bool boolean = 1; - int32 i8 = 2; - int32 i16 = 3; - int32 i32 = 5; - int64 i64 = 7; - float fp32 = 10; - double fp64 = 11; - string string = 12; - bytes binary = 13; - // Timestamp in units of microseconds since the UNIX epoch. - int64 timestamp = 14; - // Date in units of days since the UNIX epoch. - int32 date = 16; - // Time in units of microseconds past midnight - int64 time = 17; - IntervalYearToMonth interval_year_to_month = 19; - IntervalDayToSecond interval_day_to_second = 20; - string fixed_char = 21; - VarChar var_char = 22; - bytes fixed_binary = 23; - Decimal decimal = 24; - Struct struct = 25; - Map map = 26; - // Timestamp in units of microseconds since the UNIX epoch. - int64 timestamp_tz = 27; - bytes uuid = 28; - Type null = 29; // a typed null literal - List list = 30; - Type.List empty_list = 31; - Type.Map empty_map = 32; - UserDefined user_defined = 33; - } - - // whether the literal type should be treated as a nullable type. Applies to - // all members of union other than the Typed null (which should directly - // declare nullability). - bool nullable = 50; - - // optionally points to a type_variation_anchor defined in this plan. - // Applies to all members of union other than the Typed null (which should - // directly declare the type variation). - uint32 type_variation_reference = 51; - - message VarChar { - string value = 1; - uint32 length = 2; - } - - message Decimal { - // little-endian twos-complement integer representation of complete value - // (ignoring precision) Always 16 bytes in length - bytes value = 1; - // The maximum number of digits allowed in the value. - // the maximum precision is 38. - int32 precision = 2; - // declared scale of decimal literal - int32 scale = 3; - } - - message Map { - message KeyValue { - Literal key = 1; - Literal value = 2; - } - - repeated KeyValue key_values = 1; - } - - message IntervalYearToMonth { - int32 years = 1; - int32 months = 2; - } - - message IntervalDayToSecond { - int32 days = 1; - int32 seconds = 2; - int32 microseconds = 3; - } - - message Struct { - // A possibly heterogeneously typed list of literals - repeated Literal fields = 1; - } - - message List { - // A homogeneously typed list of literals - repeated Literal values = 1; - } - - message UserDefined { - // points to a type_anchor defined in this plan - uint32 type_reference = 1; - - // the value of the literal, serialized using some type-specific - // protobuf message - google.protobuf.Any value = 2; - } - } message ScalarFunction { // points to a function_anchor defined in this plan From e5c5c6cc98a6fef47291a5ffedf64abab94ff6c4 Mon Sep 17 00:00:00 2001 From: Carlo Curino Date: Tue, 28 Jun 2022 21:49:35 +0200 Subject: [PATCH 02/16] feat: lint/style fixes (#128) --- proto/substrait/algebra.proto | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/proto/substrait/algebra.proto b/proto/substrait/algebra.proto index 1e3f0cc72..f41320943 100644 --- a/proto/substrait/algebra.proto +++ b/proto/substrait/algebra.proto @@ -55,6 +55,13 @@ message RelCommon { } } +// A base table. The list of string is used to represent namespacing (e.g., mydb.mytable). +// This assumes shared catalog between systems exchanging a message. +message NamedTable { + repeated string names = 1; + substrait.extensions.AdvancedExtension advanced_extension = 10; +} + // The scan operator of base data (physical or virtual), including filtering and projection. message ReadRel { RelCommon common = 1; @@ -71,16 +78,9 @@ message ReadRel { ExtensionTable extension_table = 8; } - // A base table. The list of string is used to represent namespacing (e.g., mydb.mytable). - // This assumes shared catalog between systems exchanging a message. - message NamedTable { - repeated string names = 1; - substrait.extensions.AdvancedExtension advanced_extension = 10; - } - // A table composed of literals. message VirtualTable { - repeated Expression.Literal.Struct values = 1; + repeated Literal.Struct values = 1; } // A stub type that can be used to extend/introduce new table types outside @@ -370,7 +370,7 @@ message WriteRel { // The table to be modified NamedTable table = 1; // The columns that will be modified (representing after-image of a schema change) - NamedStruct base_schema = 2; + NamedStruct base_schema = 2; // The default values for the columns (representing after-image of a schema change) repeated Literal defaults = 3; @@ -554,7 +554,6 @@ message Expression { } } - message ScalarFunction { // points to a function_anchor defined in this plan uint32 function_reference = 1; From a3a0096d3740d16874025edc1923f4a4400ff7e1 Mon Sep 17 00:00:00 2001 From: Carlo Curino Date: Tue, 28 Jun 2022 21:52:42 +0200 Subject: [PATCH 03/16] feat: more lint fixes (#128) --- proto/substrait/algebra.proto | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/proto/substrait/algebra.proto b/proto/substrait/algebra.proto index f41320943..1cf780de0 100644 --- a/proto/substrait/algebra.proto +++ b/proto/substrait/algebra.proto @@ -389,19 +389,20 @@ message WriteRel { Rel output = 6; enum WriteOp { - CREATE_TABLE = 0; - DROP_TABLE = 1; - ALTER_TABLE = 2; - TRUNCATE_TABLE = 3; - - CREATE_VIEW = 4; - CREATE_OR_REPLACE = 5; - DROP_VIEW = 6; - - INSERT = 7; - INSERT_OR_REPLACE = 8; - DELETE = 9; - UPDATE = 10; + WRITE_OP_UNSPECIFIED = 0; + WRITE_OP_CREATE_TABLE = 1; + WRITE_OP_DROP_TABLE = 2; + WRITE_OP_ALTER_TABLE = 3; + WRITE_OP_TRUNCATE_TABLE = 4; + + WRITE_OP_CREATE_VIEW = 5; + WRITE_OP_CREATE_OR_REPLACE = 6; + WRITE_OP_DROP_VIEW = 7; + + WRITE_OP_INSERT = 8; + WRITE_OP_INSERT_OR_REPLACE = 9; + WRITE_OP_DELETE = 10; + WRITE_OP_UPDATE = 11; } } From 4a624b56ae3a469c7996cac6548701f2e81dc03d Mon Sep 17 00:00:00 2001 From: Carlo Curino Date: Tue, 28 Jun 2022 22:01:15 +0200 Subject: [PATCH 04/16] feat: moving Literal back to Expression (#128) --- proto/substrait/algebra.proto | 223 +++++++++++++++++----------------- 1 file changed, 111 insertions(+), 112 deletions(-) diff --git a/proto/substrait/algebra.proto b/proto/substrait/algebra.proto index 1cf780de0..9b7d66401 100644 --- a/proto/substrait/algebra.proto +++ b/proto/substrait/algebra.proto @@ -55,13 +55,6 @@ message RelCommon { } } -// A base table. The list of string is used to represent namespacing (e.g., mydb.mytable). -// This assumes shared catalog between systems exchanging a message. -message NamedTable { - repeated string names = 1; - substrait.extensions.AdvancedExtension advanced_extension = 10; -} - // The scan operator of base data (physical or virtual), including filtering and projection. message ReadRel { RelCommon common = 1; @@ -78,9 +71,16 @@ message ReadRel { ExtensionTable extension_table = 8; } + // A base table. The list of string is used to represent namespacing (e.g., mydb.mytable). + // This assumes shared catalog between systems exchanging a message. + message NamedTable { + repeated string names = 1; + substrait.extensions.AdvancedExtension advanced_extension = 10; + } + // A table composed of literals. message VirtualTable { - repeated Literal.Struct values = 1; + repeated Expression.Literal.Struct values = 1; } // A stub type that can be used to extend/introduce new table types outside @@ -361,18 +361,17 @@ message Rel { ExtensionMultiRel extension_multi = 10; ExtensionLeafRel extension_leaf = 11; CrossRel cross = 12; - WriteRel write = 13; } } // The operator that modifies the schema/content of a database message WriteRel { // The table to be modified - NamedTable table = 1; + ReadRel.NamedTable table = 1; // The columns that will be modified (representing after-image of a schema change) NamedStruct base_schema = 2; // The default values for the columns (representing after-image of a schema change) - repeated Literal defaults = 3; + repeated Expression.Literal defaults = 3; //TODO add constraints/indexes/etc..? @@ -422,107 +421,6 @@ message FunctionArgument { } } -message Literal { - oneof literal_type { - bool boolean = 1; - int32 i8 = 2; - int32 i16 = 3; - int32 i32 = 5; - int64 i64 = 7; - float fp32 = 10; - double fp64 = 11; - string string = 12; - bytes binary = 13; - // Timestamp in units of microseconds since the UNIX epoch. - int64 timestamp = 14; - // Date in units of days since the UNIX epoch. - int32 date = 16; - // Time in units of microseconds past midnight - int64 time = 17; - IntervalYearToMonth interval_year_to_month = 19; - IntervalDayToSecond interval_day_to_second = 20; - string fixed_char = 21; - VarChar var_char = 22; - bytes fixed_binary = 23; - Decimal decimal = 24; - Struct struct = 25; - Map map = 26; - // Timestamp in units of microseconds since the UNIX epoch. - int64 timestamp_tz = 27; - bytes uuid = 28; - Type null = 29; // a typed null literal - List list = 30; - Type.List empty_list = 31; - Type.Map empty_map = 32; - UserDefined user_defined = 33; - } - - // whether the literal type should be treated as a nullable type. Applies to - // all members of union other than the Typed null (which should directly - // declare nullability). - bool nullable = 50; - - // optionally points to a type_variation_anchor defined in this plan. - // Applies to all members of union other than the Typed null (which should - // directly declare the type variation). - uint32 type_variation_reference = 51; - - message VarChar { - string value = 1; - uint32 length = 2; - } - - message Decimal { - // little-endian twos-complement integer representation of complete value - // (ignoring precision) Always 16 bytes in length - bytes value = 1; - // The maximum number of digits allowed in the value. - // the maximum precision is 38. - int32 precision = 2; - // declared scale of decimal literal - int32 scale = 3; - } - - message Map { - message KeyValue { - Literal key = 1; - Literal value = 2; - } - - repeated KeyValue key_values = 1; - } - - message IntervalYearToMonth { - int32 years = 1; - int32 months = 2; - } - - message IntervalDayToSecond { - int32 days = 1; - int32 seconds = 2; - int32 microseconds = 3; - } - - message Struct { - // A possibly heterogeneously typed list of literals - repeated Literal fields = 1; - } - - message List { - // A homogeneously typed list of literals - repeated Literal values = 1; - } - - message UserDefined { - // points to a type_anchor defined in this plan - uint32 type_reference = 1; - - // the value of the literal, serialized using some type-specific - // protobuf message - google.protobuf.Any value = 2; - } -} - message Expression { oneof rex_type { Literal literal = 1; @@ -555,6 +453,107 @@ message Expression { } } + message Literal { + oneof literal_type { + bool boolean = 1; + int32 i8 = 2; + int32 i16 = 3; + int32 i32 = 5; + int64 i64 = 7; + float fp32 = 10; + double fp64 = 11; + string string = 12; + bytes binary = 13; + // Timestamp in units of microseconds since the UNIX epoch. + int64 timestamp = 14; + // Date in units of days since the UNIX epoch. + int32 date = 16; + // Time in units of microseconds past midnight + int64 time = 17; + IntervalYearToMonth interval_year_to_month = 19; + IntervalDayToSecond interval_day_to_second = 20; + string fixed_char = 21; + VarChar var_char = 22; + bytes fixed_binary = 23; + Decimal decimal = 24; + Struct struct = 25; + Map map = 26; + // Timestamp in units of microseconds since the UNIX epoch. + int64 timestamp_tz = 27; + bytes uuid = 28; + Type null = 29; // a typed null literal + List list = 30; + Type.List empty_list = 31; + Type.Map empty_map = 32; + UserDefined user_defined = 33; + } + + // whether the literal type should be treated as a nullable type. Applies to + // all members of union other than the Typed null (which should directly + // declare nullability). + bool nullable = 50; + + // optionally points to a type_variation_anchor defined in this plan. + // Applies to all members of union other than the Typed null (which should + // directly declare the type variation). + uint32 type_variation_reference = 51; + + message VarChar { + string value = 1; + uint32 length = 2; + } + + message Decimal { + // little-endian twos-complement integer representation of complete value + // (ignoring precision) Always 16 bytes in length + bytes value = 1; + // The maximum number of digits allowed in the value. + // the maximum precision is 38. + int32 precision = 2; + // declared scale of decimal literal + int32 scale = 3; + } + + message Map { + message KeyValue { + Literal key = 1; + Literal value = 2; + } + + repeated KeyValue key_values = 1; + } + + message IntervalYearToMonth { + int32 years = 1; + int32 months = 2; + } + + message IntervalDayToSecond { + int32 days = 1; + int32 seconds = 2; + int32 microseconds = 3; + } + + message Struct { + // A possibly heterogeneously typed list of literals + repeated Literal fields = 1; + } + + message List { + // A homogeneously typed list of literals + repeated Literal values = 1; + } + + message UserDefined { + // points to a type_anchor defined in this plan + uint32 type_reference = 1; + + // the value of the literal, serialized using some type-specific + // protobuf message + google.protobuf.Any value = 2; + } + } + message ScalarFunction { // points to a function_anchor defined in this plan uint32 function_reference = 1; From 5ff43061ee0d19c4c73d54be621b63cf149ce691 Mon Sep 17 00:00:00 2001 From: Carlo Curino Date: Tue, 28 Jun 2022 22:04:15 +0200 Subject: [PATCH 05/16] feat: basic website changed (more needed) (#128) --- site/docs/relations/logical_relations.md | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/site/docs/relations/logical_relations.md b/site/docs/relations/logical_relations.md index c62a015be..5cbe40715 100644 --- a/site/docs/relations/logical_relations.md +++ b/site/docs/relations/logical_relations.md @@ -305,12 +305,12 @@ The aggregate operation groups input data on one or more sets of grouping keys, The write operator is an operator that consumes one output and writes it to storage. A simple example would be writing Parquet files. It is expected that many types of writes will be added over time. -| Signature | Value | -| -------------------- | --------------- | -| Inputs | 1 | -| Outputs | 0 | -| Property Maintenance | N/A (no output) | -| Direct Output Order | N/A (no output) | +| Signature | Value | +| -------------------- |----------------------| +| Inputs | 1 | +| Outputs | 1 | +| Property Maintenance | N/A (no output) | +| Direct Output Order | Unchanged from input | ### Write Properties @@ -326,6 +326,13 @@ The write operator is an operator that consumes one output and writes it to stor Write definition types are built by the community and added to the specification. This is a portion of specification that is expected to grow rapidly. + +=== "WriteRel Message" + + ```proto +%%% proto.algebra.WriteRel %%% +``` + #### Virtual Table | Property | Description | Required | From be055bbcc21e0436910c4e24991ac253b071bab7 Mon Sep 17 00:00:00 2001 From: Carlo Curino Date: Wed, 29 Jun 2022 00:22:56 +0200 Subject: [PATCH 06/16] feat: splitting DDL and DML writes, and adding more write_type (#128) --- proto/substrait/algebra.proto | 67 ++++++++++++++++++++++++++--------- 1 file changed, 50 insertions(+), 17 deletions(-) diff --git a/proto/substrait/algebra.proto b/proto/substrait/algebra.proto index 9b7d66401..6c2857d8b 100644 --- a/proto/substrait/algebra.proto +++ b/proto/substrait/algebra.proto @@ -364,17 +364,59 @@ message Rel { } } -// The operator that modifies the schema/content of a database -message WriteRel { - // The table to be modified - ReadRel.NamedTable table = 1; +// A base table for writing. The list of string is used to represent namespacing (e.g., mydb.mytable). +// This assumes shared catalog between systems exchanging a message. +// it also includes a base schema, and default types +message NamedTableWrite { + repeated string names = 1; // The columns that will be modified (representing after-image of a schema change) NamedStruct base_schema = 2; // The default values for the columns (representing after-image of a schema change) repeated Expression.Literal defaults = 3; + substrait.extensions.AdvancedExtension advanced_extension = 10; +} + + +message DdlRel { + + // Definition of which type of scan operation is to be performed + oneof write_type { + NamedTableWrite named_table = 1; + ReadRel.ExtensionTable extension_table = 2; + } + //TODO add constraints/indexes/etc..? + // The type of operation to perform + DdlOp op = 3; + + // The body of the CREATE VIEW / CTAS + Rel query = 4; + + enum DdlOp { + DDL_OP_UNSPECIFIED = 0; + DDL_OP_CREATE_TABLE = 1; + DDL_OP_DROP_TABLE = 2; + DDL_OP_ALTER_TABLE = 3; + + DDL_OP_CREATE_VIEW = 4; + DDL_OP_CREATE_OR_REPLACE = 5; + DDL_OP_DROP_VIEW = 6; + } +} + + +// The operator that modifies the schema/content of a database +message WriteRel { + + // Definition of which type of write operation is to be performed + oneof write_type { + NamedTableWrite named_table = 1; + ReadRel.LocalFiles local_files = 2; + ReadRel.ExtensionTable extension_table = 3; + } + // The type of operation to perform WriteOp op = 4; @@ -389,19 +431,10 @@ message WriteRel { enum WriteOp { WRITE_OP_UNSPECIFIED = 0; - WRITE_OP_CREATE_TABLE = 1; - WRITE_OP_DROP_TABLE = 2; - WRITE_OP_ALTER_TABLE = 3; - WRITE_OP_TRUNCATE_TABLE = 4; - - WRITE_OP_CREATE_VIEW = 5; - WRITE_OP_CREATE_OR_REPLACE = 6; - WRITE_OP_DROP_VIEW = 7; - - WRITE_OP_INSERT = 8; - WRITE_OP_INSERT_OR_REPLACE = 9; - WRITE_OP_DELETE = 10; - WRITE_OP_UPDATE = 11; + WRITE_OP_INSERT = 1; + WRITE_OP_INSERT_OR_REPLACE = 2; + WRITE_OP_DELETE = 3; + WRITE_OP_UPDATE = 4; } } From f2acc998f16868021c2996f41576b0f793d09f62 Mon Sep 17 00:00:00 2001 From: Carlo Curino Date: Wed, 29 Jun 2022 00:23:57 +0200 Subject: [PATCH 07/16] feat: formatting (#128) --- proto/substrait/algebra.proto | 4 ---- 1 file changed, 4 deletions(-) diff --git a/proto/substrait/algebra.proto b/proto/substrait/algebra.proto index 6c2857d8b..8e99d3545 100644 --- a/proto/substrait/algebra.proto +++ b/proto/substrait/algebra.proto @@ -377,9 +377,7 @@ message NamedTableWrite { substrait.extensions.AdvancedExtension advanced_extension = 10; } - message DdlRel { - // Definition of which type of scan operation is to be performed oneof write_type { NamedTableWrite named_table = 1; @@ -406,10 +404,8 @@ message DdlRel { } } - // The operator that modifies the schema/content of a database message WriteRel { - // Definition of which type of write operation is to be performed oneof write_type { NamedTableWrite named_table = 1; From 6bddcf553f245f12ddba0ed4547e0c83fed10c5b Mon Sep 17 00:00:00 2001 From: Jeroen van Straten Date: Mon, 4 Jul 2022 02:07:06 +0200 Subject: [PATCH 08/16] feat: introduce compound (parameterizable) extension types and variations (#196) * feat: introduce compound (parameterizable) extension types and variations --- proto/substrait/algebra.proto | 4 ++ proto/substrait/type.proto | 20 ++++++++ site/docs/types/type_classes.md | 79 +++++++++++++++++++++++++++++- text/simple_extensions_schema.yaml | 36 +++++++++++++- 4 files changed, 137 insertions(+), 2 deletions(-) diff --git a/proto/substrait/algebra.proto b/proto/substrait/algebra.proto index 8e99d3545..ceb0d4bc5 100644 --- a/proto/substrait/algebra.proto +++ b/proto/substrait/algebra.proto @@ -577,6 +577,10 @@ message Expression { // points to a type_anchor defined in this plan uint32 type_reference = 1; + // The parameters to be bound to the type class, if the type class is + // parameterizable. + repeated Type.Parameter type_parameters = 3; + // the value of the literal, serialized using some type-specific // protobuf message google.protobuf.Any value = 2; diff --git a/proto/substrait/type.proto b/proto/substrait/type.proto index c417b3a10..a7f1c665d 100644 --- a/proto/substrait/type.proto +++ b/proto/substrait/type.proto @@ -3,6 +3,8 @@ syntax = "proto3"; package substrait; +import "google/protobuf/empty.proto"; + option csharp_namespace = "Substrait.Protobuf"; option go_package = "github.com/substrait-io/substrait-go/proto"; option java_multiple_files = true; @@ -180,6 +182,24 @@ message Type { uint32 type_reference = 1; uint32 type_variation_reference = 2; Nullability nullability = 3; + repeated Parameter type_parameters = 4; + } + + message Parameter { + oneof parameter { + // Explicitly null/unspecified parameter, to select the default value (if + // any). + google.protobuf.Empty null = 1; + + // Data type parameters, like the i32 in LIST. + Type data_type = 2; + + // Value parameters, like the 10 in VARCHAR<10>. + bool boolean = 3; + int64 integer = 4; + string enum = 5; + string string = 6; + } } } diff --git a/site/docs/types/type_classes.md b/site/docs/types/type_classes.md index 1abc0c8f6..948632c37 100644 --- a/site/docs/types/type_classes.md +++ b/site/docs/types/type_classes.md @@ -44,7 +44,7 @@ Compound type classes are type classes that need to be configured by means of a ## User-Defined Types -User-defined type classes can be created using a combination of pre-defined types. User-defined types are defined as part of [simple extensions](../extensions/index.md#simple-extensions). An extension can declare an arbitrary number of user defined extension types. Initially, user defined types must be simple types (although they can be constructed of a number of inner compound and simple types). +User-defined type classes can be created using a combination of pre-defined types. User-defined types are defined as part of [simple extensions](../extensions/index.md#simple-extensions). An extension can declare an arbitrary number of user defined extension types. A YAML example of an extension type is below: @@ -58,3 +58,80 @@ A YAML example of an extension type is below: This declares a new type (namespaced to the associated YAML file) called "point". This type is composed of two `i32` values named longitude and latitude. Once a type has been declared, it can be used in function declarations. [TBD: should field references be allowed to dereference the components of a user defined type?] Literals for user-defined types are represented using protobuf [Any](https://developers.google.com/protocol-buffers/docs/proto3#any) messages. + +### Compound User-Defined Types + +User-defined types may be turned into compound types by requiring parameters to be passed to them. The supported "meta-types" for parameters are data types (like those used in `LIST`, `MAP`, and `STRUCT`), booleans, integers, enumerations, and strings. Using parameters, we could redefine "point" with different types of coordinates. For example: + +```yaml +name: point +parameters: + - name: T + description: | + The type used for the longitude and latitude + components of the point. + type: dataType +``` + +or: + +```yaml +name: point +parameters: + - name: coordinate_type + type: enumeration + options: + - integer + - double +``` + +or: + +```yaml +name: point +parameters: + - name: LONG + type: dataType + - name: LAT + type: dataType +``` + +We can't specify the internal structure in this case, because there is currently no support for derived types in the structure. + +The allowed range can be limited for integer parameters. For example: + +```yaml +name: vector +parameters: + - name: T + type: dataType + - name: dimensions + type: integer + min: 2 + max: 3 +``` + +This specifies a vector that can be either 2- or 3-dimensional. Note however that it's not currently possible to put constraints on data type, string, or (technically) boolean parameters. + +Similar to function arguments, the last parameter may be specified to be variadic, allowing it to be specified one or more times instead of only once. For example: + +```yaml +name: union +parameters: + - name: T + type: dataType +variadic: true +``` + +This defines a type that can be parameterized with one or more other data types, for example `union` but also `union`. Zero or more is also possible, by making the last argument optional: + +```yaml +name: tuple +parameters: + - name: T + type: dataType + optional: true +variadic: true +``` + +This would also allow for `tuple<>`, to define a zero-tuple. diff --git a/text/simple_extensions_schema.yaml b/text/simple_extensions_schema.yaml index 52ffbc782..6fe274c78 100644 --- a/text/simple_extensions_schema.yaml +++ b/text/simple_extensions_schema.yaml @@ -16,6 +16,10 @@ properties: type: object additionalProperties: $ref: "#/$defs/type" + parameters: # parameter list for compound types + $ref: "#/$defs/type_param_defs" + variadic: # when set, last parameter may be specified one or more times + type: boolean type_variations: type: array minItems: 1 @@ -25,7 +29,7 @@ properties: required: [parent, name] properties: parent: - type: string + $ref: "#/$defs/type" name: type: string description: @@ -47,6 +51,36 @@ $defs: oneOf: - type: string - type: object + type_param_defs: # an array of compound type parameter definitions + type: array + items: + type: object + required: [type] + properties: + name: # name of the parameter (for documentation only) + type: string + description: # description (for documentation only) + type: string + type: # expected metatype for the parameter + type: string + enum: + - dataType + - boolean + - integer + - enumeration + - string + min: # for integers, the minimum supported value (inclusive) + type: number + max: # for integers, the maximum supported value (inclusive) + type: number + options: # for enums, the list of supported values + type: array + minItems: 1 + uniqueItems: true + items: + type: string + optional: # when set to true, the parameter may be omitted at the end or skipped using null + type: boolean arguments: # an array of arguments type: array items: From c9469ee8d3c46f51842b6260d0d98cf37a465b2d Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 11 Jul 2022 16:08:13 +0000 Subject: [PATCH 09/16] chore(release): 0.7.0 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d21f4ee6..cd052e1f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,13 @@ Release Notes --- +## [0.7.0](https://github.com/substrait-io/substrait/compare/v0.6.0...v0.7.0) (2022-07-11) + + +### Features + +* introduce compound (parameterizable) extension types and variations ([#196](https://github.com/substrait-io/substrait/issues/196)) ([a79eb07](https://github.com/substrait-io/substrait/commit/a79eb07a15cfa157e795f028a83f746967c98805)) + ## [0.6.0](https://github.com/substrait-io/substrait/compare/v0.5.0...v0.6.0) (2022-06-26) From 7f5d120e26dc1a8b9299492995a08b3a9d666611 Mon Sep 17 00:00:00 2001 From: Weston Pace Date: Tue, 12 Jul 2022 09:00:00 -0700 Subject: [PATCH 10/16] fix: add overflow behavior to integer division (#223) BREAKING CHANGE: The signature of divide functions for multiple types now specify an enumeration prior to specifying operands. --- extensions/functions_arithmetic.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/extensions/functions_arithmetic.yaml b/extensions/functions_arithmetic.yaml index 00d7f84de..a750df682 100644 --- a/extensions/functions_arithmetic.yaml +++ b/extensions/functions_arithmetic.yaml @@ -126,18 +126,26 @@ scalar_functions: description: "Divide one value by another. Partial values are truncated." impls: - args: + - options: [ SILENT, SATURATE, ERROR ] + required: false - value: i8 - value: i8 return: i8 - args: + - options: [ SILENT, SATURATE, ERROR ] + required: false - value: i16 - value: i16 return: i16 - args: + - options: [ SILENT, SATURATE, ERROR ] + required: false - value: i32 - value: i32 return: i32 - args: + - options: [ SILENT, SATURATE, ERROR ] + required: false - value: i64 - value: i64 return: i64 From e6179cfffdbb1b3ff3623d7b72347530193d6df1 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sun, 17 Jul 2022 03:38:35 +0000 Subject: [PATCH 11/16] chore(release): 0.8.0 --- CHANGELOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cd052e1f7..feb6da2d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,17 @@ Release Notes --- +## [0.8.0](https://github.com/substrait-io/substrait/compare/v0.7.0...v0.8.0) (2022-07-17) + + +### ⚠ BREAKING CHANGES + +* The signature of divide functions for multiple types now specify an enumeration prior to specifying operands. + +### Bug Fixes + +* add overflow behavior to integer division ([#223](https://github.com/substrait-io/substrait/issues/223)) ([cf552d7](https://github.com/substrait-io/substrait/commit/cf552d7c76da9a91bce992391356c6ffb5a969ac)) + ## [0.7.0](https://github.com/substrait-io/substrait/compare/v0.6.0...v0.7.0) (2022-07-11) From 8a5ff26b857ceeb6e336fdb00fa3edab0749033f Mon Sep 17 00:00:00 2001 From: Carlo Curino Date: Tue, 28 Jun 2022 21:43:03 +0200 Subject: [PATCH 12/16] feat: add initial support for DDL and data modification operator (#128) --- proto/substrait/algebra.proto | 142 ++++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) diff --git a/proto/substrait/algebra.proto b/proto/substrait/algebra.proto index ceb0d4bc5..2f8859028 100644 --- a/proto/substrait/algebra.proto +++ b/proto/substrait/algebra.proto @@ -361,6 +361,47 @@ message Rel { ExtensionMultiRel extension_multi = 10; ExtensionLeafRel extension_leaf = 11; CrossRel cross = 12; + WriteRel write = 13; + } +} + +// The operator that modifies the schema/content of a database +message WriteRel { + // The table to be modified + NamedTable table = 1; + // The columns that will be modified (representing after-image of a schema change) + NamedStruct base_schema = 2; + // The default values for the columns (representing after-image of a schema change) + repeated Literal defaults = 3; + + //TODO add constraints/indexes/etc..? + + // The type of operation to perform + WriteOp op = 4; + + // The relation that determines the tuples to add/remove/modify + // remaining columns are left as-is or to default (for INSERT). + Rel query = 5; + + // The rel that specifies the output of the operator. It allows references + // to DELETED. or INSERTED.. It can also compute simply count of + // affected tuples (per common behavior of most RDBMS). Defaults to no output. + Rel output = 6; + + enum WriteOp { + CREATE_TABLE = 0; + DROP_TABLE = 1; + ALTER_TABLE = 2; + TRUNCATE_TABLE = 3; + + CREATE_VIEW = 4; + CREATE_OR_REPLACE = 5; + DROP_VIEW = 6; + + INSERT = 7; + INSERT_OR_REPLACE = 8; + DELETE = 9; + UPDATE = 10; } } @@ -450,6 +491,107 @@ message FunctionArgument { } } +message Literal { + oneof literal_type { + bool boolean = 1; + int32 i8 = 2; + int32 i16 = 3; + int32 i32 = 5; + int64 i64 = 7; + float fp32 = 10; + double fp64 = 11; + string string = 12; + bytes binary = 13; + // Timestamp in units of microseconds since the UNIX epoch. + int64 timestamp = 14; + // Date in units of days since the UNIX epoch. + int32 date = 16; + // Time in units of microseconds past midnight + int64 time = 17; + IntervalYearToMonth interval_year_to_month = 19; + IntervalDayToSecond interval_day_to_second = 20; + string fixed_char = 21; + VarChar var_char = 22; + bytes fixed_binary = 23; + Decimal decimal = 24; + Struct struct = 25; + Map map = 26; + // Timestamp in units of microseconds since the UNIX epoch. + int64 timestamp_tz = 27; + bytes uuid = 28; + Type null = 29; // a typed null literal + List list = 30; + Type.List empty_list = 31; + Type.Map empty_map = 32; + UserDefined user_defined = 33; + } + + // whether the literal type should be treated as a nullable type. Applies to + // all members of union other than the Typed null (which should directly + // declare nullability). + bool nullable = 50; + + // optionally points to a type_variation_anchor defined in this plan. + // Applies to all members of union other than the Typed null (which should + // directly declare the type variation). + uint32 type_variation_reference = 51; + + message VarChar { + string value = 1; + uint32 length = 2; + } + + message Decimal { + // little-endian twos-complement integer representation of complete value + // (ignoring precision) Always 16 bytes in length + bytes value = 1; + // The maximum number of digits allowed in the value. + // the maximum precision is 38. + int32 precision = 2; + // declared scale of decimal literal + int32 scale = 3; + } + + message Map { + message KeyValue { + Literal key = 1; + Literal value = 2; + } + + repeated KeyValue key_values = 1; + } + + message IntervalYearToMonth { + int32 years = 1; + int32 months = 2; + } + + message IntervalDayToSecond { + int32 days = 1; + int32 seconds = 2; + int32 microseconds = 3; + } + + message Struct { + // A possibly heterogeneously typed list of literals + repeated Literal fields = 1; + } + + message List { + // A homogeneously typed list of literals + repeated Literal values = 1; + } + + message UserDefined { + // points to a type_anchor defined in this plan + uint32 type_reference = 1; + + // the value of the literal, serialized using some type-specific + // protobuf message + google.protobuf.Any value = 2; + } +} + message Expression { oneof rex_type { Literal literal = 1; From 187375a14cb57e92d25048de6159a41b3b531be6 Mon Sep 17 00:00:00 2001 From: Carlo Curino Date: Wed, 20 Jul 2022 15:21:04 +0200 Subject: [PATCH 13/16] feat: rebase merge (#128) --- proto/substrait/algebra.proto | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/proto/substrait/algebra.proto b/proto/substrait/algebra.proto index 2f8859028..6a5cb102d 100644 --- a/proto/substrait/algebra.proto +++ b/proto/substrait/algebra.proto @@ -55,6 +55,13 @@ message RelCommon { } } +// A base table. The list of string is used to represent namespacing (e.g., mydb.mytable). +// This assumes shared catalog between systems exchanging a message. +message NamedTable { + repeated string names = 1; + substrait.extensions.AdvancedExtension advanced_extension = 10; +} + // The scan operator of base data (physical or virtual), including filtering and projection. message ReadRel { RelCommon common = 1; @@ -71,16 +78,9 @@ message ReadRel { ExtensionTable extension_table = 8; } - // A base table. The list of string is used to represent namespacing (e.g., mydb.mytable). - // This assumes shared catalog between systems exchanging a message. - message NamedTable { - repeated string names = 1; - substrait.extensions.AdvancedExtension advanced_extension = 10; - } - // A table composed of literals. message VirtualTable { - repeated Expression.Literal.Struct values = 1; + repeated Literal.Struct values = 1; } // A stub type that can be used to extend/introduce new table types outside @@ -370,7 +370,7 @@ message WriteRel { // The table to be modified NamedTable table = 1; // The columns that will be modified (representing after-image of a schema change) - NamedStruct base_schema = 2; + NamedStruct base_schema = 2; // The default values for the columns (representing after-image of a schema change) repeated Literal defaults = 3; @@ -624,6 +624,7 @@ message Expression { } } +<<<<<<< HEAD message Literal { oneof literal_type { bool boolean = 1; @@ -729,6 +730,8 @@ message Expression { } } +======= +>>>>>>> e5c5c6c (feat: lint/style fixes (#128)) message ScalarFunction { // points to a function_anchor defined in this plan uint32 function_reference = 1; From baf2337fe24c431f1d3e561e68643a61514fc2bb Mon Sep 17 00:00:00 2001 From: Carlo Curino Date: Wed, 20 Jul 2022 15:22:33 +0200 Subject: [PATCH 14/16] feat: rebase (#128) --- proto/substrait/algebra.proto | 129 +++++----------------------------- 1 file changed, 16 insertions(+), 113 deletions(-) diff --git a/proto/substrait/algebra.proto b/proto/substrait/algebra.proto index 6a5cb102d..5f75b1b2e 100644 --- a/proto/substrait/algebra.proto +++ b/proto/substrait/algebra.proto @@ -55,13 +55,6 @@ message RelCommon { } } -// A base table. The list of string is used to represent namespacing (e.g., mydb.mytable). -// This assumes shared catalog between systems exchanging a message. -message NamedTable { - repeated string names = 1; - substrait.extensions.AdvancedExtension advanced_extension = 10; -} - // The scan operator of base data (physical or virtual), including filtering and projection. message ReadRel { RelCommon common = 1; @@ -78,9 +71,16 @@ message ReadRel { ExtensionTable extension_table = 8; } + // A base table. The list of string is used to represent namespacing (e.g., mydb.mytable). + // This assumes shared catalog between systems exchanging a message. + message NamedTable { + repeated string names = 1; + substrait.extensions.AdvancedExtension advanced_extension = 10; + } + // A table composed of literals. message VirtualTable { - repeated Literal.Struct values = 1; + repeated Expression.Literal.Struct values = 1; } // A stub type that can be used to extend/introduce new table types outside @@ -361,18 +361,17 @@ message Rel { ExtensionMultiRel extension_multi = 10; ExtensionLeafRel extension_leaf = 11; CrossRel cross = 12; - WriteRel write = 13; } } // The operator that modifies the schema/content of a database message WriteRel { // The table to be modified - NamedTable table = 1; + ReadRel.NamedTable table = 1; // The columns that will be modified (representing after-image of a schema change) NamedStruct base_schema = 2; // The default values for the columns (representing after-image of a schema change) - repeated Literal defaults = 3; + repeated Expression.Literal defaults = 3; //TODO add constraints/indexes/etc..? @@ -491,107 +490,6 @@ message FunctionArgument { } } -message Literal { - oneof literal_type { - bool boolean = 1; - int32 i8 = 2; - int32 i16 = 3; - int32 i32 = 5; - int64 i64 = 7; - float fp32 = 10; - double fp64 = 11; - string string = 12; - bytes binary = 13; - // Timestamp in units of microseconds since the UNIX epoch. - int64 timestamp = 14; - // Date in units of days since the UNIX epoch. - int32 date = 16; - // Time in units of microseconds past midnight - int64 time = 17; - IntervalYearToMonth interval_year_to_month = 19; - IntervalDayToSecond interval_day_to_second = 20; - string fixed_char = 21; - VarChar var_char = 22; - bytes fixed_binary = 23; - Decimal decimal = 24; - Struct struct = 25; - Map map = 26; - // Timestamp in units of microseconds since the UNIX epoch. - int64 timestamp_tz = 27; - bytes uuid = 28; - Type null = 29; // a typed null literal - List list = 30; - Type.List empty_list = 31; - Type.Map empty_map = 32; - UserDefined user_defined = 33; - } - - // whether the literal type should be treated as a nullable type. Applies to - // all members of union other than the Typed null (which should directly - // declare nullability). - bool nullable = 50; - - // optionally points to a type_variation_anchor defined in this plan. - // Applies to all members of union other than the Typed null (which should - // directly declare the type variation). - uint32 type_variation_reference = 51; - - message VarChar { - string value = 1; - uint32 length = 2; - } - - message Decimal { - // little-endian twos-complement integer representation of complete value - // (ignoring precision) Always 16 bytes in length - bytes value = 1; - // The maximum number of digits allowed in the value. - // the maximum precision is 38. - int32 precision = 2; - // declared scale of decimal literal - int32 scale = 3; - } - - message Map { - message KeyValue { - Literal key = 1; - Literal value = 2; - } - - repeated KeyValue key_values = 1; - } - - message IntervalYearToMonth { - int32 years = 1; - int32 months = 2; - } - - message IntervalDayToSecond { - int32 days = 1; - int32 seconds = 2; - int32 microseconds = 3; - } - - message Struct { - // A possibly heterogeneously typed list of literals - repeated Literal fields = 1; - } - - message List { - // A homogeneously typed list of literals - repeated Literal values = 1; - } - - message UserDefined { - // points to a type_anchor defined in this plan - uint32 type_reference = 1; - - // the value of the literal, serialized using some type-specific - // protobuf message - google.protobuf.Any value = 2; - } -} - message Expression { oneof rex_type { Literal literal = 1; @@ -624,7 +522,6 @@ message Expression { } } -<<<<<<< HEAD message Literal { oneof literal_type { bool boolean = 1; @@ -720,18 +617,24 @@ message Expression { // points to a type_anchor defined in this plan uint32 type_reference = 1; +<<<<<<< HEAD // The parameters to be bound to the type class, if the type class is // parameterizable. repeated Type.Parameter type_parameters = 3; +======= +>>>>>>> 4a624b5 (feat: moving Literal back to Expression (#128)) // the value of the literal, serialized using some type-specific // protobuf message google.protobuf.Any value = 2; } } +<<<<<<< HEAD ======= >>>>>>> e5c5c6c (feat: lint/style fixes (#128)) +======= +>>>>>>> 4a624b5 (feat: moving Literal back to Expression (#128)) message ScalarFunction { // points to a function_anchor defined in this plan uint32 function_reference = 1; From dfe98a98a2a5d8be43a943b25c6e246661222670 Mon Sep 17 00:00:00 2001 From: Carlo Curino Date: Wed, 20 Jul 2022 16:02:50 +0200 Subject: [PATCH 15/16] feat: addressing comments, and fixing site (#128) --- proto/substrait/algebra.proto | 54 ++---------------------- site/docs/relations/logical_relations.md | 17 +++++++- 2 files changed, 19 insertions(+), 52 deletions(-) diff --git a/proto/substrait/algebra.proto b/proto/substrait/algebra.proto index 5f75b1b2e..99cfd8493 100644 --- a/proto/substrait/algebra.proto +++ b/proto/substrait/algebra.proto @@ -364,49 +364,9 @@ message Rel { } } -// The operator that modifies the schema/content of a database -message WriteRel { - // The table to be modified - ReadRel.NamedTable table = 1; - // The columns that will be modified (representing after-image of a schema change) - NamedStruct base_schema = 2; - // The default values for the columns (representing after-image of a schema change) - repeated Expression.Literal defaults = 3; - - //TODO add constraints/indexes/etc..? - - // The type of operation to perform - WriteOp op = 4; - - // The relation that determines the tuples to add/remove/modify - // remaining columns are left as-is or to default (for INSERT). - Rel query = 5; - - // The rel that specifies the output of the operator. It allows references - // to DELETED. or INSERTED.. It can also compute simply count of - // affected tuples (per common behavior of most RDBMS). Defaults to no output. - Rel output = 6; - - enum WriteOp { - CREATE_TABLE = 0; - DROP_TABLE = 1; - ALTER_TABLE = 2; - TRUNCATE_TABLE = 3; - - CREATE_VIEW = 4; - CREATE_OR_REPLACE = 5; - DROP_VIEW = 6; - - INSERT = 7; - INSERT_OR_REPLACE = 8; - DELETE = 9; - UPDATE = 10; - } -} - // A base table for writing. The list of string is used to represent namespacing (e.g., mydb.mytable). // This assumes shared catalog between systems exchanging a message. -// it also includes a base schema, and default types +// it also includes a base schema, and default values message NamedTableWrite { repeated string names = 1; // The columns that will be modified (representing after-image of a schema change) @@ -430,7 +390,7 @@ message DdlRel { DdlOp op = 3; // The body of the CREATE VIEW / CTAS - Rel query = 4; + Rel input = 4; enum DdlOp { DDL_OP_UNSPECIFIED = 0; @@ -458,7 +418,7 @@ message WriteRel { // The relation that determines the tuples to add/remove/modify // remaining columns are left as-is or to default (for INSERT). - Rel query = 5; + Rel input = 5; // The rel that specifies the output of the operator. It allows references // to DELETED. or INSERTED.. It can also compute simply count of @@ -617,24 +577,16 @@ message Expression { // points to a type_anchor defined in this plan uint32 type_reference = 1; -<<<<<<< HEAD // The parameters to be bound to the type class, if the type class is // parameterizable. repeated Type.Parameter type_parameters = 3; -======= ->>>>>>> 4a624b5 (feat: moving Literal back to Expression (#128)) // the value of the literal, serialized using some type-specific // protobuf message google.protobuf.Any value = 2; } } -<<<<<<< HEAD -======= ->>>>>>> e5c5c6c (feat: lint/style fixes (#128)) -======= ->>>>>>> 4a624b5 (feat: moving Literal back to Expression (#128)) message ScalarFunction { // points to a function_anchor defined in this plan uint32 function_reference = 1; diff --git a/site/docs/relations/logical_relations.md b/site/docs/relations/logical_relations.md index 5cbe40715..8aa1af6a4 100644 --- a/site/docs/relations/logical_relations.md +++ b/site/docs/relations/logical_relations.md @@ -299,7 +299,22 @@ The aggregate operation groups input data on one or more sets of grouping keys, ```proto %%% proto.algebra.AggregateRel %%% ``` +## DDL Operator +The operator that defines modifications of a database schema (CREATE/DROP/ALTER for TABLE and VIEWS). + +| Signature | Value | +| -------------------- |-----------------| +| Inputs | 1 | +| Outputs | 0 | +| Property Maintenance | N/A (no output) | +| Direct Output Order | N/A | + +=== "DdlRel Message" + + ```proto +%%% proto.algebra.DdlRel %%% + ``` ## Write Operator @@ -331,7 +346,7 @@ Write definition types are built by the community and added to the specification ```proto %%% proto.algebra.WriteRel %%% -``` + ``` #### Virtual Table From 2b56fe8336b0a48d32c9de48bcc3ad60b7a360fe Mon Sep 17 00:00:00 2001 From: Carlo Curino Date: Wed, 20 Jul 2022 16:27:43 +0200 Subject: [PATCH 16/16] feat: minor reordering in docs (#128) --- site/docs/relations/logical_relations.md | 34 +++++++++++++----------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/site/docs/relations/logical_relations.md b/site/docs/relations/logical_relations.md index 8aa1af6a4..edfe35f0e 100644 --- a/site/docs/relations/logical_relations.md +++ b/site/docs/relations/logical_relations.md @@ -299,22 +299,6 @@ The aggregate operation groups input data on one or more sets of grouping keys, ```proto %%% proto.algebra.AggregateRel %%% ``` -## DDL Operator - -The operator that defines modifications of a database schema (CREATE/DROP/ALTER for TABLE and VIEWS). - -| Signature | Value | -| -------------------- |-----------------| -| Inputs | 1 | -| Outputs | 0 | -| Property Maintenance | N/A (no output) | -| Direct Output Order | N/A | - -=== "DdlRel Message" - - ```proto -%%% proto.algebra.DdlRel %%% - ``` ## Write Operator @@ -365,7 +349,25 @@ Write definition types are built by the community and added to the specification | Format | Enumeration of available formats. Only current option is PARQUET. | Required | +## DDL Operator + +The operator that defines modifications of a database schema (CREATE/DROP/ALTER for TABLE and VIEWS). + +| Signature | Value | +| -------------------- |-----------------| +| Inputs | 1 | +| Outputs | 0 | +| Property Maintenance | N/A (no output) | +| Direct Output Order | N/A | + +=== "DdlRel Message" + + ```proto +%%% proto.algebra.DdlRel %%% + ``` ## Discussion Points * How to handle correlated operations? + +