diff --git a/extensions/functions_arithmetic_decimal.yaml b/extensions/functions_arithmetic_decimal.yaml index 29a2eabd1..00897a82e 100644 --- a/extensions/functions_arithmetic_decimal.yaml +++ b/extensions/functions_arithmetic_decimal.yaml @@ -20,7 +20,7 @@ scalar_functions: delta = init_prec - 38 prec = min(init_prec, 38) scale_after_borrow = max(init_scale - delta, min_scale) - scale = init_prec > 38 ? scale_after_borrow : init_scale + scale = if init_prec > 38 then scale_after_borrow else init_scale DECIMAL - name: "subtract" @@ -40,7 +40,7 @@ scalar_functions: delta = init_prec - 38 prec = min(init_prec, 38) scale_after_borrow = max(init_scale - delta, min_scale) - scale = init_prec > 38 ? scale_after_borrow : init_scale + scale = if init_prec > 38 then scale_after_borrow else init_scale DECIMAL - name: "multiply" @@ -60,7 +60,7 @@ scalar_functions: delta = init_prec - 38 prec = min(init_prec, 38) scale_after_borrow = max(init_scale - delta, min_scale) - scale = init_prec > 38 ? scale_after_borrow : init_scale + scale = if init_prec > 38 then scale_after_borrow else init_scale DECIMAL - name: "divide" @@ -80,7 +80,7 @@ scalar_functions: delta = init_prec - 38 prec = min(init_prec, 38) scale_after_borrow = max(init_scale - delta, min_scale) - scale = init_prec > 38 ? scale_after_borrow : init_scale + scale = if init_prec > 38 then scale_after_borrow else init_scale DECIMAL - name: "modulus" @@ -100,7 +100,7 @@ scalar_functions: delta = init_prec - 38 prec = min(init_prec, 38) scale_after_borrow = max(init_scale - delta, min_scale) - scale = init_prec > 38 ? scale_after_borrow : init_scale + scale = if init_prec > 38 then scale_after_borrow else init_scale DECIMAL aggregate_functions: - name: "sum" @@ -114,8 +114,8 @@ aggregate_functions: value: "DECIMAL" nullability: DECLARED_OUTPUT decomposable: MANY - intermediate: "DECIMAL<38,S>?" - return: "DECIMAL<38,S>?" + intermediate: "DECIMAL?<38,S>" + return: "DECIMAL?<38,S>" - name: "avg" description: Average a set of values. impls: @@ -137,8 +137,8 @@ aggregate_functions: value: "DECIMAL" nullability: DECLARED_OUTPUT decomposable: MANY - intermediate: "DECIMAL?" - return: "DECIMAL?" + intermediate: "DECIMAL?" + return: "DECIMAL?" - name: "max" description: Max a set of values. impls: @@ -147,5 +147,5 @@ aggregate_functions: value: "DECIMAL" nullability: DECLARED_OUTPUT decomposable: MANY - intermediate: "DECIMAL?" - return: "DECIMAL?" + intermediate: "DECIMAL?" + return: "DECIMAL?" diff --git a/site/docs/expressions/Extension_Functions/_config b/site/docs/expressions/Extension_Functions/_config new file mode 100644 index 000000000..2b737f30d --- /dev/null +++ b/site/docs/expressions/Extension_Functions/_config @@ -0,0 +1,6 @@ +arrange: + - index.md + - scalar_functions.md + - aggregate_functions.md + - window_functions.md + - table_functions.md diff --git a/site/docs/expressions/aggregate_functions.md b/site/docs/expressions/Extension_Functions/aggregate_functions.md similarity index 54% rename from site/docs/expressions/aggregate_functions.md rename to site/docs/expressions/Extension_Functions/aggregate_functions.md index e40b61a9a..a43cfa292 100644 --- a/site/docs/expressions/aggregate_functions.md +++ b/site/docs/expressions/Extension_Functions/aggregate_functions.md @@ -13,8 +13,6 @@ Aggregate function signatures contain all the properties defined for [scalar fun | Intermediate Output Type | If the function is decomposable, represents the intermediate output type that is used, if the function is defined as either `ONE` or `MANY` decomposable. Will be a struct in many cases. | Required for `ONE` and `MANY`. | | Invocation | Whether the function uses all or only distinct values in the aggregation calculation. Valid options are: `ALL`, `DISTINCT`. | Optional, defaults to `ALL` | - - ## Aggregate Binding When binding an aggregate function, the binding must include the following additional properties beyond the standard scalar binding properties: @@ -24,3 +22,31 @@ When binding an aggregate function, the binding must include the following addit | Phase | Describes the input type of the data: [INITIAL_TO_INTERMEDIATE, INTERMEDIATE_TO_INTERMEDIATE, INITIAL_TO_RESULT, INTERMEDIATE_TO_RESULT] describing what portion of the operation is required. For functions that are NOT decomposable, the only valid option will be INITIAL_TO_RESULT. | | Ordering | Zero or more ordering keys along with key order (ASC\|DESC\|NULL FIRST, etc.), declared similar to the sort keys in an `ORDER BY` relational operation. If no sorts are specified, the records are not sorted prior to being passed to the aggregate function. | +When the phase is `*_TO_INTERMEDIATE`, the return type of the aggregate function is overridden to the intermediate type. When the phase is `INTERMEDIATE_TO_*`, non-constant value argument slots are overridden to behave like type argument slots instead, and an extra value argument is expected at the end of the actual argument list, of which the type matches the derived intermediate type exactly. Using the following function as an example: + +``` +min_max_difference(T??) -> STRUCT -> + assert T == i8 || T == i16 || T == i32 || T == i64 + T? +``` + + - `INITIAL_TO_RESULT` would bind to `min_max_difference(i32)` and yield `i32?`; + - `INITIAL_TO_INTERMEDIATE` would bind to `min_max_difference(i32)` and yield `STRUCT`; + - `INTERMEDIATE_TO_INTERMEDIATE` would bind to `min_max_difference(type i32, STRUCT)` and yield `STRUCT`; + - `INTERMEDIATE_TO_RESULT` would bind to `min_max_difference(type i32, STRUCT)` and yield `i32?`; + +!!! note + + The value to type argument replacement is necessary, because the intermediate and return type derivations may depend on it in a nontrivial and (in general) non-reversable way. + +## Pattern Matching and Evaluation Order + +The patterns used to define the argument types and return type are processed in the following order. + + - Match the actual argument types against the argument slot patterns from left to right. The pattern from the last argument slot may be matched any number of times if the function is variadic. For `INTERMEDIATE_TO_*` bindings, the intermediate data input is not matched yet. + - Evaluate any statements in the return type specification from top to bottom/left to right. + - Evaluate the return type pattern (even for `*_TO_INTERMEDIATE` bindings where the result is not used; evaluation may still affect whether the function exists or not). + - Evaluate the intermediate type pattern, if one is specified (even for `INITIAL_TO_RESULT` bindings; note also that it is evaluated and *then* matched even for `INTERMEDIATE_TO_RESULT`). + - For `INTERMEDIATE_TO_*` bindings, match the above evaluation result against the data type passed to the last argument; they must be exactly equal. + +If any pattern fails to match or evaluate, the function is said to not match the given argument pack. diff --git a/site/docs/expressions/Extension_Functions/index.md b/site/docs/expressions/Extension_Functions/index.md new file mode 100644 index 000000000..82cd52ebb --- /dev/null +++ b/site/docs/expressions/Extension_Functions/index.md @@ -0,0 +1,229 @@ +# Basics + +Rather than specifying a wide variety of operators and standard functions to express behavior in expression context, Substrait only specifies a bare minimum of special expression types (field references, literals, casts, a few conditionals, and subqueries), and covers the rest with generalized functions. Function implementations can be embedded in a plan, but more commonly are declared using simple extensions. This allows the implementation of a function's behavior to be decoupled from its behavior specification, such that Substrait plans only have to be concerned with the latter. + +The expectation is that each consumer will use its own extensions to describe exactly the set of functions they support internally. This allows producers to generate plans that are tailored specifically to a particular consumer. + +!!! note + + Outside of Substrait, the term "user-defined function" (or just UDF) refers to a function provided by the *end* user, i.e. the producer. Within Substrait, however, the term "user-defined function" usually refers to a consumer-specific function. This is the exact opposite in practice, so be careful not to confuse the terms! What's normally called a UDF is called an "[embedded function](../embedded_functions.md)" in Substrait, instead. + +Substrait also defines a number of functions of its own. These functions provide primitives like arithmetic operators, inequalities, basic string manipulation, timestamp handling, and so on. While they are *inspired* by the functions provided by existing database management systems, they do not necessarily match any particular implementation exactly; rather, they aim to provide the building blocks needed to represent the behavior of implementations in general. This allows producers to also generate plans that are *not* tailored to any particular consumer yet; rather than defining exactly which physical function implementations they want the consumer to use, they specify the desired behavior, and leave it up to the consumer to choose suitable implementations. + +Supporting *both* consumer-specific and consumer-agnostic plans is useful because it allows generic, vendor-agnostic transformations to be written. Examples might include (cross) compilers, optimizers, and planners. The ability to centralize the development of such tools helps avoid duplication of this logic across vendors. Furthermore, by using the *same* format for consumer-specific and consumer-agnostic plans (and indirectly also supporting everything in between), such tools can also be chained together. A complete pipeline might consist of a SQL parser from vendor A, a generic optimizer from vendor B, a more engine-specific optimizer from vendor C, a generic planner and distribution engine from vendor D, finally resulting in a number of distributed plans to be run on an otherwise single-node query engine from vendor C, all managed by a framework from vendor E. The vendors can now focus on what they're good at, rather than having to do a little bit of everything, and the user can swap out bits of the pipeline as their needs change without breaking their workflow. + +For all of this to work and actually be generic, though, the generic tools need to be able to manipulate consumer-specific functions, at least to some degree. For example, in order to do common subexpression elimination, a transformation tool needs to know whether the functions used in the common subexpression are deterministic. If consumer-specific functions are basically black boxes, the tool would not be able to make this assumption. Thus, Substrait requires all function extensions to be declared using [simple extensions](../../extensions/#simple-extensions). Even the consumer-agnostic functions defined by Substrait are published as extensions; we call these the "core" extensions. The expectation is that a plan that uses only core extensions is consumer-agnostic, although a consumer may not support everything supported by core Substrait without some compilation happening in between. + +While a function declaration *may* include everything down to the implementation of the function (expressed in formats such as WebAssembly or a pickled Python function), this is not required, and as such the provided information is usually not enough for a *machine* to execute the function. However, all behavior that cannot easily be expressed generically in a machine-readable way can instead be covered using descriptions written in natural language. This also makes the extensions useful as a source of documentation. This documentation use case is also the primary reason why simple extension files are normally written in a human-friendly YAML format, rather than in a more machine-friendly format like protobuf. + +!!! note + + The first producer and last consumer in the pipeline typically don't need to use the extension files for anything. The information contained in them is redundant, because the consumer already needs an innate understanding of what the functions do in order to implement them, and the producer typically needs to know what the functions do in order to use them. In fact, the extension files need not necessarily even exist for a compatible consumer/producer pair to exchange a Substrait plan successfully. That being said, it is strongly recommended for consumers to publish simple extension files, if only for the purpose of documentation. + +Of particular importance for transformation tools is the ability to do type propagation, and to determine whether a candidate function is usable in a certain context. Therefore, extension files must include machine-readable specifications for constraints on the use of a function, and for the data type returned by a function. The majority of this is described using a domain-specific mini-language, operating on a vastly simplified type system that we refer to as the [meta type system](../../types/meta_type_system.md) (named such because data types are a *value* in that system, allowing them to be manipulated). In broad terms, the programs formed using this mini language are used to match the data types of an incoming argument pack against a number of patterns, and ultimately evaluate the data type returned by the function based on how these patterns matched. + +!!! note + + While the system is *very* minimalistic compared to a typical programming language, the exact specifications of programming languages tend to be hard to read, and this is no exception. Since the majority of our users need not concern themselves with the implementation, we will describe the commonly-used features using intuitive language and examples in the following subsections. When in doubt, refer to the more in-depth specification in the [Meta Type System](../../types/meta_type_system.md) section. The definitions in that section, and in particular the ANTLR grammar specification, are leading in case of conflict. + +## Function Declarations and Identification + +As stated, functions are specified using [simple extension files](../../extensions/#simple-extensions). In order for a plan to bind a function, it must thus somehow refer to the extension file and to a particular function declared within it. The former is always done using a URI within Substrait; the latter is done via case-insensitive name matching. A function name can be any non-empty UTF-8 string. + +In addition to the name, a function declaration typically includes a behavior description written in a natural language (typically English). The purpose of this description is to capture all behavior of the function that the declaration cannot describe in a machine-readable way. For example, Substrait does not provide a structured way to indicate that a particular function adds two numbers or matches a string against a regular expression. Its purpose is similar to that of a docstring; in conjunction with the name and structured behavior specifications, it should be clear to a person how the function behaves for all supported inputs. + +Substrait allows multiple function declarations within a single extension to share the same simple name. In this case, functions must be referred to by their [compound name](../../extensions/#function-signature-compound-names). The compound name is derived from the data type patterns provided by the function declaration. The compound names resulting from this derivation must be unique among all function declarations within the scope of an extension file. In addition to the above, a single function declaration may accept a variety of argument packs. The reason for this complexity is to be able to describe functions and operators that exist within the comparatively weak type systems typically employed by query engines. + +For example, a simple function that adds two `i8` numbers in a source language where nullability is not a first-class concept actually requires two implementations, one nullable (`add(i8?, i8?) -> i8?`) and one non-nullable (`add(i8, i8) -> i8`). This captures the behavior that if neither input can be nullable, the output can never be nullable either, but if either input is null, the output will be null. Because of how common this pattern is, Substrait only requires you to specify the `add(i8, i8) -> i8` variant, and by default automatically derives the nullable variant, and in fact also the mixed cases (`add(i8?, i8) -> i8?` and `add(i8, i8?) -> i8?`), so you don't need to promote a non-nullable type to a nullable type using a cast to match the other argument. We call this `MIRROR` nullability. + +Variadic functions are another example. We might for example want to define our `i8` addition function such that it can take any number of arguments. This definition technically matches an infinite number of argument packs, especially when we combine it with `MIRROR` nullability: the number of implementations is then exponential with the maximum number of arguments. + +Of course, addition is normally defined on all numeric types, not just `i8`, and some of these types have much more complicated rules. Take decimal addition as an example: + +``` +add(decimal, decimal) -> + init_scale = max(S1,S2) + init_prec = init_scale + max(P1 - S1, P2 - S2) + 1 + min_scale = min(init_scale, 6) + delta = init_prec - 38 + prec = min(init_prec, 38) + scale_after_borrow = max(init_scale - delta, min_scale) + scale = if init_prec > 38 then scale_after_borrow else init_scale + DECIMAL +``` + +While it's probably possible to construct constraints for a single function declaration that cover a whole variety of numeric types in one go, it often makes more sense to describe the individual types independenty. This makes it easier for a person to understand the extension, and probably more closely matches the implementation anyway: an execution engine is likely to use a completely different implementation for adding integers than it does for adding floating-point numbers or decimals, but will likely have a single implementation that covers all variations of decimals. + +!!! note + + The above means that the compound name of a function does *not* in general uniquely identify the argument pack expected by a function; it merely identifies a declaration. The declaration may be as generic as `function(T...) -> T`, which, depending on the nullability and variadic behavior specifications, may support *any* argument pack. Nevertheless, the compound name for this function will be `function:any`, regardless of the actual arguments bound to the function. Phrased differently, it is not possible to determine the compound name of a function using only a simple name and the data types of the bound arguments; you need the actual list of declarations and their argument patterns for that as well. + +## Function Types + +Substrait functions are distinguished by the vector/scalar nature of their inputs and outputs. This leads to four function types. + +| Function type | Input | Output | Contexts | +|-------------------------------------|--------|--------|--------------------------------------------------------------------------------| +| [Scalar](scalar_functions.md) | Scalar | Scalar | Scalar expressions, evaluated piecewise. | +| [Aggregate](aggregate_functions.md) | Vector | Scalar | Measures in aggregation relations. | +| [Window](window_functions.md) | Vector | Vector | Scalar expressions, including context from some number of neighboring records. | +| [Table](table_functions.md) | Scalar | Vector | TBD | + +The properties specific to these function types are described in more detail in the linked sections. + +!!! note + + Do not confuse the scalar/vector nature of a function with the nestedness of the argument and return types. A scalar function may operate on or return `LIST` types, for example, and still be called scalar in this context. Scalar vs. vector refers to whether the function implicitly operates on/returns just one instance of the specified data types, or multiple of them. + +## Arguments + +When a function is used in a plan, it is bound to an argument pack, consisting of zero or more individual arguments. Currently, all arguments are positionally matched and mandatory; that is, the number of arguments bound to a function must match the number of argument slots in the declaration exactly, except when the last argument slot is variadic. + +There are four main types of arguments: value arguments, type arguments, required enumerations, and optional enumerations. These are described in detail in the following subsections. + +### Value Arguments + +Value arguments are arguments that refer to a data value. These could be constants (literal expressions defined in the plan) or variables (a reference expression that references data being processed by the plan). This is the most common type of argument. The data type expected by the argument is defined using a [data type pattern](#data-type-patterns). + +| Property | Description | Required | +|----------|-------------------------------------------------|----------| +| Metadata | See [common metadata](#common-metadata). | Optional | +| Pattern | A [data type pattern](#data-type-patterns). | Required | +| Constant | Whether this argument is required to be a constant for invocation. For example, in some system a regular expression pattern would only be accepted as a literal and not a column value reference. | Optional, defaults to false | + +!!! note + + The *value* of the argument is not usable for return type derivations and constraints, even for constant value arguments. For example, it is not currently possible to assert that a particular function only accepts `i8` values between 0 and 100, nor is it possible to declare a function like `new_zero_decimal(P: i8, S: i8) -> DECIMAL`. + +!!! note + + Constant value arguments effectively always act like scalars, even for function types that take vectors as input. For example, an aggregate function like `count_matching(regex: const string, values: string) -> i64` would always only need to compile the passed regular expression once, while `count_matching(regex: string, values: string) -> i64` is expected to work even if the regex varies from record to record. + +### Type Arguments + +Type arguments are arguments that are used only to inform the evaluation and/or type derivation of the function. For example, you might have a function which is `truncate(type DECIMAL, DECIMAL, i32)`. This function declares two value arguments and a type argument. The difference between them is that the type argument has no value at runtime, while the value arguments do. + +| Property | Description | Required | +|----------|-------------------------------------------------|----------| +| Metadata | See [common metadata](#common-metadata). | Optional | +| Pattern | A [data type pattern](#data-type-patterns). | Required | + +### Required Enumeration Arguments + +Required enumerations are arguments that support a fixed set of declared values as constant arguments. These arguments must be specified as part of an expression. While these could also have been implemented as constant string value arguments, they are formally included to improve validation/contextual help/etc. for frontend processors and IDEs. An example might use might be `extract({DAY, YEAR, MONTH}, date) -> i32`. In this example, a producer must specify a type of date part to extract. Note that the value of a required enumeration cannot be used in return type derivation. + +| Property | Description | Required | +|----------|-------------------------------------------------|----------| +| Metadata | See [common metadata](#common-metadata). | Optional | +| Options | List of valid string options for this argument. | Required | + +### Optional Enumeration Arguments + +Optional enumeration arguments are similar to required enumeration arguments, but are more focused on supporting alternative behaviors, usually for corner cases. An optional enumeration always includes an "unspecified" default option that can be bound based on the capabilities of the plan consumer. When a plan does not specify a behavior, the consumer is expected to resolve the option based on the first option the system can match. An example use case might be `OVERFLOW_BEHAVIOR: {OVERFLOW, SATURATE, ERROR}`. If unspecified, an engine would use the first of these that it implements. If specified, the engine would be expected to behave as specified or fail. Note that the value of an optional enumeration cannot be used in type derivation. + +| Property | Description | Required | +|----------|-------------------------------------------------|----------| +| Metadata | See [common metadata](#common-metadata). | Optional | +| Options | Priority-ordered list of valid string options for this argument. The pseudo-option will be the default "value" for the enumeration unless a binding specifies a specific value. | Required | + +!!! note + + Despite being classified as "optional," optional enumeration argument slots must still be bound; unlike required enumeration arguments, however, they may be bound to a special "unspecified" value. + +### Common Metadata + +All arguments share the same metadata properties. + +| Property | Description | Required | +|-------------|--------------------------------------------------------------|----------| +| Name | A human-readable name for this argument to help clarify use. | Optional | +| Description | Additional multi-line information about the argument. | Optional | + +### Data Type Patterns + +Data type patterns are used to constrain the data types supported by value and type arguments, and to bind (parts of them) to names, to be reused by later patterns or the return type derivation. + +For the most part, data type patterns are a generalization of the [syntax](../../types/meta_type_system/#syntax-parsing) Substrait uses to describe concrete types, that also allows types to be specified partially, by replacing a type class name or a parameter with an otherwise unused name. This tries to bind said name to the part of the pattern that was matched, while only allowing the name to be bound to one single partial value or type. For example, a function like `concatenate(FIXEDCHAR, FIXEDCHAR) -> FIXEDCHAR` will bind `A` to the length of the first `FIXEDCHAR` and `B` to the length of the second. If instead we'd write something like `is_equal(FIXEDCHAR, FIXEDCHAR) -> boolean` however, `A` is bound to *both* the lengths, and thus the lengths of both arguments must be equal. In extreme cases, we might replace an entire data type with a letter; for example, in `coalesce(T, T) -> T`, both arguments must have the same data type (though there may be exceptions for [nullability](#nullability)), but any data type will match. + +The above examples probably cover 99.9% of all practical function declarations. Refer to the section on [metapatterns](../../types/meta_type_system/#metapatterns) for a more precise description or for the remaining 0.1%. + +### Nullability + +Most SQL-inspired functions, in particular scalar functions, are defined to return null if any only if any argument is null. While this behavior could be implemented in Substrait by accepting only nullable arguments and always returning a nullable data type, we can do better than that: we can define a function such that it accepts any combination of nullabilities for its arguments, and returns nullable if and only if any argument is nullable. This is worth doing because knowing that something can never be nullable may open up possibilities for optimizations. We call this `MIRROR` nullability. + +Another common pattern is for a function to accept any combination of nullabilities for its arguments, but always return a nullable or non-nullable result. Most SQL-inspired aggregate functions behave like the former; they ignore nulls at the input, and return null if the input is empty. A function like `is_null(T) -> boolean` is an example of the latter; it will by definition always return true or false. We call this `DECLARED_OUTPUT` nullability. + +Because it is rather cumbersome to write these behaviors down using the metapattern system, the YAML format provides syntactic sugar for it by way of the nullability mode parameter. + +| Nullability mode | Desugaring behavior | +|--------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `MIRROR` (default) | Modify nullability of argument and return patterns to accept any combination of nullabilities and return nullable if and only if any argument is nullable. | +| `DECLARED_OUTPUT` | Modify nullability of argument patterns to accept any combination of nullabilities. | +| `DISCRETE` | Do not modify any patterns. | + +The exact effect of the first two modes is that the nullability suffix of any toplevel data type or binding pattern used in an argument is replaced with `??nullable`. For `MIRROR`, the same is also done for the return type. For example, the function `coalesce(T, T) -> T` with `MIRROR` nullability means the same as `coalesce(T??nullable, T??nullable) -> T??nullable` with `DISCRETE` nullability. In this pattern, the [nullability suffix](../../types/meta_type_system/#nullability-suffixes) in `T?X` means "match data types with nullability `X` (expressed as a boolean)," and the [inconsistent binding pattern](../../types/meta_type_system/#bindings) `?nullable` in [this context](../../types/meta_type_system/#mirror-and-declared-output-nullability) means "match any boolean, or return the boolean OR of any booleans matched previously." + +Note that, armed with this knowledge, we can do much better for coalesce and define it as follows: `coalesce(T?, T?nullable) -> T?nullable`. Now the first argument must always be nullable (otherwise the function would be no-op) and the function is specified to only return a nullable type if the inputs can both be null. We have to use `DISCRETE` nullability here though, or the default `MIRROR` behavior will cause our nullability suffixes to be ignored. + +### Variadics + +Some functions are variadic in such a way that it's either impractical to specify all argument pack variations exhaustively, or downright impossible because there is no well-defined upper limit. An example is a string formatting function that takes a format argument, followed by any number of subsequent arguments (corresponding to the format string). We might want to write something like this: `string_format(const string, T...) -> string`. + +To support declaration of such functions, Substrait allows the last argument *slot* of the function to bind any number of *actual* arguments. The slot will match one or more actual arguments unless otherwise specified, but a custom minimum (including 0) and maximum number can be specified as well. + +When the last argument slot uses a binding like `T`, we need some extra information: does our `string_format` declaration above require all the actual arguments mapping to `T` to be the *same* `T`? That's probably not what we meant for `string_format`, but it probably *is* what we mean for `coalesce(T...) -> T`. To make it easier to specify the difference, we can mark a variadic function as either `CONSISTENT` (like `coalesce`) or `INCONSISTENT` (like `string_format`). + +| Consistency | Intuitive behavior | Desugaring behavior | +|------------------------|-------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------| +| `CONSISTENT` (default) | Names used in the variadic argument slot refer to the same type/value. | Do not modify any patterns. | +| `INCONSISTENT` | Names used in the variadic argument slot can refer to different types/values. | Replace all consistent bindings used in the last argument slot with inconsistent bindings. | + +Just like the nullability mode, we can also just write what we mean directly with patterns, the syntax just gets a little weird. In this case, the `string_format` declaration above could also be written as `string_format(const string, ?T...) -> string` explicitly to avoid needing the desugaring behavior; the `?` in front of the `T` means that the first usage of `T` will bind it, but later matches will ignore the previously bound value or type and match anything that the pattern would have matched if it had not been bound yet. The exact behavior is specified [here](../../types/meta_type_system/#inconsistent-variadic-argumentparameter-slots). + +## Return Values + +Functions in Substrait return either a scalar or vector (depending on the [function type](#function-types)) of a single data type, and it must be possible to unambiguously derive what this data type is when given only the actual argument types and a particular function declaration. This derivation is accomplished with the same syntax as the argument data type patterns, but rather than matching an incoming data type, we use the pattern to generate a data type. + +Note that this imposes some constraints on the allowed patterns. For example, you can't define a function like `cast(T) -> S` because `S` is not yet defined and is therefore ambiguous. Most patterns have well-defined semantics in both match and evaluation context, though. + +### Expressions + +For some functions, such as those operating on `FIXEDCHAR` or `DECIMAL` types, we may need to do some math to derive the return type. Here's a simple example: `concatenate(FIXEDCHAR, FIXEDCHAR) -> FIXEDCHAR`. + +To keep things simple, Substrait defines only a basic set of operators, functions, and metatypes for type derivations, and this set of functions is not extensible. The complete list can be found [here](../../types/meta_type_system/#functions). + +!!! note + + These functions are not to be confused with extension functions or embedded functions, or anything else that exists when a plan is being executed. They exist on a completely different abstraction layer! In fact, we need these functions in order to be able to even begin thinking about defining the more complex functions that we use in plans, and we can't use functions to define themselves. + +## Derivation Programs + +For more advanced functions, like the decimal addition we used as an example earlier, a single pattern is just not enough to derive the return type. For this purpose, the return type can include a number of statements before the final pattern, separated from each other and the pattern by newlines or semicolons. These statements more or less look and behave like assignment statements, except that the left-hand side can be any pattern, and the assignment utilizes the same matching process used for arguments. + +!!! warning + + The semantics for matching mean that the names used in these assignment statements don't act like variables when they are reused! For example, `bad_concatenate(FIXEDCHAR, FIXEDCHAR) -> A = A + B; FIXEDCHAR` will *not* behave like the `concatenate` example from before, because unless `B` equals 0, the statement would fail to match `A` against its intended new value. + +This simple rule not only lets us eliminate common subexpressions to make the decimal `add` somewhat comprehensible, but it also lets us write essentially arbitrary constraints. For example, we could define an integer addition that works with any combination of integers as follows: + +``` +casting_add(LHS, RHS, type RETURN) -> + true = LHS == i8 || LHS == i16 || LHS == i32 || LHS == i64 + true = RHS == i8 || RHS == i16 || RHS == i32 || RHS == i64 + true = RETURN == i8 || RETURN == i16 || RETURN == i32 || RETURN == i64 + RETURN +``` + +!!! note + + We *have* to specify the return type as a type argument here, because otherwise the return type would be ambiguous. + +The `true = ...` pattern works because the `true` pattern *matches* `true`, so as long as we assign something that is also true, the match will succeed. However, because this pattern looks rather odd and is relatively common, we also have some syntactic sugar for it: `assert ...`. So, we can rewrite the above declaration a bit nicer as follows: + +``` +casting_add(LHS, RHS, type RETURN) -> + assert LHS == i8 || LHS == i16 || LHS == i32 || LHS == i64 + assert RHS == i8 || RHS == i16 || RHS == i32 || RHS == i64 + assert RETURN == i8 || RETURN == i16 || RETURN == i32 || RETURN == i64 + RETURN +``` diff --git a/site/docs/expressions/Extension_Functions/scalar_functions.md b/site/docs/expressions/Extension_Functions/scalar_functions.md new file mode 100644 index 000000000..81ab7e1c8 --- /dev/null +++ b/site/docs/expressions/Extension_Functions/scalar_functions.md @@ -0,0 +1,25 @@ +# Scalar Functions + +A function is a scalar function if that function takes in values from a single record and produces an output value. To clearly specify the definition of functions, Substrait declares an extensible specification plus binding approach to function resolution. + +A scalar function implementation includes the following properties. + +| Property | Description | Required | +|------------------------|--------------------------------------------------------------------------------------------------------------------------------|-------------------------------------| +| Name | The UTF-8 string that is used to case-insensitively reference this function. | Required | +| Description | Additional description of function for implementers or users. Should be written human-readable to allow exposure to end users. | Optional | +| Arguments | As defined [here](index.md#arguments). | Optional, defaults to niladic | +| Return type | As defined [here](index.md#return-values). | Required | +| Deterministic | Whether this function is expected to reproduce the same output when it is invoked multiple times with the same input. This informs a plan consumer on whether it can constant-reduce the defined function. An example would be a random() function, which is typically expected to be evaluated repeatedly despite having the same set of inputs. | Optional, defaults to true | +| Session-dependent | Whether this function is influenced by the session context it is invoked within. For example, a function may be influenced by a user who is invoking the function, the time zone of a session, or some other non-obvious parameter. This can inform caching systems on whether a particular function is cacheable. | Optional, defaults to false | +| Implementation Map | A map of implementation locations for one or more implementations of the given function. Each key is a function implementation type. Implementation types include examples such as: AthenaArrowLambda, TrinoV361Jar, ArrowCppKernelEnum, GandivaEnum, LinkedIn Transport Jar, etc. [Definition TBD]. Implementation type has one or more properties associated with retrieval of that implementation. | Optional | + +## Pattern Matching and Evaluation Order + +The patterns used to define the argument types and return type are processed in the following order. + + - Match the actual argument types against the argument slot patterns from left to right. The pattern from the last argument slot may be matched any number of times if the function is variadic. + - Evaluate any statements in the return type specification from top to bottom/left to right. + - Evaluate the return type pattern. + +If any pattern fails to match or evaluate, the function is said to not match the given argument pack. diff --git a/site/docs/expressions/table_functions.md b/site/docs/expressions/Extension_Functions/table_functions.md similarity index 100% rename from site/docs/expressions/table_functions.md rename to site/docs/expressions/Extension_Functions/table_functions.md diff --git a/site/docs/expressions/window_functions.md b/site/docs/expressions/Extension_Functions/window_functions.md similarity index 94% rename from site/docs/expressions/window_functions.md rename to site/docs/expressions/Extension_Functions/window_functions.md index 4f5a9130f..a613721af 100644 --- a/site/docs/expressions/window_functions.md +++ b/site/docs/expressions/Extension_Functions/window_functions.md @@ -2,8 +2,6 @@ Window functions are functions which consume values from multiple records to produce a single output. They are similar to aggregate functions, but also have a focused window of analysis to compare to their partition window. Window functions are similar to scalar values to an end user, producing a single value for each input record. However, the consumption visibility for the production of each single record can be many records. - - Window function signatures contain all the properties defined for [aggregate functions](aggregate_functions.md). Additionally, they contain the properties below | Property | Description | Required | @@ -11,8 +9,6 @@ Window function signatures contain all the properties defined for [aggregate fun | Inherits | All properties defined for aggregate functions. | N/A | | Window Type | STREAMING or PARTITION. Describes whether the function needs to see all data for the specific partition operation simultaneously. Operations like SUM can produce values in a streaming manner with no complete visibility of the partition. NTILE requires visibility of the entire partition before it can start producing values. | Optional, defaults to PARTITION | - - When binding an aggregate function, the binding must include the following additional properties beyond the standard scalar binding properties: | Property | Description | Required | @@ -21,5 +17,6 @@ When binding an aggregate function, the binding must include the following addit | Lower Bound | Bound Following(int64), Bound Trailing(int64) or CurrentRow. | False, defaults to start of partition | | Upper Bound | Bound Following(int64), Bound Trailing(int64) or CurrentRow. | False, defaults to end of partition | +## Pattern Matching and Evaluation Order - +The pattern matching and evaluation order matches that of aggregate functions. diff --git a/site/docs/expressions/_config b/site/docs/expressions/_config index b2aaf9179..73bf38504 100644 --- a/site/docs/expressions/_config +++ b/site/docs/expressions/_config @@ -1,9 +1,6 @@ arrange: - field_references.md - - scalar_functions.md - - aggregate_functions.md - - specialized_record_expressions.md - - window_functions.md - - table_functions.md - - user_defined_functions.md + - Extension_Functions - embedded_functions.md + - specialized_record_expressions.md + - subqueries.md diff --git a/site/docs/expressions/scalar_functions.md b/site/docs/expressions/scalar_functions.md deleted file mode 100644 index d3315ed9c..000000000 --- a/site/docs/expressions/scalar_functions.md +++ /dev/null @@ -1,134 +0,0 @@ -# Scalar Functions - -A function is a scalar function if that function takes in values from a single record and produces an output value. To clearly specify the definition of functions, Substrait declares an extensible specification plus binding approach to function resolution. A scalar function signature includes the following properties: - -| Property | Description | Required | -| ---------------------- | ------------------------------------------------------------ | ----------------------------------- | -| Name | One or more user-friendly UTF-8 strings that are used to reference this function. | At least one value is required. | -| List of arguments | Argument properties are defined below. Arguments can be fully defined or calculated with a type expression. See further details below. | Optional, defaults to niladic. | -| Deterministic | Whether this function is expected to reproduce the same output when it is invoked multiple times with the same input. This informs a plan consumer on whether it can constant-reduce the defined function. An example would be a random() function, which is typically expected to be evaluated repeatedly despite having the same set of inputs. | Optional, defaults to true. | -| Session Dependent | Whether this function is influenced by the session context it is invoked within. For example, a function may be influenced by a user who is invoking the function, the time zone of a session, or some other non-obvious parameter. This can inform caching systems on whether a particular function is cacheable. | Optional, defaults to false. | -| Variadic Behavior | Whether the last argument of the function is variadic or a single argument. If variadic, the argument can optionally have a lower bound (minimum number of instances) and an upper bound (maximum number of instances). | Optional, defaults to single value. | -| Nullability Handling | Describes how nullability of input arguments maps to nullability of output arguments. Three options are: `MIRROR`, `DECLARED_OUTPUT` and `DISCRETE`. More details about nullability handling are listed below. | Optional, defaults to `MIRROR` | -| Description | Additional description of function for implementers or users. Should be written human-readable to allow exposure to end users. Presented as a map with language => description mappings. E.g. `{ "en": "This adds two numbers together.", "fr": "cela ajoute deux nombres"}`. | Optional | -| Return Value | The output type of the expression. Return types can be expressed as a fully-defined type or a type expression. See below for more on type expressions. | Required | -| Implementation Map | A map of implementation locations for one or more implementations of the given function. Each key is a function implementation type. Implementation types include examples such as: AthenaArrowLambda, TrinoV361Jar, ArrowCppKernelEnum, GandivaEnum, LinkedIn Transport Jar, etc. [Definition TBD]. Implementation type has one or more properties associated with retrieval of that implementation. | Optional | - - - -## Argument Types - -There are four main types of arguments: value arguments, type arguments, required enumerations, and optional enumerations. - -* Value arguments: arguments that refer to a data value. These could be constants (literal expressions defined in the plan) or variables (a reference expression that references data being processed by the plan). This is the most common type of argument. The value of a value argument is not available in output derivation, but its type is. Value arguments can be declared in one of two ways: concrete or parameterized. Concrete types are either simple types or compound types with all parameters fully defined (without referencing any type arguments). Examples include `i32`, `fp32`, `VARCHAR<20>`, `List`, etc. Parameterized types are discussed further below. -* Type arguments: arguments that are used only to inform the evaluation and/or type derivation of the function. For example, you might have a function which is `truncate( DECIMAL, DECIMAL, i32)`. This function declares two value arguments and a type argument. The difference between them is that the type argument has no value at runtime, while the value arguments do. -* Required enumeration: arguments that support a fixed set of declared values as constant arguments. These arguments must be specified as part of an expression. While these could also have been implemented as constant string value arguments, they are formally included to improve validation/contextual help/etc. for frontend processors and IDEs. An example might use might be `extract([DAY|YEAR|MONTH], )`. In this example, a producer must specify a type of date part to extract. Note, the value of a required enumeration cannot be used in type derivation. -* Optional enumeration: similar to required enumeration, but more focused on supporting alternative behaviors. An optional enumeration always includes an "unspecified" default option that can be bound based on the capabilities of the plan consumer. When a plan does not specify a behavior, the consumer is expected to resolve the option based on the first option the system can match. An example use case might be `OVERFLOW_BEHAVIOR:[OVERFLOW, SATURATE, ERROR]` If unspecified, an engine would use the first of these that it implements. If specified, the engine would be expected to behave as specified or fail. Note, the value of an optional enumeration cannot be used in type derivation. - -#### Value Argument Properties - -| Property | Description | Required | -| -------- | ------------------------------------------------------------ | ---------------------------------------------------------- | -| Name | A human-readable name for this argument to help clarify use. | Optional, defaults to a name based on position (e.g. `arg0`) | -| Type | A fully defined type or a type expression. | Required | -| Constant | Whether this argument is required to be a constant for invocation. For example, in some system a regular expression pattern would only be accepted as a literal and not a column value reference. | Optional, defaults to false | - -#### Type Argument Properties - -| Property | Description | Required | -| -------- | ------------------------------------------------------------------- | ---------------------------------------------------------- | -| Type | A partially or completely parameterized type. E.g. `List` or `K` | Required | -| Name | A human-readable name for this argument to help clarify use. | Optional, defaults to a name based on position (e.g. `arg0`) | - -#### Required Enumeration Properties - -| Property | Description | Required | -| -------- | ------------------------------------------------------------ | ------------------------------------------------------------ | -| Options | List of valid string options for this argument | Required | -| Name | A human-readable name for this argument to help clarify use. | Optional, defaults to a name based on position (e.g. `arg0`) | - -#### Optional Enumeration Properties - -| Property | Description | Required | -| -------- | ------------------------------------------------------------ | ------------------------------------------------------------ | -| Options | Priority-ordered list of valid string options for this argument. The pseudo-option will be the default "value" for the enumeration unless a binding specifies a specific value. | Required | -| Name | A human-readable name for this argument to help clarify use. | Optional, defaults to a name based on position (e.g. `arg0`) | - - - -### Nullability Handling - -| Mode | Description | -| --------------- | ------------------------------------------------------------ | -| MIRROR | This means that the function has the behavior that if at least one of the input arguments are nullable, the return type is also nullable. If all arguments are non-nullable, the return type will be non-nullable. An example might be the `+` function. | -| DECLARED_OUTPUT | Input arguments are accepted of any mix of nullability. The nullability of the output function is whatever the return type expression states. Example use might be the function `is_null()` where the output is always `boolean` independent of the nullability of the input. | -| DISCRETE | The input and arguments all define concrete nullability and can only be bound to the types that have those nullability. For example, if a type input is declared `i64?` and one has an `i64` literal, the `i64` literal must be specifically cast to `i64?` to allow the operation to bind. | - - - -### Parameterized Types - -Types are parameterized by two types of values: by inner types (e.g. `List`) and numeric values (e.g. `DECIMAL`). Parameter names are simple strings (frequently a single character). There are two types of parameters: integer parameters and type parameters. - -When the same parameter name is used multiple times in a function definition, the function can only bind if the exact same value is used for all parameters of that name. For example, if one had a function with a signature of `fn(VARCHAR, VARCHAR)`, the function would be only be usable if both `VARCHAR` types had the same length value `N`. This necessitates that all instances of the same parameter name must be of the same parameter type (all instances are a type parameter or all instances are an integer parameter). - -#### Type Parameter Resolution in Variadic Functions - -When the last argument of a function is variadic and declares a type parameter e.g. `fn(A, B, C...)`, the C parameter can be marked as either consistent or inconsistent. If marked as consistent, the function can only be bound to arguments where all the C types are the same concrete type. If marked as inconsistent, each unique C can be bound to a different type within the constraints of what T allows. - - - -## Output Type Derivation - -### Concrete Return Types - -A concrete return type is one that is fully known at function definition time. Example simple concrete return types would be things such as `i32`, `fp32`. For compound types, a concrete return type must be fully declared. Example of fully defined compound types: `VARCHAR<20>`, `DECIMAL<25,5>` - -### Return Type Expressions - -Any function can declare a return type expression. A return type expression uses a simplified set of expressions to describe how the return type should be returned. For example, a return expression could be as simple as the return of parameter declared in the arguments. For example `f(List) => K` or can be a simple mathematical or conditional expression such as `add(decimal, decimal) => decimal`. For the simple expression language, there is a very narrow set of types: - -* Integer: 64-bit signed integer (can be a literal or a parameter value) -* Boolean: True and False -* Type: A Substrait type (with possibly additional embedded expressions) - -These types are evaluated using a small set of operations to support common scenarios. List of valid operations: - -``` -Math: +, -, *, /, min, max -Boolean: &&, ||, !, <, >, == -Parameters: type, integer -Literals: type, integer -``` - -Fully defined with argument types: - -* `type_parameter(string name) => type` -* `integer_parameter(string name) => integer` -* `not(boolean x) => boolean` -* `and(boolean a, boolean b) => boolean` -* `or(boolean a, boolean b) => boolean` -* `multiply(integer a, integer b) => integer` -* `divide(integer a, integer b) => integer` -* `add(integer a, integer b) => integer` -* `subtract(integer a, integer b) => integer` -* `min(integer a, integer b) => integer` -* `max(integer a, integer b) => integer` -* `equal(integer a, integer b) => boolean` -* `greater_than(integer a, integer b) => boolean` -* `less_than(integer a, integer b) => boolean` -* `covers(Type a, Type b) => boolean` Covers means that type b matches type A for as much as type B is defined. For example, if type A is `VARCHAR<20>` and type B is `VARCHAR`, type B would be considered covering. Similarlily if type A was `List>`and type B was `List>`, it would be considered covering. Note that this is directional "as in B covers A" or "B can be further enhanced to match the definition A". -* `if(boolean a) then (integer) else (integer)` -* `if(boolean a) then (type) else (type)` - -#### Example Type Expressions - -For reference, here are are some common output type derivations and how they can be expressed with a return type expression: - -| Operation | Definition | -| ------------------------------------------------------------ | ------------------------------------------------------------ | -| Add item to list | `add(, T>) => List` | -| Decimal Division | `divide(Decimal, Decimal) => Decimal` | -| Select a subset of map keys based on a regular expression (requires stringlike keys) | `extract_values(regex:string, map:Map) => List WHERE K IN [STRING, VARCHAR, FIXEDCHAR]` | -| Concatenate two fixed sized character strings | `concat(FIXEDCHAR, FIXEDCHAR) => FIXEDCHAR` | -| Make a struct of a set of fields and a struct definition. | `make_struct( T, K...) => T` | diff --git a/site/docs/expressions/user_defined_functions.md b/site/docs/expressions/user_defined_functions.md deleted file mode 100644 index c5c23031e..000000000 --- a/site/docs/expressions/user_defined_functions.md +++ /dev/null @@ -1,3 +0,0 @@ -# User-Defined Functions - -Substrait supports the creation of custom functions using [simple extensions](../extensions/index.md#simple-extensions), using the facilities described in [scalar functions](scalar_functions.md). In fact, the functions defined by Substrait use the same mechanism. The extension files for them can be found [here](https://github.com/substrait-io/substrait/tree/main/extensions). diff --git a/site/docs/extensions/index.md b/site/docs/extensions/index.md index e75b56bc6..dae42adc5 100644 --- a/site/docs/extensions/index.md +++ b/site/docs/extensions/index.md @@ -21,7 +21,7 @@ A Substrait plan can reference one or more YAML files via URI for extension. In | ------------------ | ------------------------------------------------------------ | | Type | The name as defined on the type object. | | Type Variation | The name as defined on the type variation object. | -| Function Signature | In a specific YAML, if there is only one function implementation with a specific name, a extension type declaration can reference the function using either simple or compound references. Simple references are simply the name of the function (e.g. `add`). Compound references (e.g. `add:i8_i8`)are described below. | +| Function Signature | In a specific YAML, if there is only one function implementation with a specific name, a extension type declaration can reference the function using either simple or compound references. Simple references are simply the name of the function (e.g. `add`). Compound references (e.g. `add:i8_i8`) are described below. | ### Function Signature Compound Names @@ -35,45 +35,52 @@ Rather than using a full data type representation, the input argument types (`sh !!! note -It is required that two function implementation with the same simple name must resolve to different compound names using types. If two function implementations in a YAML file resolve to the same compound name, the YAML file is invalid and behavior is undefined. - -| Argument Type | Signature Name | -| -------------------------- | -------------- | -| Optional Enumeration | opt | -| Required Enumeration | req | -| i8 | i8 | -| i16 | i16 | -| i32 | i32 | -| i64 | i64 | -| fp32 | fp32 | -| fp64 | fp64 | -| string | str | -| binary | vbin | -| timestamp | ts | -| timestamp_tz | tstz | -| date | date | -| time | time | -| interval_year | iyear | -| interval_day | iday | -| uuid | uuid | -| fixedchar<N> | fchar | -| varchar<N> | vchar | -| fixedbinary<N> | fbin | -| decimal<P,S> | dec | -| struct<T1,T2,...,TN> | struct | -| list<T> | list | -| map<K,V> | map | -| any[\d]? | any | -| user defined type | u!name | + It is required that two function implementation with the same simple name must resolve to different compound names using types. If two function implementations in a YAML file resolve to the same compound name, the YAML file is invalid and behavior is undefined. + +| Argument type | Signature name | +|----------------------|----------------| +| Optional enumeration | `opt` | +| Required enumeration | `req` | +| Value | see below | +| Type | see below | + +Value and type arguments receive names based on the type class they match. + +| Type class | Signature name | +|----------------------|----------------| +| `i8` | `i8` | +| `i16` | `i16` | +| `i32` | `i32` | +| `i64` | `i64` | +| `fp32` | `fp32` | +| `fp64` | `fp64` | +| `string` | `str` | +| `binary` | `vbin` | +| `timestamp` | `ts` | +| `timestamp_tz` | `tstz` | +| `date` | `date` | +| `time` | `time` | +| `interval_year` | `iyear` | +| `interval_day` | `iday` | +| `uuid` | `uuid` | +| `fixedchar` | `fchar` | +| `varchar` | `vchar` | +| `fixedbinary` | `fbin` | +| `decimal` | `dec` | +| `struct` | `struct` | +| `list` | `list` | +| `map` | `map` | +| user-defined | `u!` followed by the name as referred to in the extension | +| unspecified | `any` | #### Examples -| Function Signature | Function Name | -| ------------------------------------------------- | ---------------- | -| `add(optional enumeration, i8, i8) => i8` | `add:opt_i8_i8` | -| `avg(fp32) => fp32` | `avg:fp32` | -| `extract(required enumeration, timestamp) => i64` | `extract:req_ts` | -| `sum(any1) => any1` | `sum:any` | +| Function signature | Compound name | +| ---------------------------------------------------|--------------------| +| `add(opt {SILENT, SATURATE, ERROR}, i8, i8) => i8` | `add:opt_i8_i8` | +| `avg(fp32) => fp32` | `avg:fp32` | +| `extract({YEAR, MONTH, DAY}, timestamp) => i64` | `extract:req_ts` | +| `coalesce(T, T) => T` | `coalesce:any_any` | diff --git a/site/docs/types/_config b/site/docs/types/_config index ad75f5784..81e825410 100644 --- a/site/docs/types/_config +++ b/site/docs/types/_config @@ -1,5 +1,5 @@ arrange: - - type_system.md - - type_classes.md - - type_variations.md - - type_parsing.md + - data_type_system.md + - data_type_classes.md + - data_type_variations.md + - meta_type_system.md diff --git a/site/docs/types/type_classes.md b/site/docs/types/data_type_classes.md similarity index 99% rename from site/docs/types/type_classes.md rename to site/docs/types/data_type_classes.md index bcad59dd4..f42eda165 100644 --- a/site/docs/types/type_classes.md +++ b/site/docs/types/data_type_classes.md @@ -1,4 +1,4 @@ -# Type Classes +# Data Type Classes In Substrait, the "class" of a type, not to be confused with the concept from object-oriented programming, defines the set of non-null values that instances of a type may assume. diff --git a/site/docs/types/data_type_system.md b/site/docs/types/data_type_system.md new file mode 100644 index 000000000..f34d459a5 --- /dev/null +++ b/site/docs/types/data_type_system.md @@ -0,0 +1,16 @@ +# Data Type System + +Substrait tries to cover the most common types used in data manipulation. Types beyond this common core may be represented using [simple extensions](../extensions/index.md#simple-extensions). + +Substrait types fundamentally consist of four components: + +| Component | Condition | Examples | Description +| ------------------------------------ | ------------------- | ----------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +| [Class](data_type_classes.md) | Always | `i8`, `string`, `STRUCT`, extensions | Together with the parameter pack, describes the set of non-null values supported by the type. Subdivided into simple and compound type classes. +| Nullability | Always | Either `NULLABLE` (`?` suffix) or `REQUIRED` (no suffix) | Describes whether values of this type can be null. Note that null is considered to be a special value of a nullable type, rather than the only value of a special null type. +| [Variation](data_type_variations.md) | Always | No suffix or explicitly `[0]` (system-preferred), or an extension | Allows different variations of the same type class to exist in a system at a time, usually distinguished by in-memory format. +| Parameters | Compound types only | `<10, 2>` (for `DECIMAL`), `` (for `STRUCT`) | Some combination of zero or more data types or integers. The expected set of parameters and the significance of each parameter depends on the type class. + +Refer to [Syntax Parsing](meta_type_system.md#syntax-parsing) for a description of the syntax used to describe types. + +Note that Substrait employs a strict type system without any coercion rules. All changes in types must be made explicit via [cast expressions](../expressions/specialized_record_expressions.md). diff --git a/site/docs/types/type_variations.md b/site/docs/types/data_type_variations.md similarity index 87% rename from site/docs/types/type_variations.md rename to site/docs/types/data_type_variations.md index 8a2ae4aad..3dbfbda5b 100644 --- a/site/docs/types/type_variations.md +++ b/site/docs/types/data_type_variations.md @@ -1,8 +1,8 @@ -# Type Variations +# Data Type Variations Type variations may be used to represent differences in representation between different consumers. For example, an engine might support dictionary encoding for a string, or could be using either a row-wise or columnar representation of a struct. All variations of a type are expected to have the same semantics when operated on by functions or other expressions. -All variations except the "system-preferred" variation (a.k.a. `[0]`, see [Type Parsing](type_parsing.md)) must be defined using [simple extensions](../extensions/index.md#simple-extensions). The key properties of these variations are: +All variations except the "system-preferred" variation (a.k.a. `[0]`, see [Syntax Parsing](meta_type_system.md#syntax-parsing)) must be defined using [simple extensions](../extensions/index.md#simple-extensions). The key properties of these variations are: | Property | Description | | ----------------- | ------------------------------------------------------------ | diff --git a/site/docs/types/meta_type_system.md b/site/docs/types/meta_type_system.md new file mode 100644 index 000000000..830ca31e3 --- /dev/null +++ b/site/docs/types/meta_type_system.md @@ -0,0 +1,255 @@ +# Meta Type System + +In addition to the data type system, Substrait defines a second, much more restricted type system that is internally used for type parameters, constraint patterns for function argument types, return type derivations, and similar static constructs. In addition to basic scalar primitive types, data types (from the normal type system) are *themselves* values in this system, allowing them to be manipulated. + +The meta type system also includes a syntax definition. Combined with the pattern matching and evaluation semantics, it effectively forms a tiny domain-specific language. The syntax for data types especially is also used throughout the documentation. + +The grammar for the language can be found in ANTLR form [here](https://github.com/substrait-io/substrait/blob/main/text/SubstraitType.g4). Note that this grammar includes extensive comments about how to interpret the various syntax rules. These comments are leading in case they conflict with the definitions on this page. + +## Metatypes and Values + +Unlike the data type system, the meta type system is dynamically typed. This means that, ultimately, only one type needs to exist. Nevertheless, it makes sense to subdivide the values that this type can assume into groups, which we call metatypes. + +| Name | Description | +|------------|-------------| +| `metabool` | Contains the values `true` and `false`. | +| `metaint` | Contains all integer values from `-2^63` to `2^63-1`, i.e. the 64-bit two's-complement integers. | +| `metaenum` | Contains the set of all case-insensitive strings that form valid identifiers (matching `[a-zA-Z_$][a-zA-Z0-9_$]*`). These are normally constrained to a particular subset in order to behave like a proper enumeration. | +| `metastr` | Contains the set of all Unicode strings. | +| `typename` | Contains the set of all valid Substrait data type quadruplets (class, nullability, variation, and parameter pack). Parameters may include names in the binding for documentation purposes and/or to represent `NSTRUCT`. | + +Note that metatypes cannot be extended, only include scalar types, and don't include a `null` type or value. + +### Syntax Parsing + +While the syntax for representing concrete metavalues is fully covered by the pattern syntax defined below, let us first introduce it outside of pattern context to ease the learning curve. + +| Metatype | Syntax for values | +|--------------|-------------------| +| Booleans | `true` or `false`. Case insensitive, customarily written in lowercase. | +| Integers | Represented using the usual syntax for decimal integers. In regex form: `[+-](0|[1-9][0-9]*)`. Integers in other bases (hexadecimal, binary, or octal) are not supported. | +| Enumerations | Represented using the enumeration variant identifier on its own. Case insensitive, customarily written in UPPERCASE. | +| Strings | Represented using double quotes (`"`) as delimiters. Escape sequences are not currently supported, so `"` itself cannot be represented. Support for this may be added later by using two successive double quotes (`""`) as the escape sequence for a single `"`. | +| Data types | See below. | + +Data types are a bit more complicated. The basic structure is: + +``` +name?[variation] +``` + +The components of this expression are: + +| Component | Description | Required | +|-----------------------|-------------|----------| +| Name | Each type has a name, also referred to as the type class. The name is matched case-insensitively (e.g. `varchar` and `vArChAr` are equivalent), but is customarily written in lowercase for simple types and uppercase for compound types. | Yes | +| Nullability indicator | A type is either non-nullable or nullable. To express nullable types, a type name is appended with a question mark. The lack of a question mark suffix or (in rare occasions) an exclamation mark suffix expresses non-nullable. To explicitly refer to a type with either nullability, one may append two question marks. | Optional, defaults to non-nullable | +| Variation | When expressing a type, a user can define the type based on a type variation. Some systems use type variations to describe different underlying representations of the same data type. Such custom variations are usually specified by their name between square brackets. Within the context of a plan, they may also be referred to by their anchor index. To refer to the implicit system-preferred variation, use `[0]` or leave the variation unspecified. | Optional, defaults to [0] | +| Parameters | Compound types may have one or more configurable properties, specified via parameters. The expected number of parameters and their types is defined along with the type class. User-defined compound types may mark parameter slots as optional, in which case the parameter slot may be skipped using the `null` keyword. Non-null parameters may be named using `name: value` or `"name": value` syntax, where the latter must be used if `name` does not match `[a-zA-Z_$][a-zA-Z0-9_$]*`. | Required for (concrete) compound types, illegal for simple types | + +Parameter names are currently only used for the `NSTRUCT` pseudotype. Here, each parameter corresponds with a field data type, and the parameter name is used to represent the field name. Note however that in core Substrait algebra fields are unnamed and references are always based on zero-based ordinal positions; the named structs are only intended to be used to annotate types used at the inputs and outputs of a plan to ease understanding for humans, and to support consumers and producers that rely on field names at the peripheries of the plan. + +## Metapatterns + +All operations on metavalues are done using metapatterns. Most metapattern can be conceptualized as a procedurally-generated *set* of metavalues. We then define two operations on these patterns: + +| Name | Approximate prototype | Description | +|----------|------------------------------|-------------| +| Match | `match(pattern, value)` | Asserts that the given value is contained in the set represented by the given pattern. Throws an error if this is not the case. | +| Evaluate | `evaluate(pattern) -> value` | Asserts that the given pattern contains only one value, and then returns that value. Throws an error if this is not the case. | + +Note that for some pattern types the set analogy does not apply exactly or is heavily influenced by context. + +The following metapatterns are defined. + +| Syntax | Name | Metavalues contained in set | +|-------------------------------------|-----------------------|------------------------------------------------------------------------| +| `?` | Any | All metavalues. | +| `metabool` | Boolean any | `true` and `false`. | +| `true` | True | Only `true`. | +| `false` | False | Only `false`. | +| `metaint` | Integer any | All integer metavalues. | +| `..` | Integer at least | All integers greater than or equal to the given literal. | +| `..` | Integer at most | All integers less than or equal to the given literal. | +| `..` | Integer within | All integers between and including the given literals. | +| `` | Integer exactly | Only the given integer literal. | +| `metaenum` | Enum variant any | All enumeration variant metavalues. | +| `{, ...}` | Enum variant set | Exactly those enumeration variants specified between the curly braces. | +| ``* | Enum variant exactly | Only the given enumeration variant. | +| `metastr` | String any | All string metavalues. | +| `` | String exactly | Exactly the given string, delimited by double quotes. Escape sequences are currently not defined, so strings containing `"` literally cannot be parsed. | +| `typename` | Type with nullability | All data type metavalues matching the given nullability pattern. | +| ``* | Type with class | All data type metavalues belonging to the given type class that match the given nullability, variation, and parameter pack suffixes. | +| ``* | Consistent binding | Context-sensitive. See section on bindings. | +| `?`* | Inconsistent binding | Context-sensitive. See section on bindings. | +| `()` + expressions | Function call | Only the metavalue returned by the function. See section on functions. | + +Note some of these patterns are syntactically ambiguous. The parser must resolve the ambiguity based on what the identifier resolves to, being either a type class, an enum variant, or a binding. If a name does not resolve to anything, a binding is implicitly declared. + +### Nullability Suffixes + +Nullability suffixes are used to match or evaluate the nullability of a data type. The following nullability suffixes are defined. + +| Syntax | Name | Metavalues contained in set | +|---------------------|-------------------------|-----------------------------------------------------------------------------------------------------------------------| +| No suffix | Non-nullable | Non-nullable typenames and all non-typename metavalues. | +| `!` suffix | Explicitly non-nullable | Non-nullable typenames only. | +| `?` suffix | Nullable | Nullable typenames only. | +| `?` suffix | Pattern nullability | Only typenames, with nullability as specified in the pattern, using `true` for nullable and `false` for non-nullable. | + +For "type with class" patterns, not specifying a nullability suffix means the same thing as the `!` suffix. It is customarily omitted in this case. Because this is by far the most common pattern, you may not encounter it very often. For the other pattern types, however, not specifying a nullability suffix syntactically results in a different pattern type. + +Some additional syntactic sugar exists for function definitions, to make common nullability behavior a bit more readable. This is specified in the YAML files by means of the `nullability` field. The effect of this field on the patterns is as follows: + +| Syntax | Effect | +|--------------------|--------------------------------------------------------------------------------------------------------------------------------| +| `MIRROR` (default) | Replace all optional nullability patterns of toplevel argument, return type, and intermediate type patterns with `??nullable`. | +| `DECLARED_OUTPUT` | Replace all optional nullability patterns of toplevel arguments with `??nullable`. | +| `DISCRETE` | No effect. | + +In the above, "toplevel pattern" means that a pattern like `i8` on its own, but not like the `i8` in `STRUCT`. Combined with the semantics of inconsistent bindings, this replacement models the specified behavior of the `nullability` field exactly, so no further special cases are required to match functions against their prototypes. + +For some exotic functions, `MIRROR` and `DECLARED_OUTPUT` may not capture the intended behavior exactly. For example, a function may have one argument that doesn't participate in the `MIRROR` or `DECLARED_OUTPUT` behavior, or the nullability of nested types may need to participate. In these cases, `DISCRETE` nullability must be used, and the nullability patterns must be specified manually. + +### Type Variation Suffixes + +Type variation suffixes are used to match or evaluate the type variation of a data type. In the syntax column of the pattern table, they are represented using ``. The following type variation suffixes are defined. + +| Syntax | Name | Type variations contained in set | +|--------------------|----------------------------|---------------------------------------------------| +| No suffix | Compatible variations | When matching, the system-preferred variation and any user-defined variations with `INHERIT` function behavior defined for the related type class. When evaluating, always returns the system-preferred variation. | +| `[?]` suffix | Any variation | Any variation defined for the related type class. | +| `[0]` suffix | System-preferred variation | Only the system-preferred variation. | +| `[]` suffix | User-defined variation | Only the specified user-defined variation. | + +### Parameter Pack Suffixes + +Parameter pack suffixes are used to match or evaluate the parameter packs of compound data types. In the syntax column of the pattern table, they are represented using ``. The following parameter pack suffixes are defined. + +| Syntax | Name | Parameter packs contained in set | +|-------------------------|-------------------------|-----------------------------------------------------------------------------------------------------------------------------| +| No suffix | Any parameter pack | All parameter packs that the related type class supports. When evaluating, always returns an empty parameter pack. | +| `<>` suffix | Empty parameter pack | Only the empty parameter pack. | +| `<, ...>` suffix | Matching parameter pack | Only parameter packs where each parameter matches the given parameter pattern. The number of parameters must match exactly. | + +The parameter binding patterns themselves are defined as follows. + +| Syntax | Name | Parameter bindings contained in set | +|-------------|--------------------------|---------------------------------------------------------------------| +| `null` | Skipped parameter | Matches only explicitly-skipped optional parameters. | +| `?` | Skipped or any metavalue | Matches any parameter, including skipped parameters. | +| `` | Pattern parameter | Match the metavalue bound to the parameter using the given pattern. | + +Parameter binding patterns can also be given an optional prefix to associated a name with the parameter. Note that names are only used when evaluating the pattern, and that they are currently only applicable to the `NSTRUCT` pseudotype. + +| Prefix | Name | Parameter bindings contained in set | +|-------------|--------------------------|------------------------------------------------------------| +| No prefix | Unnamed parameter | The parameter binding is unnamed. | +| `:` | Named parameter | The parameter binding is named using the given identifier. | +| `:` | String-named parameter | The parameter binding is named using the given string. | + +### Bindings + +"Binding" is the name we use for identifiers that get bound to a value. For example, if we define a function with prototype `func(T) -> T`, the identifier `T` is a binding. + +The basic logic is quite simple: when a name is first used in a binding pattern, that name is bound to a value, and when the same name is used again later, the binding pattern will make use of the previously bound value. Another way to put it is that bindings behave more or less like single-assignment variables. The behavior is, however, complicated by nullability. In order to match the behavior of a pattern like `i32`, which only matches non-nullable `i32`s, `T` must also only match non-nullable patterns. Furthermore, if `T` were bound to a nullable type via `T?`, using `T` later would still refer to the non-nullable variant. This is captured by the following rules: + + - non-typename metavalues can only be bound and used by binding patterns with no nullability suffix; + - if a typename is bound to a name, the bound typename is always non-nullable; + - when a binding evaluates to a typename, the nullability suffix overrides the nullability. + +We further distinguish between two separate sets of semantics for binding patterns: consistent and inconsistent bindings. Consistent bindings behave the most like single-assignment variables and are what you would normally use, whereas inconsistent bindings have special-cased semantics that are useful for representing `MIRROR` nullability and inconsistently-typed variadic argument/parameter slots. Note that inconsistent bindings only very rarely need to be specified manually. Refer to the subsections below for more information. + +The exact semantics of the various binding patterns are as follows. + +| Binding pattern type | Match, not yet bound | Match, previously bound | Evaluate, not yet bound | Evaluate, previously bound | +|---------------------------------------|----------------------|-------------------------|-------------------------|----------------------------| +| Consistent binding w/o nullability | Match any metavalue, except nullable typenames. Bind name to matched value. | Match only the previously bound metavalue. | Always fails. | Evaluate to the previously bound metavalue. | +| Consistent binding with nullability | Match any typename metavalue for which the nullability suffix matches. Bind name to matched value, but with nullability overridden to non-nullable. | Match only typename metavalues, and fail if the previously bound metavalue is not a typename. Match the class, variation, and parameter pack against the previously bound value. Match the nullability against the nullability suffix. Bound value is not modified. | Always fails. | Fail if the previously bound value is not a typename. Otherwise, evaluate to the previously bound metavalue, with nullability overridden by the evaluation result of the nullability pattern. | +| Inconsistent binding w/o nullability | Match any metavalue, except nullable typenames. Bind name to matched value. | Match any metavalue, except nullable typenames. If the matched value is `true` and the previously bound value is `false`, rebind the name to `true`; otherwise, bound value is not modified. | Returns `false`. | Evaluate to the previously bound metavalue. | +| Inconsistent binding with nullability | Match any typename metavalue for which the nullability suffix matches. Bind name to matched value, but with nullability overridden to non-nullable. | Match only typename metavalues, and fail if the previously bound metavalue is not a typename. Only match the nullability against the nullability suffix. Bound value is not modified. | Always fails. | Fail if the previously bound value is not a typename. Otherwise, evaluate to the previously bound metavalue, with nullability overridden by the evaluation result of the nullability pattern. | + +#### Inconsistent Variadic Argument/Parameter Slots + +Variadic functions with `INCONSISTENT` argument behavior can be represented by using inconsistent bindings instead of consistent bindings. For example, a function defined like `func(?T...) -> T` with will return the type of the first argument passed to it but will allow any type to be specified for any additional arguments passed to it, whereas if it were defined like `func(T...) -> T`, all arguments must be of type `T`. As a more complex example, `func(STRUCT...) -> STRUCT` will take two-tuples where the first field is inconsistent and the second is consistent. + +The exact behavior of specifying that a function has `INCONSISTENT` argument behavior is for all bindings in the last argument slot to be implicitly turned inconsistent. `CONSISTENT` argument behavior has no effect. Therefore, specifications of exotic behavior like the `STRUCT` example above must use `CONSISTENT` argument behavior. + +#### Mirror and Declared-Output Nullability + +When acting on the nullability booleans (as used in the `??nullable` nullability suffixes for `MIRROR` and `DECLARED_OUTPUT`), inconsistent bindings will bind and yield true to `nullable` if and only if any nullable arguments were matched. The special case of returning `false` when the name is not yet bound handles niladic functions, or variadic functions with no bound arguments. + +The binding pattern types with nullability override are automatically used in place of the regular versions when `MIRROR` or `DECLARED_OUTPUT` is specified. + +### Functions + +Function patterns allow a metavalue to be derived based on a number of other metavalues. All functions can be written using the usual `()` syntax (for example `add(1, 3)`), but many functions can also be specified implicitly with infix operators (for example `1 + 3`). + +Many of the functions have metatype requirements on their inputs. If a value is passed that does not match this metatype, evaluation fails. + +The following functions are defined. Note that this set of functions cannot be user-extended. + +| Prototype | Description | +|-----------------------------------------------|------------------------------------------------------------------------------------------------------| +| `not(metabool) -> metabool` | Boolean NOT. | +| `and(metabool*) -> metabool` | Boolean AND. Evaluated lazily from left to right. | +| `or(metabool*) -> metabool` | boolean OR. Evaluated lazily from left to right. | +| `negate(metaint) -> metaint` | Integer negation. 64-bit two's complement overflow must be detected and cause evaluation to fail. | +| `add(metaint*) -> metaint` | Integer sum. 64-bit two's complement overflow must be detected and cause evaluation to fail. | +| `subtract(metaint, metaint) -> metaint` | Integer subtraction. 64-bit two's complement overflow must be detected and cause evaluation to fail. | +| `multiply(metaint*) -> metaint` | Integer product. 64-bit two's complement overflow must be detected and cause evaluation to fail. | +| `divide(metaint, metaint) -> metaint` | Integer division. Divisions by zero and 64-bit two's complement overflow (-2^63 / -1) must be detected and cause evaluation to fail. Divisions round toward zero. | +| `min(metaint+) -> metaint` | Returns the minimum integer value. | +| `max(metaint+) -> metaint` | Returns the maximum integer value. | +| `equal(T, T) -> metabool` | Returns whether the two metavalues are equal. Data type parameter names should be ignored. | +| `not_equal(T, T) -> metabool` | Returns whether the two metavalues are not equal. Data type parameter names should be ignored. | +| `greater_than(metaint, metaint) -> metabool` | Returns whether the left integer is greater than the right. | +| `less_than(metaint, metaint) -> metabool` | Returns whether the left integer is less than the right. | +| `greater_equal(metaint, metaint) -> metabool` | Returns whether the left integer is greater than or equal to the right. | +| `less_equal(metaint, metaint) -> metabool` | Returns whether the left integer is less than or equal to the right. | +| `covers(value, pattern) -> metabool` | Returns whether the left value matches the right pattern. Side effects of the match operation (i.e. changes to bound values) should only be committed when the complete pattern matches. For example, `covers(struct, struct)` yields false, even though `T` matched `i8` before the match failed. Therefore, `T` should *not* be bound to `i8`. | +| `if_then_else(metabool, T, T) -> T` | If-then-else expression. Evaluated lazily. That is, the second argument is only evaluated if the first evaluated to true, and the third is only evaluated if the first evaluated to false. | + +In addition, the following infix expressions are defined. Parentheses can be used to override precedence order. + +| Syntax | Function | Precedence | Associativity | +|----------------------|-----------------|------------|---------------| +| `!A` | `not` | 1 | Right to left | +| `-A` | `negate` | 1 | Right to left | +| `A * B` | `multiply` | 2 | Left to right | +| `A / B` | `divide` | 2 | Left to right | +| `A + B` | `add` | 3 | Left to right | +| `A - B` | `subtract` | 3 | Left to right | +| `A < B` | `less_than` | 4 | Left to right | +| `A <= B` | `less_equal` | 4 | Left to right | +| `A > B` | `greater_than` | 4 | Left to right | +| `A >= B` | `greater_equal` | 4 | Left to right | +| `A == B` | `equal` | 5 | Left to right | +| `A != B` | `not_equal` | 5 | Left to right | +| `A && B` | `and` | 6 | Left to right | +| `A || B` | `or` | 7 | Left to right | +| `if A then B else C` | `if_then_else` | N/A | N/A | + +Note that the C-style `A ? B : C` ternary operator syntax is not supported, as the question mark makes it highly ambiguous with nullability patterns for parsers that do not support arbitrary recursive descent. + +#### Matching Behavior + +Functions are typically used only in evaluation context, but also work in match context; a function pattern will simply match the incoming value against its evaluation result. + +!!! warning + + Constructs like `A + B = C` do *not* work as you may expect them to. Here, `A + B` is used in match context, but as part of the match operation, `A` and `B` will end up being evaluated. The net result is that all three bindings need to have been previously defined for this operation to work, in which case it will fail if the equality is not true. This may still be useful for checking constraints, but rewriting the expression to `A = C - B`, `B = C - A`, or `C = A + B` makes it more powerful, in that now the name on the left-hand side need not be bound yet. + +## Metastatements and Derivation Programs + +Metastatements allow for the specification of more complex constraints and expressions than can be described using just patterns. + +| Syntax | Name | Description | +|----------------------|-----------------|--------------------------------------------------------------------------------------------------------------------| +| `A = B` | Assignment | Evaluate `B`, then match the result against `A`. If either the evaluation or the match fails, the statement fails. | +| `assert A` | Assertion | Evaluate `A`. If this fails or returns anything other than `true`, the statement fails. | +| `assert A matches B` | Match assertion | Evaluate `A`, then match the result against `B`. If either the evaluation or the match fails, the statement fails. | + +Note that, ultimately, these are all variations of the same thing; assertions can be regarded as syntactic sugar for assignments. The assertion variants exist because `assert A && B` and `assert T matches struct` communicate intent better than `true = A && B` and `struct = T`, but are otherwise exactly the same thing. + +Synactically, statements usually appear in derivation programs, used to derive the return type or intermediate type of a function, or the structure type of a user-defined type class. Such derivation programs consist of zero or more newline-separated statements followed by a final pattern. The statements are executed in-order before the final pattern is evaluated to compute the desired type. If the final pattern does not yield a typename in these contexts, derivation fails. diff --git a/site/docs/types/type_parsing.md b/site/docs/types/type_parsing.md deleted file mode 100644 index 396215e60..000000000 --- a/site/docs/types/type_parsing.md +++ /dev/null @@ -1,36 +0,0 @@ -# Type Syntax Parsing - -In many places, it is useful to have a human-readable string representation of data types. Substrait has a custom syntax for type declaration. The basic structure of a type declaration is: - -``` -name?[variation] -``` - -The components of this expression are: - -| Component | Description | Required | -| ---------------------- | ------------------------------------------------------------ | ------------------------------------- | -| Name | Each type has a name. A type is expressed by providing a name. This name can be expressed in arbitrary case (e.g. `varchar` and `vArChAr` are equivalent). | | -| Nullability indicator | A type is either non-nullable or nullable. To express nullability, a type name is appended with a question mark. | Optional, defaults to non-nullable | -| Variation | When expressing a type, a user can define the type based on a type variation. Some systems use type variations to describe different underlying representations of the same data type. This is expressed as a bracketed integer such as [2]. | Optional, defaults to [0] | -| Parameters | Compound types may have one or more configurable properties. The two main types of properties are integer and type properties. The parameters for each type correspond to a list of known properties associated with a type as declared in the order defined in the type specification. For compound types (types that contain types), the data type syntax will include nested type declarations. The one exception is structs, which are further outlined below. | Required where parameters are defined | - -### Grammars - -It is relatively easy in most languages to produce simple parser & emitters for the type syntax. To make that easier, Substrait also includes an ANTLR [impl pending] grammar to ease consumption and production of types. - -### Structs & Named Structs - -Structs are unique from other types because they have an arbitrary number of parameters. The parameters can also include one or two subproperties. Struct parsing is thus declared in the following two ways: - -``` -# Struct -struct?[variation] - -# Named Struct -nstruct?[variation] -``` - -In the normal (non-named) form, struct declares a set of types that are fields within that struct. In the named struct form, the parameters are formed by tuples of names + types, delineated by a colon. Names that are composed only of numbers and letters can be left unquoted. For other characters, names should be quoted with double quotes and use backslash for double-quote escaping. - -Note, in core Substrait algebra, fields are unnamed and references are always based on zero-index ordinal positions. However, data inputs must declare name-to-ordinal mappings and outputs must declare ordinal-to-name mappings. As such, Substrait also provides a named struct which is a pseudo-type that is useful for human consumption. Outside these places, most structs in a Substrait plan are structs, not named-structs. The two cannot be used interchangeably. diff --git a/site/docs/types/type_system.md b/site/docs/types/type_system.md deleted file mode 100644 index 56362d127..000000000 --- a/site/docs/types/type_system.md +++ /dev/null @@ -1,16 +0,0 @@ -# Type System - -Substrait tries to cover the most common types used in data manipulation. Types beyond this common core may be represented using [simple extensions](../extensions/index.md#simple-extensions). - -Substrait types fundamentally consist of four components: - -| Component | Condition | Examples | Description -| ------------------------------- | ------------------- | ----------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -| [Class](type_classes.md) | Always | `i8`, `string`, `STRUCT`, extensions | Together with the parameter pack, describes the set of non-null values supported by the type. Subdivided into simple and compound type classes. -| Nullability | Always | Either `NULLABLE` (`?` suffix) or `REQUIRED` (no suffix) | Describes whether values of this type can be null. Note that null is considered to be a special value of a nullable type, rather than the only value of a special null type. -| [Variation](type_variations.md) | Always | No suffix or explicitly `[0]` (system-preferred), or an extension | Allows different variations of the same type class to exist in a system at a time, usually distinguished by in-memory format. -| Parameters | Compound types only | `<10, 2>` (for `DECIMAL`), `` (for `STRUCT`) | Some combination of zero or more data types or integers. The expected set of parameters and the significance of each parameter depends on the type class. - -Refer to [Type Parsing](type_parsing.md) for a description of the syntax used to describe types. - -Note that Substrait employs a strict type system without any coercion rules. All changes in types must be made explicit via [cast expressions](../expressions/specialized_record_expressions.md). diff --git a/site/mkdocs.yml b/site/mkdocs.yml index 311088b6f..bc23cb588 100644 --- a/site/mkdocs.yml +++ b/site/mkdocs.yml @@ -47,9 +47,18 @@ plugins: indent_depth: 4 # required to make superfences happy - redirects: redirect_maps: - 'types/simple_logical_types.md': 'types/type_classes.md' - 'types/compound_logical_types.md': 'types/type_classes.md' - 'types/user_defined_types.md': 'types/type_classes.md' + 'types/simple_logical_types.md': 'types/data_type_classes.md' + 'types/compound_logical_types.md': 'types/data_type_classes.md' + 'types/user_defined_types.md': 'types/data_type_classes.md' + 'types/type_system.md': 'types/data_type_system.md' + 'types/type_classes.md': 'types/data_type_classes.md' + 'types/type_variations.md': 'types/data_type_variations.md' + 'types/type_parsing.md': 'types/meta_type_system.md' + 'expressions/scalar_functions.md': 'expressions/Extension_Functions/scalar_functions.md' + 'expressions/aggregate_functions.md': 'expressions/Extension_Functions/aggregate_functions.md' + 'expressions/window_functions.md': 'expressions/Extension_Functions/window_functions.md' + 'expressions/table_functions.md': 'expressions/Extension_Functions/table_functions.md' + 'expressions/user_defined_functions.md': 'expressions/Extension_Functions/index.md' - gen-files: scripts: - docs/extensions/generate_function_docs.py diff --git a/text/SubstraitType.g4 b/text/SubstraitType.g4 new file mode 100644 index 000000000..546e233c3 --- /dev/null +++ b/text/SubstraitType.g4 @@ -0,0 +1,596 @@ +grammar SubstraitType; + +// Note: this grammar is intentionally written to avoid ANTLR-specific features +// that someone who hasn't used ANTLR before might not know about, including +// explicitly avoiding left recursion, such that it can easily be ported to +// other parser generators if necessary. In this way, it hopefully doubles as a +// human-readable specification for this DSL. +// +// This comes at the cost of not generating very nice parse trees. You can use +// this grammar with ANTLR directly if you want, but you might want to rewrite +// it if you intend to use the listener or generated AST directly. +// +// Some things that you will need to know if you've never seen ANTLR before: +// - ANTLR distinguishes between tokenizer rules and parser rules by +// capitalization of the rule name: if the first letter is uppercase, the +// rule is a token rule; if it is lowercase, it is a parser rule. Yuck. +// - When multiple token rules match: +// - choose the token that matches the most text; +// - if same length, use the one defined earlier. +// (ANTLR supports implicit tokens as well, but we don't use them) +// - Parse conflicts are solved using PEG rules. That is, for alternations, +// the first alternative that matches the input is used. For ?, *, and +, +// matching is greedy. +// - The ~ symbol is used to negate character sets, as opposed to the [^...] +// syntax from regular expressions. + + +//============================================================================= +// Whitespace and comment tokens +//============================================================================= + +// Whitespace and comment handling. You can use C-style line and block +// comments. +LineComment : '//' ~[\r\n]* -> channel(HIDDEN) ; +BlockComment : ( '/*' ( ~'*' | '*'+ ~[*/] ) '*'* '*/' ) -> channel(HIDDEN) ; +Whitespace : [ \t]+ -> channel(HIDDEN) ; + +// Type derivations are newline-sensitive, so they're not ignored. +Newline : [\r\n]+ ; + +// Newlines can be embedded by escaping the newline character itself with a +// backslash. +EscNewline : '\\' [\r\n]+ -> channel(HIDDEN) ; + + +//============================================================================= +// Keyword tokens +//============================================================================= + +// Substrait is case-insensitive, ANTLR is not. So, in order to define our +// keywords in a somewhat readable way, we have to define these shortcuts. +// If you've never seen ANTLR before, fragment rules are pretty much just +// glorified preprocessor/search-and-replace macros. +fragment A : [aA]; fragment B : [bB]; fragment C : [cC]; fragment D : [dD]; +fragment E : [eE]; fragment F : [fF]; fragment G : [gG]; fragment H : [hH]; +fragment I : [iI]; fragment J : [jJ]; fragment K : [kK]; fragment L : [lL]; +fragment M : [mM]; fragment N : [nN]; fragment O : [oO]; fragment P : [pP]; +fragment Q : [qQ]; fragment R : [rR]; fragment S : [sS]; fragment T : [tT]; +fragment U : [uU]; fragment V : [vV]; fragment W : [wW]; fragment X : [xX]; +fragment Y : [yY]; fragment Z : [zZ]; + +// Syntactic keywords. +Assert : A S S E R T ; +Matches : M A T C H E S ; +If : I F ; +Then : T H E N ; +Else : E L S E ; + +// Named literal values. +Null : N U L L ; +True : T R U E ; +False : F A L S E ; + +// Metatype identification keywords. +Metabool : M E T A B O O L ; +Metaint : M E T A I N T ; +Metaenum : M E T A E N U M ; +Metastr : M E T A S T R ; +Typename : T Y P E N A M E ; + +// Note that data type classes are not keywords. We support user-defined type +// classes anyway, so name resolution has to be done after parsing anyway. + + +//============================================================================= +// Symbol tokens +//============================================================================= + +// Symbols used. +Period : '.' ; // identifier paths +Comma : ',' ; // separator for pattern lists +Colon : ':' ; // separator for named parameters +Semicolon : ';' ; // separator for statements +Question : '?' ; // any, inconsistent bindings & nullable type suffix +Bang : '!' ; // boolean NOT & explicitly non-nullable type suffix +OpenParen : '(' ; // precedence override & function call args (open) +CloseParen : ')' ; // precedence override & function call args (close) +OpenCurly : '{' ; // enum set patterns (open) +CloseCurly : '}' ; // enum set patterns (close) +OpenSquare : '[' ; // data type variation suffix (open) +CloseSquare : ']' ; // data type variation suffix (close) +Assign : '=' ; // assignment statements +BooleanOr : '||' ; // boolean OR expression +BooleanAnd : '&&' ; // boolean AND expression +Equal : '==' ; // equality expression +NotEqual : '!=' ; // not-equals expression +LessThan : '<' ; // less-than expression & data type parameter pack +LessEqual : '<=' ; // less-equal expression +GreaterThan : '>' ; // greater-than expression & data type parameter pack +GreaterEqual : '>=' ; // greater-equal expression +Plus : '+' ; // additions and integer literal sign +Minus : '-' ; // subtractions, negation, and integer literal sign +Multiply : '*' ; // multiplication expression +Divide : '/' ; // division expression +Range : '..' ; // integer set patterns + + +//============================================================================= +// Procedurally-matched tokens +//============================================================================= + +// Tokens for integer literals. +Nonzero : [1-9] [0-9]* ; +Zero : '0' ; + +// String literal token. +String : '"' ~["] '"' ; + +// Identifier token. Note that $ signs are legal in identifiers, and note that +// all identifier matching is case-insensitive. Note also that keywords take +// precedence. +Identifier : [a-zA-Z_$] [a-zA-Z0-9_$]* ; + + +//============================================================================= +// Grammar rules +//============================================================================= + +// Most things in the simple extension YAMLs that refer to a type are parsed +// using patterns; patterns can both matched and evaluated (not ALL patterns +// can do both, but there is considerable overlap between the two classes, +// so they were conceptually merged). When a type needs to be derived based on +// a number of given metavalues, such as the data types of arguments passed to +// a function, a derivation program is used. Syntactically, the only difference +// is that programs can include a set of statements before the final pattern. +// Newlines can optionally go before or after a type derivation pattern or +// program without affecting syntax. +startPattern : Whitespace* Newline* pattern Newline* EOF ; +startProgram : Whitespace* Newline* program Newline* EOF ; + +// A type derivation program consists of zero or more statements followed by +// the final pattern that should evaluate to the derived data type. +program : ( statement statementSeparator )* pattern ; + +// Statements are separated from each other and from the final derivation +// expression using newlines or a semicolon. +statementSeparator : Newline* ( Newline | Semicolon Newline* ) ; + +// Statements manipulate the state of the type derivation interpreter before +// the final derivation expression is evaluated. They look like assignment +// statements at first glance, but act more like equality or set containment +// assertions: the right-hand side is evaluated like an expression as you +// might expect, but the left-hand side acts just like the patterns that are +// used to match function argument types. While this is perhaps not the most +// intuitive ruleset, it is extremely easy to implement (it only reuses +// features we already needed anyway), while also being a much more powerful +// primitive than a simple assignment statement, because it can also be used +// for bounds checking and other assertions. For example, if we have a +// function like `fn(VARCHAR(a), VARCHAR(b))` and the implementation of the +// function requires that a + b equals 10, we can simply write "10 = a + b". +// This works, because the pattern "10" will only match the value 10, and +// a pattern mismatch at any point during the matching and evaluation process +// indicates that the implementation is incompatible with the given argument +// types. If you find this syntax confusing, you may also write +// "assert a + b matches 10" or "assert a + b == 10"; the former does the +// exact same thing, while the latter reduces to "true = a + b == 10", which is +// functionally the same thing. +// +// Note that when you use these statements like assignment statements, you can +// only ever reassign a binding to the same value. For example, "a = 10; a = 20" +// will always fail, because a cannot both be 10 and 20 at the same time (more +// accurately, a is bound to 10, so the second statement behaves like +// "10 = 20", and 20 does not match 10). +statement + : pattern Assign pattern #Normal + | Assert pattern Matches pattern #Match + | Assert pattern #Assert + ; + +// Patterns are at the core of the type derivation interpreter; they are used +// both for matching and as expressions. However, note that not all types of +// patterns work in both contexts. +pattern : patternOr ; + +// Lazily-evaluated boolean OR expression. Maps to builtin or() function if +// more than one pattern is parsed. +patternOr : patternAnd ( operatorOr patternAnd )* ; +operatorOr : BooleanOr #Or ; + +// Lazily-evaluated boolean AND expression. Maps to builtin and() function if +// more than one pattern is parsed. +patternAnd : patternEqNeq ( operatorAnd patternEqNeq )* ; +operatorAnd : BooleanAnd #And ; + +// Equality and not-equality expressions. These map to the builtin equal() +// and not_equal() functions in left-to-right order. +patternEqNeq : patternIneq ( operatorEqNeq patternIneq )* ; +operatorEqNeq : Equal #Eq | NotEqual #Neq ; + +// Integer inequality expressions. These map to the builtin greater_than(), +// less_than(), greater_equal(), and less_equal() functions in left-to-right +// order. +patternIneq : patternAddSub ( operatorIneq patternAddSub )* ; +operatorIneq : LessThan #Lt | LessEqual #Le | GreaterThan #Gt | GreaterEqual #Ge ; + +// Integer addition and subtraction. These map to the builtin add() and +// subtract() functions in left-to-right order. +patternAddSub : patternMulDiv ( operatorAddSub patternMulDiv )* ; +operatorAddSub : Plus #Add | Minus #Sub ; + +// Integer multiplication and division. These map to the builtin multiply() and +// divide() functions in left-to-right order. +patternMulDiv : patternMisc ( operatorMulDiv patternMisc )* ; +operatorMulDiv : Multiply #Mul | Divide #Div ; + +// Miscellaneous patterns that don't need special rules for precedence or +// avoiding left-recursion. +patternMisc + + // Parentheses for overriding operator precedence. + : OpenParen pattern CloseParen #parentheses + + // If-then-else pattern. Can only be evaluated. The first pattern must + // evaluate to a boolean. The second or third pattern is then evaluated + // based on that boolean and returned. The branch that is not selected is + // also not evaluated (i.e. evaluation is lazy). + | If pattern Then pattern Else pattern #ifThenElse + + // Unary not function. Can only be evaluated and can only be applied to + // booleans. + | Bang pattern #unaryNot + + // The "anything" pattern. This matches everything, and cannot be evaluated. + // It's primarily intended for matching (parts of) argument types, when you + // don't need or want a binding. For example, `equals(?, ?) -> boolean` would + // allow for any combination of argument types. This distinguishes it from + // `equals(any1, any1) -> boolean`, which only accepts equal types; instead + // it behaves like `equals(any1, any2) -> boolean`. `?` is especially useful + // when you want this type of behavior for a variadic function; for example, + // `serialize(?...) -> binary` will match any number and combination of + // argument types, while `serialize(any1...) -> binary` would only accept any + // number of any *one* data type. + | Question #any + + // Matches any boolean value. Cannot be evaluated. + | Metabool #boolAny + + // Matches and evaluates to the boolean value "true". + | True #boolTrue + + // Matches and evaluates to the boolean value "false". + | False #boolFalse + + // Matches any integer value. Cannot be evaluated. + | Metaint #intAny + + // Matches any integer value within the specified inclusive range. Can only + // be evaluated if the two bounds are equal, in which case it reduces to just + // a single integer. + | integer Range integer #intRange + + // Matches any integer value that equals at least the given number. Cannot be + // evaluated. + | integer Range #intAtLeast + + // Matches any integer value that equals at most the given number. Cannot be + // evaluated. + | Range integer #intAtMost + + // Matches and evaluates to exactly the given integer. + | integer #intExactly + + // Matches any enumeration constant. + | Metaenum #enumAny + + // Matches an enumeration constant in the given set. If only a single + // constant is specified, the pattern evaluates to that constant, otherwise + // it cannot be evaluated. + | OpenCurly Identifier (Comma Identifier)* CloseCurly #enumSet + + // Matches any string. + | Metastr #strAny + + // Matches and evaluates to exactly the given string. + | String #strExactly + + // Matches any typename for which the nullability matches the nullability + // suffix. Use `typename??` for either nullability. + | Typename nullability? #dtAny + + // Evaluates a function. When a function is used in match context, the + // function (and its arguments) will be *evaluated* instead, and the incoming + // value is matched against the result. This means that it is legal to define + // a function like f(VARCHAR(x), VARCHAR(y), VARCHAR(x + y)) because the x + // and y bindings are captured before x + y is evaluated, but it is NOT legal + // to define it like f(VARCHAR(x + y), VARCHAR(x), VARCHAR(y)) because x and + // y are not yet bound when x + y is evaluated. + // f(VARCHAR(x), VARCHAR(x + y), VARCHAR(y)) is also NOT legal, again because + // some of the function bindings have not yet been captured, even though + // mathematically this could be rewritten from x + y <- input to + // y <= input - x (the evaluator is not smart enough for this, and this + // rewriting cannot be generalized over all functions). + // + // The following functions are currently available: + // + // - "not(metabool) -> metabool": boolean NOT. + // - "and(metabool*) -> metabool": boolean AND. Evaluated lazily from left + // to right. + // - "or(metabool*) -> metabool": boolean OR. Evaluated lazily from left to + // right. + // - "negate(metaint) -> metaint": integer negation. 64-bit two's complement + // overflow must be detected, and implies that the function implementation + // that the program belongs to does not match the given argument types. + // - "add(metaint*) -> metaint": integer sum. Overflow handled as above. + // - "subtract(metaint, metaint) -> metaint": subtracts an integer from + // another. Overflow handled as above. + // - "multiply(metaint*) -> metaint": integer product. Overflow handled as + // above. + // - "divide(metaint, metaint) -> metaint": divides an integer over + // another. Overflow and division by zero handled as above. + // - "min(metaint+) -> metaint": return the minimum integer value. + // - "max(metaint+) -> metaint": return the maximum integer value. + // - "equal(T, T) -> metabool": return whether the two values are equal. + // - "not_equal(T, T) -> metabool": return whether the two values are not + // equal. + // - "greater_than(metaint, metaint) -> metabool": return whether the left + // integer is greater than the right. + // - "less_than(metaint, metaint) -> metabool": return whether the left + // integer is less than the right. + // - "greater_equal(metaint, metaint) -> metabool": return whether the left + // integer is greater than or equal to the right. + // - "less_equal(metaint, metaint) -> metabool": return whether the left + // integer is less than or equal to the right. + // - "covers(value, pattern) -> metabool": return whether the left value + // matches the pattern. The pattern may make use of bindings that were + // previously defined. New bindings are captured if and only if covers + // returns true. This allows for patterns like + // assert if covers(x, struct) then a < 10 \ + // else if covers(x, struct) then a + b < 10 \ + // else false; + // to be written and work as expected. + // - "if_then_else(metabool, T, T) -> T": if-then-else expression. Evaluated + // lazily. + // + // Note that many of the functions also have corresponding expressions. These + // expressions are simply syntactic sugar for calling the functions directly. + | Identifier OpenParen ( pattern (Comma pattern)* )? CloseParen #function + + // This pattern matches one of three things, which are too context-sensitive + // to distinguish at this time: + // + // - a data type pattern; + // - an enum constant; + // - a normal binding; or + // - a binding with nullability override. + // + // The type depends on the identifier path, and must be disambiguated as + // follows during name resolution: + // + // - Keep track of a case-insensitive mapping from name to binding, enum + // constant, or type class while analyzing the parse tree. It will be + // empty initially. + // - Whenever this pattern appears, resolve the name using this mapping: + // - If resolution fails, resolve the name as a type class instead (it + // could be the name of a builtin type class, a type class defined + // in the current extension, or a type class defined in a dependency + // if appropriately prefixed with the dependency namespace): + // - If this succeeds, add an entry to the name mapping, mapping the + // incoming identifier path to the type class. If the type class is + // user-defined, and the type class has enum parameter slots, also + // add entries to the name mapping for all the enum variants; if a + // name was already defined, do NOT update the mapping. Finally, + // disambiguate the pattern as a data type pattern. + // - If this fails and the identifier path consists of only a single + // element, map the incoming identifier path to a binding, and + // disambiguate the pattern as a normal binding or a binding + // with nullability override, depending on the presence of the + // nullability field. + // - If the above fails and the identifier path consists of multiple + // elements, analysis should fail. + // - If resolution yields a binding, disambiguate the pattern as a + // normal binding or a binding with nullability override, depending on + // the presence of the nullability field. + // - If resolution yields an enum constant, disambiguate the pattern as + // an enum constant. + // - If resolution yields a type class, disambiguate the pattern as a + // data type pattern. + // + // If the optional nullability, variation, or parameters fields are non-empty + // when they can't be according to the rules of the disambiguated pattern + // type, analysis should fail. + // + // Note that the `!` suffix disambiguates between a normal binding and a + // binding with a non-nullable nullability override. For a data type pattern, + // non-nullable is the default, so something like `i32` is exactly the same + // as `i32!`. + // + // The behavior for the resolved pattern types is: + // - Data type pattern: + // - Matches a metavalue if and only if: + // - the metavalue is a typename; + // - the type class matches the identified class; + // - the nullability of the type matches the rules detailed in the + // comments of the nullability rule; + // - the variation of the type matches the rules detailed in the + // comments of the variation rule; and + // - the parameter pack matches the rules detailed in the comments + // of the parameters rule. + // - Evaluates to a data type with the specified type class and the + // evaluation result of the nullability, variation, and parameters + // fields. If any of those things cannot be evaluated, the data type + // pattern cannot be evaluated. If any parameter pack constraint + // violations result from this, they are treated as pattern match + // failures (i.e., if this happens in a return type derivation of + // a function, the function is said to not match the given arguments). + // + // - Enum constant: + // - Matches a metavalue if and only if it is exactly the specified enum + // variant. + // - Evaluates to the specified enum variant. + // - The nullability, variation, and parameters fields are illegal and + // must be blank. + // + // - Consistent binding without nullability suffix: + // - If this is the first use of the name, matches non-typename + // metavalues and non-nullable typenames. The incoming metavalue is + // bound to the name as a side effect. + // - If the name was previously bound, matches only if the incoming + // metavalue is exactly equal to the previous binding. + // - Can only be evaluated if the name was previously bound, in which + // case it yields the bound value exactly. + // - The variation and parameter pack fields are illegal and must be + // blank. + // + // - Consistent binding with nullability suffix: + // - If this is the first use of the name, matches if and only if: + // - the incoming metavalue is a typename; and + // - the nullability of the incoming type matches the nullability + // suffix. + // If the above rules match, the incoming typename, with its + // nullability overridden to non-nullable, is bound to the name as a + // side effect. + // - If the name was previously bound, matches if and only if: + // - the incoming metavalue is a typename; + // - the nullability of the incoming type matches the nullability + // suffix; + // - the previously bound metavalue is a typename; and + // - the incoming type matches the previously bound type, ignoring + // nullability and parameter names. + // - Can only be evaluated if the name was previously bound. If the + // previously bound metavalue is not a typename, evaluation fails. The + // returned type is the previously bound type, with its nullability + // adjusted according to the nullability suffix evaluation rules. + // - The variation and parameters fields are illegal and must be blank. + | identifierPath nullability? variation? parameters? #datatypeBindingOrConstant + + // Pattern for inconsistent bindings. Inconsistent bindings are variations + // of normal bindings and bindings with nullability override with looser + // matching and extended evaluation rules. These rules are designed + // specifically for matching inconsistent variadic arguments and for + // modelling MIRROR nullability behavior. Specifically: + // + // - Use `?T` instead of `T` for a variadic argument slot to capture the + // value of the first argument and ignore the rest, thus rendering it + // inconsistent. + // - Use `type??nullable` instead of `type` for argument slots and the + // return type to match both nullable and non-nullable data types for + // the argument, and yield a nullable return type only if any of the + // bound arguments are nullable. + // + // The exacty behavior for the pattern types is as follows. Rules that differ + // from the consistent binding rules are highlighted with (!). + // + // - Inconsistent binding without nullability suffix: + // - If this is the first use of the name, matches non-typename + // metavalues and non-nullable typenames. The incoming metavalue is + // bound to the name as a side effect. + // - (!) If the name was previously bound, still matches all + // non-typename metavalues and non-nullable typenames. If the + // incoming metavalue is boolean `true`, and the currently bound + // metavalue is boolean `false`, rebind the name to `true` as a side + // effect. Otherwise, leave it unchanged. + // - (!) If this is the first use of the name, evaluation yields + // the metabool `false` (for the nullability of the return type in + // a MIRROR function). + // - If the name was previously bound, evaluation yields the bound + // value exactly. + // + // - Inconsistent binding with nullability override: + // - If this is the first use of the name, matches if and only if: + // - the incoming metavalue is a typename; and + // - the nullability of the incoming type matches the nullability + // field. + // If the above rules match, the incoming typename, with its + // nullability overridden to non-nullable, is bound to the name as a + // side effect. + // - (!) If the binding was previously bound, matches if and only if: + // - the incoming metavalue is a typename; and + // - the nullability of the incoming type matches the nullability + // field. + // There are no side effects in this case. + // - Can only be evaluated if the name was previously bound. If the + // previously bound metavalue is not a typename, evaluation fails. The + // returned type is the previously bound type, with its nullability + // adjusted according to the nullability field evaluation rules. + | Question Identifier nullability? #inconsistent + + // Unary negation function. Can only be evaluated and can only be applied to + // integers. Note that this is all the way at the back because signed integer + // literals should be preferred, since those can also be matched, and can + // deal with -2^63 without overflow. + | Minus pattern #unaryNegate + ; + +// Nullability suffix. +// +// - If there is no such suffix, or the suffix is "!", the pattern matches +// only non-nullable types, and also evaluates to a non-nullable type if +// applicable. The "!" suffix changes the semantics of bindings slightly +// compared to no suffix (specifically, `x!` only matches non-nullable +// typenames, but `x` also matches non-typename metavalues), but is +// otherwise optional and customarily not written. +// - If this suffix is just "?", the pattern matches only nullable types, +// and also evaluates to a nullable type if applicable. +// - If this suffix is a "?" followed by a pattern, the pattern is matched +// against false for non-nullable and true for nullable types. Likewise for +// evaluation; if the pattern evaluates to false the type will be +// non-nullable, if it evaluates to true it will be nullable. +nullability + : Bang #nonNullable + | Question #nullable + | Question pattern #nullableIf + ; + +// Type variation suffix. +// +// - If there is no such suffix, the pattern matches any variation that is +// marked as compatible with the system-preferred variation via the function +// behavior option of the variation, as well as the system-preferred +// variation itself. It will evaluate to the system-preferred variation. +// - If the suffix is [?], the pattern matches any variation, and cannot be +// evaluated. +// - If the suffix is [0], the pattern matches and evaluates to the +// system-preferred variation exactly. +// - If the suffix is [ident], the pattern matches and evaluates to the named +// variation exactly. The variation must be in scope. +variation : OpenSquare variationBody CloseSquare ; +variationBody + : Question #varAny + | Zero #varSystemPreferred + | identifierPath #varUserDefined + ; + +// Type parameter pack suffix. +// +// - If there is no such suffix, the pattern accepts any number of parameters +// for the type (assuming that the type class accepts this as well), and +// will attempt to evaluate to a type with no parameters. +// - If there is a "<>" suffix, the pattern accepts only types with zero +// parameters, and will attempt to evaluate to a type with no parameters. +// - If parameters are specified, the pattern accepts only types with exactly +// the specified number of parameters, and will attempt to evaluate to a +// type with exactly those parameters. +parameters : LessThan ( parameter (Comma parameter)* )? GreaterThan ; + +// Type parameter pattern. The name prefix is only used when evaluated (it is +// never matched), and is currently only accepted by the NSTRUCT (pseudo)type. +parameter : ( identifierOrString Colon )? parameterValue ; + +// A pattern for matching potentially-optional parameter values. "null" may be +// used to match or evaluate to explicitly-skipped optional parameters; +// otherwise, the given pattern is used for the parameter value. The "?" (any) +// pattern is special-cased to also match explicitly-skipped parameter slots. +parameterValue : Null #Null | pattern #Specified; + +// Integer literals. +integer : ( Plus | Minus )? ( Zero | Nonzero ) ; + +// When identifying user-defined types and variations, period-separated +// namespace paths are supported. +identifierPath : ( Identifier Period )* Identifier ; + +// The names of parameters (i.e. NSTRUCT field names) can be specified using +// both identifiers and strings. The latter is idiomatic only when the field +// name is not a valid Substrait identifier. +identifierOrString : String #Str | Identifier #Ident ;