feat: introduce ConverterProvider to control conversion behaviour (#649)

Introduces the ConverterProvider class as a single source of
configuration for
all conversion related classes between SQL, Calcite and Substrait. All
conversion classes now have constructors consuming a ConverterProvider.

The no-arg ConverterProvider constructor provides reasonable defaults
for
conversions. Other constructors can override some behaviours, and full
customization can be achieved by extending the ConverterProvider, as is
done in
DynamicConverterProvider

BREAKING CHANGE: removed FeatureBoard class
BREAKING CHANGE: all constructors consuming a FeatureBoard have been
removed
BREAKING CHANGE: a number of methods/constructors have been replaced
with ConverterProvider equivalents

Author: vbarua

isthmus-cli/src/main/java/io/substrait/isthmus/cli/IsthmusEntryPoint.java

@@ -1,12 +1,9 @@
 package io.substrait.isthmus.cli;
 
-import com.google.common.annotations.VisibleForTesting;
 import com.google.protobuf.Message;
 import com.google.protobuf.TextFormat;
 import com.google.protobuf.util.JsonFormat;
-import io.substrait.extension.DefaultExtensionCatalog;
-import io.substrait.isthmus.FeatureBoard;
-import io.substrait.isthmus.ImmutableFeatureBoard;
+import io.substrait.isthmus.ConverterProvider;
 import io.substrait.isthmus.SqlExpressionToSubstrait;
 import io.substrait.isthmus.SqlToSubstrait;
 import io.substrait.isthmus.sql.SubstraitCreateStatementParser;
@@ -18,6 +15,7 @@
 import java.util.concurrent.Callable;
 import org.apache.calcite.avatica.util.Casing;
 import org.apache.calcite.prepare.Prepare;
+import org.apache.calcite.sql.parser.SqlParser;
 import picocli.CommandLine;
 import picocli.CommandLine.Command;
 import picocli.CommandLine.Option;
@@ -62,6 +60,15 @@ enum OutputFormat {
       description = "Calcite's casing policy for unquoted identifiers: ${COMPLETION-CANDIDATES}")
   private Casing unquotedCasing = Casing.TO_UPPER;
 
+  private ConverterProvider converterProvider() {
+    return new ConverterProvider() {
+      @Override
+      public SqlParser.Config getSqlParserConfig() {
+        return super.getSqlParserConfig().withUnquotedCasing(unquotedCasing);
+      }
+    };
+  }
+
   /**
    * Standard Java Main method invoked by the isthmus CLI command.
    *
@@ -89,15 +96,13 @@ public static void main(String... args) {
 
   @Override
   public Integer call() throws Exception {
-    FeatureBoard featureBoard = buildFeatureBoard();
     // Isthmus image is parsing SQL Expression if that argument is defined
     if (sqlExpressions != null) {
-      SqlExpressionToSubstrait converter =
-          new SqlExpressionToSubstrait(featureBoard, DefaultExtensionCatalog.DEFAULT_COLLECTION);
+      SqlExpressionToSubstrait converter = new SqlExpressionToSubstrait(converterProvider());
       ExtendedExpression extendedExpression = converter.convert(sqlExpressions, createStatements);
       printMessage(extendedExpression);
     } else { // by default Isthmus image are parsing SQL Query
-      SqlToSubstrait converter = new SqlToSubstrait(featureBoard);
+      SqlToSubstrait converter = new SqlToSubstrait(converterProvider());
       Prepare.CatalogReader catalog =
           SubstraitCreateStatementParser.processCreateStatementsToCatalog(
               createStatements.toArray(String[]::new));
@@ -116,9 +121,4 @@ private void printMessage(Message message) throws IOException {
       message.writeTo(System.out);
     }
   }
-
-  @VisibleForTesting
-  FeatureBoard buildFeatureBoard() {
-    return ImmutableFeatureBoard.builder().unquotedCasing(unquotedCasing).build();
-  }
 }

isthmus/src/main/java/io/substrait/isthmus/ConverterProvider.java

@@ -0,0 +1,261 @@
+package io.substrait.isthmus;
+
+import io.substrait.extension.DefaultExtensionCatalog;
+import io.substrait.extension.SimpleExtension;
+import io.substrait.isthmus.calcite.SubstraitOperatorTable;
+import io.substrait.isthmus.expression.AggregateFunctionConverter;
+import io.substrait.isthmus.expression.CallConverters;
+import io.substrait.isthmus.expression.ExpressionRexConverter;
+import io.substrait.isthmus.expression.FieldSelectionConverter;
+import io.substrait.isthmus.expression.RexExpressionConverter;
+import io.substrait.isthmus.expression.ScalarFunctionConverter;
+import io.substrait.isthmus.expression.SqlArrayValueConstructorCallConverter;
+import io.substrait.isthmus.expression.SqlMapValueConstructorCallConverter;
+import io.substrait.isthmus.expression.WindowFunctionConverter;
+import io.substrait.relation.Rel;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Function;
+import org.apache.calcite.avatica.util.Casing;
+import org.apache.calcite.config.CalciteConnectionConfig;
+import org.apache.calcite.config.CalciteConnectionProperty;
+import org.apache.calcite.jdbc.CalciteSchema;
+import org.apache.calcite.prepare.Prepare;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.sql.SqlOperatorTable;
+import org.apache.calcite.sql.parser.SqlParser;
+import org.apache.calcite.sql.parser.ddl.SqlDdlParserImpl;
+import org.apache.calcite.sql.validate.SqlConformanceEnum;
+import org.apache.calcite.sql2rel.SqlToRelConverter;
+import org.apache.calcite.tools.Frameworks;
+import org.apache.calcite.tools.RelBuilder;
+
+/**
+ * ConverterProvider provides a single-point of configuration for a number of conversions: {@code
+ * SQl <-> Calcite <-> Substrait}
+ *
+ * <p>It is consumed by all conversion classes as their primary source of configuration.
+ *
+ * <p>The no argument constructor {@link #ConverterProvider()} provides reasonable system defaults.
+ *
+ * <p>Other constructors allow for further customization of conversion behaviours.
+ *
+ * <p>More in-depth customization can be achieved by extending this class, as is done in {@link
+ * DynamicConverterProvider}.
+ */
+public class ConverterProvider {
+
+  protected RelDataTypeFactory typeFactory;
+
+  protected final SimpleExtension.ExtensionCollection extensions;
+
+  protected ScalarFunctionConverter scalarFunctionConverter;
+  protected AggregateFunctionConverter aggregateFunctionConverter;
+  protected WindowFunctionConverter windowFunctionConverter;
+
+  protected TypeConverter typeConverter;
+
+  public ConverterProvider() {
+    this(DefaultExtensionCatalog.DEFAULT_COLLECTION, SubstraitTypeSystem.TYPE_FACTORY);
+  }
+
+  public ConverterProvider(SimpleExtension.ExtensionCollection extensions) {
+    this(extensions, SubstraitTypeSystem.TYPE_FACTORY);
+  }
+
+  public ConverterProvider(
+      SimpleExtension.ExtensionCollection extensions, RelDataTypeFactory typeFactory) {
+    this(
+        typeFactory,
+        extensions,
+        new ScalarFunctionConverter(extensions.scalarFunctions(), typeFactory),
+        new AggregateFunctionConverter(extensions.aggregateFunctions(), typeFactory),
+        new WindowFunctionConverter(extensions.windowFunctions(), typeFactory),
+        TypeConverter.DEFAULT);
+  }
+
+  public ConverterProvider(
+      RelDataTypeFactory typeFactory,
+      SimpleExtension.ExtensionCollection extensions,
+      ScalarFunctionConverter sfc,
+      AggregateFunctionConverter afc,
+      WindowFunctionConverter wfc,
+      TypeConverter tc) {
+    this.typeFactory = typeFactory;
+    this.extensions = extensions;
+    this.scalarFunctionConverter = sfc;
+    this.aggregateFunctionConverter = afc;
+    this.windowFunctionConverter = wfc;
+    this.typeConverter = tc;
+  }
+
+  // SQL to Calcite Processing
+
+  /**
+   * {@link SqlParser.Config} is a Calcite class which controls SQL parsing behaviour like
+   * identifier casing.
+   */
+  public SqlParser.Config getSqlParserConfig() {
+    return SqlParser.Config.DEFAULT
+        .withUnquotedCasing(Casing.TO_UPPER)
+        .withParserFactory(SqlDdlParserImpl.FACTORY)
+        .withConformance(SqlConformanceEnum.LENIENT);
+  }
+
+  /**
+   * {@link CalciteConnectionConfig} is a Calcite class which controls SQL processing behaviour like
+   * table name case-sensitivity.
+   */
+  public CalciteConnectionConfig getCalciteConnectionConfig() {
+    return CalciteConnectionConfig.DEFAULT.set(CalciteConnectionProperty.CASE_SENSITIVE, "false");
+  }
+
+  /**
+   * {@link SqlToRelConverter.Config} is a Calcite class which controls SQL processing behaviour
+   * like field-trimming.
+   */
+  public SqlToRelConverter.Config getSqlToRelConverterConfig() {
+    return SqlToRelConverter.config().withTrimUnusedFields(true).withExpand(false);
+  }
+
+  /**
+   * {@link SqlOperatorTable} is a Calcite class which stores the {@link
+   * org.apache.calcite.sql.SqlOperator}s available and controls valid identifiers during SQL
+   * processing.
+   */
+  public SqlOperatorTable getSqlOperatorTable() {
+    return SubstraitOperatorTable.INSTANCE;
+  }
+
+  // Substrait to Calcite Processing
+
+  /**
+   * {@link SubstraitToCalcite} is an Isthmus class for converting a Substrait {@link Rel} or {@link
+   * io.substrait.plan.Plan.Root} to a Calcite {@link org.apache.calcite.rel.RelNode} or {@link
+   * org.apache.calcite.rel.RelRoot}
+   */
+  protected SubstraitToCalcite getSubstraitToCalcite() {
+    return new SubstraitToCalcite(this);
+  }
+
+  /**
+   * {@link SubstraitToCalcite} is an Isthmus class for converting a Substrait {@link Rel} or {@link
+   * io.substrait.plan.Plan.Root} to a Calcite {@link org.apache.calcite.rel.RelNode} or {@link
+   * org.apache.calcite.rel.RelRoot}
+   *
+   * @param catalogReader a Calcite {@link Prepare.CatalogReader} used to construct a {@link
+   *     RelBuilder} for use in creating Calcite {@link org.apache.calcite.rel.RelNode}s
+   */
+  protected SubstraitToCalcite getSubstraitToCalcite(Prepare.CatalogReader catalogReader) {
+    return new SubstraitToCalcite(this, catalogReader);
+  }
+
+  // Calcite to Substrait Processing
+
+  /**
+   * A {@link SubstraitRelVisitor} converts Calcite {@link org.apache.calcite.rel.RelNode}s to
+   * Substrait {@link Rel}s
+   */
+  public SubstraitRelVisitor getSubstraitRelVisitor() {
+    return new SubstraitRelVisitor(this);
+  }
+
+  /**
+   * A {@link RexExpressionConverter} converts Calcite {@link org.apache.calcite.rex.RexNode}s to
+   * Substrait equivalents.
+   */
+  public RexExpressionConverter getRexExpressionConverter(SubstraitRelVisitor srv) {
+    return new RexExpressionConverter(
+        srv, getCallConverters(), getWindowFunctionConverter(), getTypeConverter());
+  }
+
+  /**
+   * {@link CallConverter}s are used to convert Calcite {@link org.apache.calcite.rex.RexCall}s to
+   * Substrait equivalents.
+   */
+  public List<CallConverter> getCallConverters() {
+    ArrayList<CallConverter> callConverters = new ArrayList<>();
+    callConverters.add(new FieldSelectionConverter(typeConverter));
+    callConverters.add(CallConverters.CASE);
+    callConverters.add(CallConverters.CAST.apply(typeConverter));
+    callConverters.add(CallConverters.REINTERPRET.apply(typeConverter));
+    callConverters.add(new SqlArrayValueConstructorCallConverter(typeConverter));
+    callConverters.add(new SqlMapValueConstructorCallConverter());
+    callConverters.add(CallConverters.CREATE_SEARCH_CONV.apply(new RexBuilder(typeFactory)));
+    callConverters.add(scalarFunctionConverter);
+    return callConverters;
+  }
+
+  // Substrait To Calcite Processing
+
+  /**
+   * When converting from Substrait to Calcite, Calcite needs to have a schema available. The
+   * default strategy uses a {@link SchemaCollector} to generate a {@link CalciteSchema} on the fly
+   * based on the leaf nodes of a Substrait plan.
+   *
+   * <p>Override to customize the schema generation behaviour
+   */
+  public Function<Rel, CalciteSchema> getSchemaResolver() {
+    SchemaCollector schemaCollector = new SchemaCollector(this);
+    return schemaCollector::toSchema;
+  }
+
+  /**
+   * A {@link SubstraitRelNodeConverter} is used when converting from Substrait {@link Rel}s to
+   * Calcite {@link org.apache.calcite.rel.RelNode}s.
+   */
+  public SubstraitRelNodeConverter getSubstraitRelNodeConverter(RelBuilder relBuilder) {
+    return new SubstraitRelNodeConverter(relBuilder, this);
+  }
+
+  /**
+   * A {@link ExpressionRexConverter} converts Substrait {@link io.substrait.expression.Expression}
+   * to Calcite equivalents
+   */
+  public ExpressionRexConverter getExpressionRexConverter(
+      SubstraitRelNodeConverter relNodeConverter) {
+    ExpressionRexConverter erc =
+        new ExpressionRexConverter(
+            getTypeFactory(),
+            getScalarFunctionConverter(),
+            getWindowFunctionConverter(),
+            getTypeConverter());
+    erc.setRelNodeConverter(relNodeConverter);
+    return erc;
+  }
+
+  /**
+   * A {@link RelBuilder} is a Calcite class used as a factory for creating {@link
+   * org.apache.calcite.rel.RelNode}s.
+   */
+  public RelBuilder getRelBuilder(CalciteSchema schema) {
+    return RelBuilder.create(Frameworks.newConfigBuilder().defaultSchema(schema.plus()).build());
+  }
+
+  // Utility Getters
+
+  public RelDataTypeFactory getTypeFactory() {
+    return typeFactory;
+  }
+
+  public SimpleExtension.ExtensionCollection getExtensions() {
+    return extensions;
+  }
+
+  public ScalarFunctionConverter getScalarFunctionConverter() {
+    return scalarFunctionConverter;
+  }
+
+  public AggregateFunctionConverter getAggregateFunctionConverter() {
+    return aggregateFunctionConverter;
+  }
+
+  public WindowFunctionConverter getWindowFunctionConverter() {
+    return windowFunctionConverter;
+  }
+
+  public TypeConverter getTypeConverter() {
+    return typeConverter;
+  }
+}