diff --git a/backend/pom.xml b/backend/pom.xml index b31ca86ca3..5d5b69d9f8 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -373,6 +373,11 @@ testcontainers-postgresql test + + org.testcontainers + testcontainers-clickhouse + test + org.testcontainers testcontainers-solr @@ -383,6 +388,12 @@ ngdbc 2.17.10 + + com.clickhouse + clickhouse-jdbc + 0.9.8 + all + io.prometheus simpleclient_dropwizard diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java b/backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java index 74d4966d5f..8768f24c98 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java @@ -33,6 +33,7 @@ import org.jooq.Select; import org.jooq.SelectConditionStep; import org.jooq.Table; +import org.jooq.impl.DSL; @RequiredArgsConstructor public class SqlEntityResolver implements EntityResolver { @@ -144,7 +145,8 @@ private Map resolveIds(String[][] values, List rowIndex = field(name(ROW_INDEX), Integer.class); Field externalPrimaryColumn = field(name(SharedAliases.PRIMARY_COLUMN.getAlias()), String.class); Field innerPrimaryColumn = field(name(idColumns.findPrimaryIdColumn().getField()), String.class); - Field isResolved = innerPrimaryColumn.isNotNull().as(IS_RESOLVED_ALIAS); + // Would prefer this to be `is not null`, but hana does not support that for fields + Field isResolved = case_().when(innerPrimaryColumn.isNull(), inline(false)).otherwise(inline(true)).as(IS_RESOLVED_ALIAS); Table allIdsTable = table(name(idColumns.getTable())); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java b/backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java index c1effedb58..0bb1507f68 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java @@ -1,8 +1,9 @@ package com.bakdata.conquery.models.config; import com.bakdata.conquery.sql.conversion.dialect.DialectBundle; -import com.bakdata.conquery.sql.conversion.dialect.HanaDialectBundle; -import com.bakdata.conquery.sql.conversion.dialect.PostgreDialectBundle; +import com.bakdata.conquery.sql.conversion.dialect.clickhouse.ClickhouseDialectBundle; +import com.bakdata.conquery.sql.conversion.dialect.hana.HanaDialectBundle; +import com.bakdata.conquery.sql.conversion.dialect.pg.PostgreDialectBundle; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -15,13 +16,8 @@ @Getter public enum Dialect { - /** - * Dialect for PostgreSQL database - */ POSTGRESQL(new PostgreDialectBundle()), - /** - * Dialect for SAP HANA database - */ + CLICKHOUSE(new ClickhouseDialectBundle()), HANA(new HanaDialectBundle()); private final DialectBundle dialectBundle; diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java b/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java index 59cfaf3022..cf94d9953e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java @@ -78,6 +78,10 @@ public Collection generateResultURLs(ManagedExecution exec, UriBuil return Collections.emptyList(); } + if(singleExecution.getResultInfos() == null){ + return Collections.emptyList(); + } + // Save id column count to later check if xlsx dimensions are feasible idColumnsCount = exec.getConfig().getIdColumns().getIdResultInfos().size(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java index a014283044..94938199a0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java @@ -17,6 +17,7 @@ import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; import com.bakdata.conquery.models.query.resultinfo.printers.common.OneToManyMappingPrinter; import com.bakdata.conquery.models.types.ResultType; +import com.bakdata.conquery.sql.conversion.model.select.ClickhouseDistinctSelectConverter; import com.bakdata.conquery.sql.conversion.model.select.DistinctSelectConverter; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.bakdata.conquery.sql.execution.ResultSetProcessor; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListStringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListStringPrinter.java index 218a37d71f..0e62677566 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListStringPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListStringPrinter.java @@ -1,6 +1,7 @@ package com.bakdata.conquery.models.query.resultinfo.printers.common; import java.util.Collection; +import java.util.Objects; import java.util.StringJoiner; import com.bakdata.conquery.models.config.LocaleConfig; @@ -24,7 +25,7 @@ public String apply(@NotNull Collection f) { continue; } - joiner.add(listFormat.escapeListElement(elementPrinter.apply(obj).toString())); + joiner.add(listFormat.escapeListElement(Objects.toString(elementPrinter.apply(obj)))); } return joiner.toString(); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java index 30ef3a59b3..9b89287910 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java @@ -59,12 +59,12 @@ private Selects getInvertSelects(QueryStep rowNumberStep, SqlIdColumns coalesced Field rangeStart = DSL.coalesce( QualifyingUtil.qualify(validityDate.getEnd(), ROWS_LEFT_TABLE_NAME), - functionProvider.toDateField(functionProvider.getMinDateExpression()) + (functionProvider.getMinDateExpression()) ).as(DateAggregationCte.RANGE_START); Field rangeEnd = DSL.coalesce( QualifyingUtil.qualify(validityDate.getStart(), ROWS_RIGHT_TABLE_NAME), - functionProvider.toDateField(functionProvider.getMaxDateExpression()) + (functionProvider.getMaxDateExpression()) ).as(DateAggregationCte.RANGE_END); return Selects.builder() diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/PostgreSqlDateAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/PostgreSqlDateAggregator.java index 08e74dfcf8..58d2e4083c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/PostgreSqlDateAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/PostgreSqlDateAggregator.java @@ -84,8 +84,8 @@ public QueryStep invertAggregatedIntervals(QueryStep baseStep, ConversionContext Field maxDateRange = DSL.function( "daterange", Object.class, - this.functionProvider.toDateField(this.functionProvider.getMinDateExpression()), - this.functionProvider.toDateField(this.functionProvider.getMaxDateExpression()), + (this.functionProvider.getMinDateExpression()), + (this.functionProvider.getMaxDateExpression()), DSL.inline("[]") ); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java index 17bc77eefe..1bc24379dc 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java @@ -116,7 +116,7 @@ private static QueryStep finishConceptConversion(QueryStep predecessor, CQConcep .fromTable(joinedTable) .groupBy(groupByFields) .predecessors(queriesToJoin) - .negate(context.isNegation()) + .negate(context.isNegation()) .build(); } @@ -146,7 +146,7 @@ public static SqlIdColumns convertIds(CQConcept cqConcept, CQTable cqTable, Conv return new SqlIdColumns(primaryColumn, secondaryId).withAlias(); } - private static Optional convertValidityDate(CQTable cqTable, String connectorLabel, ConversionContext context) { + private static ColumnDateRange convertValidityDate(CQTable cqTable, String connectorLabel, ConversionContext context) { SqlFunctionProvider functionProvider = context.getFunctionProvider(); ValidityDate validityDate = cqTable.findValidityDate(); ColumnDateRange sqlValidityDate; @@ -167,11 +167,11 @@ private static Optional convertValidityDate(CQTable cqTable, St sqlValidityDate = functionProvider.forCDateRange(context.getDateRestrictionRange()); } else { - return Optional.empty(); + sqlValidityDate = functionProvider.allRange(); } } - return Optional.of(sqlValidityDate.asValidityDateRange(connectorLabel)); + return sqlValidityDate.asValidityDateRange(connectorLabel); } private static Optional collectConditionFilters(List> conceptElements, CQTable cqTable, SqlFunctionProvider functionProvider) { @@ -223,11 +223,11 @@ private static SqlFilters dateRestrictionFilter(ConversionContext context, Colum List dateRestrictionSelects = new ArrayList<>(); List conditions = new ArrayList<>(); + SqlFunctionProvider functionProvider = context.getFunctionProvider(); - conditions.add(ConditionUtil.wrap(validityDate.isNotEmpty())); + conditions.add(ConditionUtil.wrap(functionProvider.isNotEmptyDateRange(validityDate))); if (context.getDateRestrictionRange() != null) { - SqlFunctionProvider functionProvider = context.getFunctionProvider(); ColumnDateRange dateRestriction = functionProvider.forCDateRange(context.getDateRestrictionRange()).as(SharedAliases.DATE_RESTRICTION.getAlias()); conditions.add(ConditionUtil.wrap(functionProvider.dateRestriction(dateRestriction, validityDate))); @@ -245,11 +245,13 @@ private static SqlFilters dateRestrictionFilter(ConversionContext context, Colum } private static ConnectorSqlSelects createConceptColumnConnectorSqlSelects(CQConcept cqConcept, SelectContext selectContext) { + + return cqConcept.getSelects().stream() .map(SelectId::resolve) .filter(select -> select instanceof ConceptColumnSelect) .findFirst() - .map(select -> select.createConverter().connectorSelect(select, selectContext)) + .map(select -> selectContext.getDialectBundle().getSelectConverter(select).connectorSelect(select, selectContext)) .orElse(ConnectorSqlSelects.none()); } @@ -289,7 +291,7 @@ private CQTableContext createTableContext(TablePath tablePath, CQConcept cqConce SqlIdColumns ids = convertIds(cqConcept, cqTable, conversionContext); ConnectorSqlTables connectorTables = tablePath.getConnectorTables(cqTable); - Optional tablesValidityDate = convertValidityDate(cqTable, connectorTables.getLabel(), conversionContext); + ColumnDateRange tablesValidityDate = convertValidityDate(cqTable, connectorTables.getLabel(), conversionContext); // convert filters SqlFunctionProvider functionProvider = conversionContext.getFunctionProvider(); @@ -299,12 +301,11 @@ private CQTableContext createTableContext(TablePath tablePath, CQConcept cqConce .forEach(allSqlFiltersForTable::add); collectConditionFilters(cqConcept.getElements().stream().>map(ConceptElementId::resolve).toList(), cqTable, functionProvider).ifPresent( allSqlFiltersForTable::add); - if (tablesValidityDate.isPresent()) { - allSqlFiltersForTable.add(dateRestrictionFilter(conversionContext, tablesValidityDate.get())); - } + + allSqlFiltersForTable.add(dateRestrictionFilter(conversionContext, tablesValidityDate)); // convert selects - SelectContext selectContext = SelectContext.create(ids, tablesValidityDate, connectorTables, conversionContext); + SelectContext selectContext = SelectContext.create(ids, Optional.of(tablesValidityDate), connectorTables, conversionContext); List allSelectsForTable = new ArrayList<>(); ConnectorSqlSelects conceptColumnSelect = createConceptColumnConnectorSqlSelects(cqConcept, selectContext); allSelectsForTable.add(conceptColumnSelect); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java index 7c83306b23..de565ccb5f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java @@ -24,7 +24,7 @@ class CQTableContext implements Context { SqlIdColumns ids; - Optional validityDate; + ColumnDateRange validityDate; List sqlSelects; List sqlFilters; ConnectorSqlTables connectorTables; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java index da826be3a0..b2f6718dfd 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java @@ -58,7 +58,7 @@ protected QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { // validity date aggregation Optional validityDate; - if (tableContext.getValidityDate().isEmpty() || !tableContext.getConnectorTables().isWithIntervalPacking()) { + if (!tableContext.getConnectorTables().isWithIntervalPacking()) { validityDate = Optional.empty(); } else { @@ -97,7 +97,7 @@ private static IntervalPackingContext createIntervalPackingContext(CQTableContex Selects predcessorSelects = tableContext.getPrevious().getQualifiedSelects(); return IntervalPackingContext.builder() .ids(predcessorSelects.getIds()) - .daterange(tableContext.getValidityDate().get()) + .daterange(tableContext.getValidityDate()) .tables(tableContext.getConnectorTables()) .build(); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java index 80f377a025..3ced541d97 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java @@ -2,6 +2,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Optional; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.QueryStep; @@ -32,7 +33,7 @@ public QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { Selects preprocessingSelects = Selects.builder() .ids(tableContext.getIds()) - .validityDate(tableContext.getValidityDate()) + .validityDate(Optional.of(tableContext.getValidityDate())) .sqlSelects(forPreprocessing) .build(); // all where clauses that don't require any preprocessing (connector/child conditions) @@ -81,7 +82,7 @@ private static QueryStep.QueryStepBuilder joinWithStratificationTable( Selects selects = Selects.builder() .ids(stratificationSelects.getIds()) - .validityDate(tableContext.getValidityDate()) + .validityDate(Optional.of(tableContext.getValidityDate())) .stratificationDate(stratificationSelects.getStratificationDate()) .sqlSelects(preprocessingSelects) .build(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/DialectBundle.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/DialectBundle.java index e66402d491..adf86a157b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/DialectBundle.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/DialectBundle.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.sql.conversion.dialect; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -7,6 +8,7 @@ import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.Dialect; +import com.bakdata.conquery.models.datasets.concepts.select.Select; import com.bakdata.conquery.models.events.MajorTypeId; import com.bakdata.conquery.models.query.Visitable; import com.bakdata.conquery.sql.conversion.Converter; @@ -20,6 +22,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.concept.CQConceptConverter; import com.bakdata.conquery.sql.conversion.forms.StratificationFunctions; import com.bakdata.conquery.sql.conversion.model.QueryStepTransformer; +import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.bakdata.conquery.sql.conversion.query.AbsoluteFormQueryConverter; import com.bakdata.conquery.sql.conversion.query.CQReusedQueryConverter; import com.bakdata.conquery.sql.conversion.query.ConceptQueryConverter; @@ -34,7 +37,6 @@ import org.jooq.Field; import org.jooq.SQLDialect; -//TODO unify with com.bakdata.conquery.models.config.Dialect public interface DialectBundle { private static > List customize(List defaults, List substitutes) { @@ -104,4 +106,17 @@ default List> getDefaultNodeConverters(DSLCon ); } + default Map, ? extends SelectConverter> getSelectConverterOverrides(){ + return Collections.emptyMap(); + } + + default SelectConverter maybeOverride = (SelectConverter