Feature/missing hana tests

backend/pom.xml

@@ -373,6 +373,11 @@
             <artifactId>testcontainers-postgresql</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>testcontainers-clickhouse</artifactId>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.testcontainers</groupId>
             <artifactId>testcontainers-solr</artifactId>
@@ -383,6 +388,12 @@
             <artifactId>ngdbc</artifactId>
             <version>2.17.10</version>
         </dependency>
+        <dependency>
+            <groupId>com.clickhouse</groupId>
+            <artifactId>clickhouse-jdbc</artifactId>
+            <version>0.9.8</version>
+            <classifier>all</classifier>
+        </dependency>
         <dependency>
             <groupId>io.prometheus</groupId>
             <artifactId>simpleclient_dropwizard</artifactId>

backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java

@@ -33,6 +33,7 @@
 import org.jooq.Select;
 import org.jooq.SelectConditionStep;
 import org.jooq.Table;
+import org.jooq.impl.DSL;
 
 @RequiredArgsConstructor
 public class SqlEntityResolver implements EntityResolver {
@@ -144,7 +145,8 @@ private Map<Integer, IdResolveInfo> resolveIds(String[][] values, List<Function<
 		Field<Integer> rowIndex = field(name(ROW_INDEX), Integer.class);
 		Field<String> externalPrimaryColumn = field(name(SharedAliases.PRIMARY_COLUMN.getAlias()), String.class);
 		Field<String> innerPrimaryColumn = field(name(idColumns.findPrimaryIdColumn().getField()), String.class);
-		Field<Boolean> isResolved = innerPrimaryColumn.isNotNull().as(IS_RESOLVED_ALIAS);
+		// Would prefer this to be `is not null`, but hana does not support that for fields
+		Field<Boolean> isResolved = case_().when(innerPrimaryColumn.isNull(), inline(false)).otherwise(inline(true)).as(IS_RESOLVED_ALIAS);
 
 		Table<Record> allIdsTable = table(name(idColumns.getTable()));
 

backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java

@@ -1,8 +1,9 @@
 package com.bakdata.conquery.models.config;
 
 import com.bakdata.conquery.sql.conversion.dialect.DialectBundle;
-import com.bakdata.conquery.sql.conversion.dialect.HanaDialectBundle;
-import com.bakdata.conquery.sql.conversion.dialect.PostgreDialectBundle;
+import com.bakdata.conquery.sql.conversion.dialect.clickhouse.ClickhouseDialectBundle;
+import com.bakdata.conquery.sql.conversion.dialect.hana.HanaDialectBundle;
+import com.bakdata.conquery.sql.conversion.dialect.pg.PostgreDialectBundle;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
 
@@ -15,13 +16,8 @@
 @Getter
 public enum Dialect {
 
-	/**
-	 * Dialect for PostgreSQL database
-	 */
 	POSTGRESQL(new PostgreDialectBundle()),
-	/**
-	 * Dialect for SAP HANA database
-	 */
+	CLICKHOUSE(new ClickhouseDialectBundle()),
 	HANA(new HanaDialectBundle());
 
 	private final DialectBundle dialectBundle;

backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java

@@ -78,6 +78,10 @@ public Collection<ResultAsset> generateResultURLs(ManagedExecution exec, UriBuil
 			return Collections.emptyList();
 		}
 
+		if(singleExecution.getResultInfos() == null){
+			return Collections.emptyList();
+		}
+
 		// Save id column count to later check if xlsx dimensions are feasible
 		idColumnsCount = exec.getConfig().getIdColumns().getIdResultInfos().size();
 

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java

@@ -17,6 +17,7 @@
 import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory;
 import com.bakdata.conquery.models.query.resultinfo.printers.common.OneToManyMappingPrinter;
 import com.bakdata.conquery.models.types.ResultType;
+import com.bakdata.conquery.sql.conversion.model.select.ClickhouseDistinctSelectConverter;
 import com.bakdata.conquery.sql.conversion.model.select.DistinctSelectConverter;
 import com.bakdata.conquery.sql.conversion.model.select.SelectConverter;
 import com.bakdata.conquery.sql.execution.ResultSetProcessor;

backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListStringPrinter.java

@@ -1,6 +1,7 @@
 package com.bakdata.conquery.models.query.resultinfo.printers.common;
 
 import java.util.Collection;
+import java.util.Objects;
 import java.util.StringJoiner;
 
 import com.bakdata.conquery.models.config.LocaleConfig;
@@ -24,7 +25,7 @@ public String apply(@NotNull Collection<T> f) {
 				continue;
 			}
 
-			joiner.add(listFormat.escapeListElement(elementPrinter.apply(obj).toString()));
+			joiner.add(listFormat.escapeListElement(Objects.toString(elementPrinter.apply(obj))));
 		}
 		return joiner.toString();
 	}

backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java

@@ -59,12 +59,12 @@ private Selects getInvertSelects(QueryStep rowNumberStep, SqlIdColumns coalesced
 
 		Field<Date> rangeStart = DSL.coalesce(
 				QualifyingUtil.qualify(validityDate.getEnd(), ROWS_LEFT_TABLE_NAME),
-				functionProvider.toDateField(functionProvider.getMinDateExpression())
+				(functionProvider.getMinDateExpression())
 		).as(DateAggregationCte.RANGE_START);
 
 		Field<Date> rangeEnd = DSL.coalesce(
 				QualifyingUtil.qualify(validityDate.getStart(), ROWS_RIGHT_TABLE_NAME),
-				functionProvider.toDateField(functionProvider.getMaxDateExpression())
+				(functionProvider.getMaxDateExpression())
 		).as(DateAggregationCte.RANGE_END);
 
 		return Selects.builder()

backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/PostgreSqlDateAggregator.java

@@ -84,8 +84,8 @@ public QueryStep invertAggregatedIntervals(QueryStep baseStep, ConversionContext
 		Field<Object> maxDateRange = DSL.function(
 				"daterange",
 				Object.class,
-				this.functionProvider.toDateField(this.functionProvider.getMinDateExpression()),
-				this.functionProvider.toDateField(this.functionProvider.getMaxDateExpression()),
+				(this.functionProvider.getMinDateExpression()),
+				(this.functionProvider.getMaxDateExpression()),
 				DSL.inline("[]")
 		);
 

backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java

@@ -116,7 +116,7 @@ private static QueryStep finishConceptConversion(QueryStep predecessor, CQConcep
 						.fromTable(joinedTable)
 						.groupBy(groupByFields)
 						.predecessors(queriesToJoin)
-                        .negate(context.isNegation())
+						.negate(context.isNegation())
 						.build();
 	}
 
@@ -146,7 +146,7 @@ public static SqlIdColumns convertIds(CQConcept cqConcept, CQTable cqTable, Conv
 		return new SqlIdColumns(primaryColumn, secondaryId).withAlias();
 	}
 
-	private static Optional<ColumnDateRange> convertValidityDate(CQTable cqTable, String connectorLabel, ConversionContext context) {
+	private static ColumnDateRange convertValidityDate(CQTable cqTable, String connectorLabel, ConversionContext context) {
 		SqlFunctionProvider functionProvider = context.getFunctionProvider();
 		ValidityDate validityDate = cqTable.findValidityDate();
 		ColumnDateRange sqlValidityDate;
@@ -167,11 +167,11 @@ private static Optional<ColumnDateRange> convertValidityDate(CQTable cqTable, St
 				sqlValidityDate = functionProvider.forCDateRange(context.getDateRestrictionRange());
 			}
 			else {
-				return Optional.empty();
+				sqlValidityDate = functionProvider.allRange();
 			}
 		}
 
-		return Optional.of(sqlValidityDate.asValidityDateRange(connectorLabel));
+		return sqlValidityDate.asValidityDateRange(connectorLabel);
 	}
 
 	private static Optional<SqlFilters> collectConditionFilters(List<ConceptElement<?>> conceptElements, CQTable cqTable, SqlFunctionProvider functionProvider) {
@@ -223,11 +223,11 @@ private static SqlFilters dateRestrictionFilter(ConversionContext context, Colum
 
 		List<SqlSelect> dateRestrictionSelects = new ArrayList<>();
 		List<WhereCondition> conditions = new ArrayList<>();
+		SqlFunctionProvider functionProvider = context.getFunctionProvider();
 
-		conditions.add(ConditionUtil.wrap(validityDate.isNotEmpty()));
+		conditions.add(ConditionUtil.wrap(functionProvider.isNotEmptyDateRange(validityDate)));
 
 		if (context.getDateRestrictionRange() != null) {
-			SqlFunctionProvider functionProvider = context.getFunctionProvider();
 			ColumnDateRange dateRestriction = functionProvider.forCDateRange(context.getDateRestrictionRange()).as(SharedAliases.DATE_RESTRICTION.getAlias());
 			conditions.add(ConditionUtil.wrap(functionProvider.dateRestriction(dateRestriction, validityDate)));
 
@@ -245,11 +245,13 @@ private static SqlFilters dateRestrictionFilter(ConversionContext context, Colum
 	}
 
 	private static ConnectorSqlSelects createConceptColumnConnectorSqlSelects(CQConcept cqConcept, SelectContext<ConnectorSqlTables> selectContext) {
+
+
 		return cqConcept.getSelects().stream()
 						.map(SelectId::resolve)
 						.filter(select -> select instanceof ConceptColumnSelect)
 						.findFirst()
-						.map(select -> select.createConverter().connectorSelect(select, selectContext))
+						.map(select -> selectContext.getDialectBundle().getSelectConverter(select).connectorSelect(select, selectContext))
 						.orElse(ConnectorSqlSelects.none());
 	}
 
@@ -289,7 +291,7 @@ private CQTableContext createTableContext(TablePath tablePath, CQConcept cqConce
 
 		SqlIdColumns ids = convertIds(cqConcept, cqTable, conversionContext);
 		ConnectorSqlTables connectorTables = tablePath.getConnectorTables(cqTable);
-		Optional<ColumnDateRange> tablesValidityDate = convertValidityDate(cqTable, connectorTables.getLabel(), conversionContext);
+		ColumnDateRange tablesValidityDate = convertValidityDate(cqTable, connectorTables.getLabel(), conversionContext);
 
 		// convert filters
 		SqlFunctionProvider functionProvider = conversionContext.getFunctionProvider();
@@ -299,12 +301,11 @@ private CQTableContext createTableContext(TablePath tablePath, CQConcept cqConce
 			   .forEach(allSqlFiltersForTable::add);
 		collectConditionFilters(cqConcept.getElements().stream().<ConceptElement<?>>map(ConceptElementId::resolve).toList(), cqTable, functionProvider).ifPresent(
 				allSqlFiltersForTable::add);
-		if (tablesValidityDate.isPresent()) {
-			allSqlFiltersForTable.add(dateRestrictionFilter(conversionContext, tablesValidityDate.get()));
-		}
+
+		allSqlFiltersForTable.add(dateRestrictionFilter(conversionContext, tablesValidityDate));
 
 		// convert selects
-		SelectContext<ConnectorSqlTables> selectContext = SelectContext.create(ids, tablesValidityDate, connectorTables, conversionContext);
+		SelectContext<ConnectorSqlTables> selectContext = SelectContext.create(ids, Optional.of(tablesValidityDate), connectorTables, conversionContext);
 		List<ConnectorSqlSelects> allSelectsForTable = new ArrayList<>();
 		ConnectorSqlSelects conceptColumnSelect = createConceptColumnConnectorSqlSelects(cqConcept, selectContext);
 		allSelectsForTable.add(conceptColumnSelect);

backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java

@@ -24,7 +24,7 @@ class CQTableContext implements Context {
 
 
 	SqlIdColumns ids;
-	Optional<ColumnDateRange> validityDate;
+	ColumnDateRange validityDate;
 	List<ConnectorSqlSelects> sqlSelects;
 	List<SqlFilters> sqlFilters;
 	ConnectorSqlTables connectorTables;

backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java

@@ -58,7 +58,7 @@ protected QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) {
 
 		// validity date aggregation
 		Optional<ColumnDateRange> validityDate;
-		if (tableContext.getValidityDate().isEmpty() || !tableContext.getConnectorTables().isWithIntervalPacking()) {
+		if (!tableContext.getConnectorTables().isWithIntervalPacking()) {
 			validityDate = Optional.empty();
 		}
 		else {
@@ -97,7 +97,7 @@ private static IntervalPackingContext createIntervalPackingContext(CQTableContex
 		Selects predcessorSelects = tableContext.getPrevious().getQualifiedSelects();
 		return IntervalPackingContext.builder()
 									 .ids(predcessorSelects.getIds())
-									 .daterange(tableContext.getValidityDate().get())
+									 .daterange(tableContext.getValidityDate())
 									 .tables(tableContext.getConnectorTables())
 									 .build();
 	}

backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java

@@ -2,6 +2,7 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Optional;
 
 import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider;
 import com.bakdata.conquery.sql.conversion.model.QueryStep;
@@ -32,7 +33,7 @@ public QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) {
 
 		Selects preprocessingSelects = Selects.builder()
 											  .ids(tableContext.getIds())
-											  .validityDate(tableContext.getValidityDate())
+											  .validityDate(Optional.of(tableContext.getValidityDate()))
 											  .sqlSelects(forPreprocessing)
 											  .build();
 		// all where clauses that don't require any preprocessing (connector/child conditions)
@@ -81,7 +82,7 @@ private static QueryStep.QueryStepBuilder joinWithStratificationTable(
 
 		Selects selects = Selects.builder()
 								 .ids(stratificationSelects.getIds())
-								 .validityDate(tableContext.getValidityDate())
+								 .validityDate(Optional.of(tableContext.getValidityDate()))
 								 .stratificationDate(stratificationSelects.getStratificationDate())
 								 .sqlSelects(preprocessingSelects)
 								 .build();

backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/DialectBundle.java

@@ -1,12 +1,14 @@
 package com.bakdata.conquery.sql.conversion.dialect;
 
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
 import com.bakdata.conquery.models.config.ConqueryConfig;
 import com.bakdata.conquery.models.config.Dialect;
+import com.bakdata.conquery.models.datasets.concepts.select.Select;
 import com.bakdata.conquery.models.events.MajorTypeId;
 import com.bakdata.conquery.models.query.Visitable;
 import com.bakdata.conquery.sql.conversion.Converter;
@@ -20,6 +22,7 @@
 import com.bakdata.conquery.sql.conversion.cqelement.concept.CQConceptConverter;
 import com.bakdata.conquery.sql.conversion.forms.StratificationFunctions;
 import com.bakdata.conquery.sql.conversion.model.QueryStepTransformer;
+import com.bakdata.conquery.sql.conversion.model.select.SelectConverter;
 import com.bakdata.conquery.sql.conversion.query.AbsoluteFormQueryConverter;
 import com.bakdata.conquery.sql.conversion.query.CQReusedQueryConverter;
 import com.bakdata.conquery.sql.conversion.query.ConceptQueryConverter;
@@ -34,7 +37,6 @@
 import org.jooq.Field;
 import org.jooq.SQLDialect;
 
-//TODO unify with com.bakdata.conquery.models.config.Dialect
 public interface DialectBundle {
 
 	private static <R, C extends Converter<?, R, ?>> List<C> customize(List<C> defaults, List<C> substitutes) {
@@ -104,4 +106,17 @@ default List<NodeConverter<? extends Visitable>> getDefaultNodeConverters(DSLCon
 		);
 	}
 
+	default Map<Class<? extends Select>, ? extends SelectConverter<? extends Select>> getSelectConverterOverrides(){
+		return Collections.emptyMap();
+	}
+
+	default SelectConverter<Select> getSelectConverter(Select select){
+		SelectConverter<Select> maybeOverride = (SelectConverter<Select>) getSelectConverterOverrides().get(select.getClass());
+
+		if (maybeOverride != null){
+			return maybeOverride;
+		}
+
+		return select.createConverter();
+	}
 }

backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java

@@ -13,7 +13,6 @@
 import com.bakdata.conquery.apiv1.query.concept.filter.CQTable;
 import com.bakdata.conquery.models.common.CDateSet;
 import com.bakdata.conquery.models.common.daterange.CDateRange;
-import com.bakdata.conquery.models.datasets.Column;
 import com.bakdata.conquery.models.datasets.concepts.DaterangeSelectOrFilter;
 import com.bakdata.conquery.models.datasets.concepts.ValidityDate;
 import com.bakdata.conquery.sql.conversion.SharedAliases;
@@ -43,9 +42,9 @@ Collection<? extends OrderField<?>> orderByValidityDates(
 			Function<Field<?>, ? extends SortField<?>> ordering,
 			List<Field<?>> validityDateFields);
 
-	String getMinDateExpression();
+	Field<Date> getMinDateExpression();
 
-	String getMaxDateExpression();
+	Field<Date> getMaxDateExpression();
 
 	<T> Field<T> cast(Field<?> field, DataType<T> type);
 
@@ -180,7 +179,7 @@ default Field<String> stringAggregation(Field<String> stringField, Field<String>
 
 	ColumnDateRange allRangeIf(Condition condition);
 
-	default Field<String> concat(List<Field<String>> fields) {
+	default Field<?> stringArray(List<Field<String>> fields) {
 		String concatenated =
 				fields.stream()
 					  // if a field is null, the whole concatenation would be null - but we just want to skip this field in this case,
@@ -229,20 +228,7 @@ default Field<Date> toDateField(String dateExpression) {
 		return toDate(dateExpression, DEFAULT_DATE_FORMAT);
 	}
 
-	default Condition validityDateFilter(ValidityDate validityDate) {
-
-		if (validityDate.isSingleColumnDaterange()) {
-			Column column = validityDate.getColumn().resolve();
-			return field(name(column.getName())).isNotNull();
-		}
-
-		Column startColumn = validityDate.getStartColumn().resolve();
-		Column endColumn = validityDate.getEndColumn().resolve();
-
-		return or(field(name(startColumn.getName())).isNotNull(),
-				  field(name(endColumn.getName())).isNotNull()
-		);
-	}
+	Condition isNotEmptyDateRange(ColumnDateRange columnDateRange);
 
 	ColumnDateRange emptyColumnDateRange();