Feature/ch support

backend/pom.xml

@@ -373,6 +373,11 @@
             <artifactId>testcontainers-postgresql</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>testcontainers-clickhouse</artifactId>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.testcontainers</groupId>
             <artifactId>testcontainers-solr</artifactId>
@@ -383,6 +388,12 @@
             <artifactId>ngdbc</artifactId>
             <version>2.17.10</version>
         </dependency>
+        <dependency>
+            <groupId>com.clickhouse</groupId>
+            <artifactId>clickhouse-jdbc</artifactId>
+            <version>0.9.8</version>
+            <classifier>all</classifier>
+        </dependency>
         <dependency>
             <groupId>io.prometheus</groupId>
             <artifactId>simpleclient_dropwizard</artifactId>

backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java

@@ -1,8 +1,9 @@
 package com.bakdata.conquery.models.config;
 
 import com.bakdata.conquery.sql.conversion.dialect.DialectBundle;
-import com.bakdata.conquery.sql.conversion.dialect.HanaDialectBundle;
-import com.bakdata.conquery.sql.conversion.dialect.PostgreDialectBundle;
+import com.bakdata.conquery.sql.conversion.dialect.clickhouse.ClickhouseDialectBundle;
+import com.bakdata.conquery.sql.conversion.dialect.hana.HanaDialectBundle;
+import com.bakdata.conquery.sql.conversion.dialect.pg.PostgreDialectBundle;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
 
@@ -15,13 +16,8 @@
 @Getter
 public enum Dialect {
 
-	/**
-	 * Dialect for PostgreSQL database
-	 */
 	POSTGRESQL(new PostgreDialectBundle()),
-	/**
-	 * Dialect for SAP HANA database
-	 */
+	CLICKHOUSE(new ClickhouseDialectBundle()),
 	HANA(new HanaDialectBundle());
 
 	private final DialectBundle dialectBundle;

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java

@@ -17,6 +17,7 @@
 import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory;
 import com.bakdata.conquery.models.query.resultinfo.printers.common.OneToManyMappingPrinter;
 import com.bakdata.conquery.models.types.ResultType;
+import com.bakdata.conquery.sql.conversion.model.select.ClickhouseDistinctSelectConverter;
 import com.bakdata.conquery.sql.conversion.model.select.DistinctSelectConverter;
 import com.bakdata.conquery.sql.conversion.model.select.SelectConverter;
 import com.bakdata.conquery.sql.execution.ResultSetProcessor;
@@ -37,7 +38,8 @@ public Aggregator<?> createAggregator() {
 
 	@Override
 	public SelectConverter<DistinctSelect> createConverter() {
-		return new DistinctSelectConverter();
+		//TODO inject this instead
+		return new ClickhouseDistinctSelectConverter();
 	}
 
 	@Override

backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListStringPrinter.java

@@ -1,6 +1,7 @@
 package com.bakdata.conquery.models.query.resultinfo.printers.common;
 
 import java.util.Collection;
+import java.util.Objects;
 import java.util.StringJoiner;
 
 import com.bakdata.conquery.models.config.LocaleConfig;
@@ -24,7 +25,7 @@ public String apply(@NotNull Collection<T> f) {
 				continue;
 			}
 
-			joiner.add(listFormat.escapeListElement(elementPrinter.apply(obj).toString()));
+			joiner.add(listFormat.escapeListElement(Objects.toString(elementPrinter.apply(obj))));
 		}
 		return joiner.toString();
 	}

backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java

@@ -59,12 +59,12 @@ private Selects getInvertSelects(QueryStep rowNumberStep, SqlIdColumns coalesced
 
 		Field<Date> rangeStart = DSL.coalesce(
 				QualifyingUtil.qualify(validityDate.getEnd(), ROWS_LEFT_TABLE_NAME),
-				functionProvider.toDateField(functionProvider.getMinDateExpression())
+				(functionProvider.getMinDateExpression())
 		).as(DateAggregationCte.RANGE_START);
 
 		Field<Date> rangeEnd = DSL.coalesce(
 				QualifyingUtil.qualify(validityDate.getStart(), ROWS_RIGHT_TABLE_NAME),
-				functionProvider.toDateField(functionProvider.getMaxDateExpression())
+				(functionProvider.getMaxDateExpression())
 		).as(DateAggregationCte.RANGE_END);
 
 		return Selects.builder()

backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/PostgreSqlDateAggregator.java

@@ -84,8 +84,8 @@ public QueryStep invertAggregatedIntervals(QueryStep baseStep, ConversionContext
 		Field<Object> maxDateRange = DSL.function(
 				"daterange",
 				Object.class,
-				this.functionProvider.toDateField(this.functionProvider.getMinDateExpression()),
-				this.functionProvider.toDateField(this.functionProvider.getMaxDateExpression()),
+				(this.functionProvider.getMinDateExpression()),
+				(this.functionProvider.getMaxDateExpression()),
 				DSL.inline("[]")
 		);
 

backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java

@@ -223,11 +223,11 @@ private static SqlFilters dateRestrictionFilter(ConversionContext context, Colum
 
 		List<SqlSelect> dateRestrictionSelects = new ArrayList<>();
 		List<WhereCondition> conditions = new ArrayList<>();
+		SqlFunctionProvider functionProvider = context.getFunctionProvider();
 
-		conditions.add(ConditionUtil.wrap(validityDate.isNotEmpty()));
+		conditions.add(ConditionUtil.wrap(functionProvider.isNotEmptyDateRange(validityDate)));
 
 		if (context.getDateRestrictionRange() != null) {
-			SqlFunctionProvider functionProvider = context.getFunctionProvider();
 			ColumnDateRange dateRestriction = functionProvider.forCDateRange(context.getDateRestrictionRange()).as(SharedAliases.DATE_RESTRICTION.getAlias());
 			conditions.add(ConditionUtil.wrap(functionProvider.dateRestriction(dateRestriction, validityDate)));
 

backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java

@@ -13,7 +13,6 @@
 import com.bakdata.conquery.apiv1.query.concept.filter.CQTable;
 import com.bakdata.conquery.models.common.CDateSet;
 import com.bakdata.conquery.models.common.daterange.CDateRange;
-import com.bakdata.conquery.models.datasets.Column;
 import com.bakdata.conquery.models.datasets.concepts.DaterangeSelectOrFilter;
 import com.bakdata.conquery.models.datasets.concepts.ValidityDate;
 import com.bakdata.conquery.sql.conversion.SharedAliases;
@@ -43,9 +42,9 @@ Collection<? extends OrderField<?>> orderByValidityDates(
 			Function<Field<?>, ? extends SortField<?>> ordering,
 			List<Field<?>> validityDateFields);
 
-	String getMinDateExpression();
+	Field<Date> getMinDateExpression();
 
-	String getMaxDateExpression();
+	Field<Date> getMaxDateExpression();
 
 	<T> Field<T> cast(Field<?> field, DataType<T> type);
 
@@ -180,7 +179,7 @@ default Field<String> stringAggregation(Field<String> stringField, Field<String>
 
 	ColumnDateRange allRangeIf(Condition condition);
 
-	default Field<String> concat(List<Field<String>> fields) {
+	default Field<?> stringArray(List<Field<String>> fields) {
 		String concatenated =
 				fields.stream()
 					  // if a field is null, the whole concatenation would be null - but we just want to skip this field in this case,
@@ -229,20 +228,7 @@ default Field<Date> toDateField(String dateExpression) {
 		return toDate(dateExpression, DEFAULT_DATE_FORMAT);
 	}
 
-	default Condition validityDateFilter(ValidityDate validityDate) {
-
-		if (validityDate.isSingleColumnDaterange()) {
-			Column column = validityDate.getColumn().resolve();
-			return field(name(column.getName())).isNotNull();
-		}
-
-		Column startColumn = validityDate.getStartColumn().resolve();
-		Column endColumn = validityDate.getEndColumn().resolve();
-
-		return or(field(name(startColumn.getName())).isNotNull(),
-				  field(name(endColumn.getName())).isNotNull()
-		);
-	}
+	Condition isNotEmptyDateRange(ColumnDateRange columnDateRange);
 
 	ColumnDateRange emptyColumnDateRange();
 

backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseCDateSetParser.java

@@ -0,0 +1,61 @@
+package com.bakdata.conquery.sql.conversion.dialect.clickhouse;
+
+import java.sql.Date;
+import java.time.LocalDate;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import com.bakdata.conquery.models.common.CDate;
+import com.bakdata.conquery.models.common.daterange.CDateRange;
+import com.bakdata.conquery.sql.execution.SqlCDateSetParser;
+import com.google.common.base.Preconditions;
+import org.apache.commons.lang3.NotImplementedException;
+
+public class ClickhouseCDateSetParser implements SqlCDateSetParser {
+
+	public static final String DATE_SEPARATOR = ",";
+
+	@Override
+	public List<List<Integer>> toEpochDayRangeList(String multiDateRange) {
+		throw new NotImplementedException();
+	}
+
+
+
+	@Override
+	public List<Integer> toEpochDayRange(String daterange) {
+		//TODO probably should also be implemented as tuples, and then we don't even supply a SqlCDateSetParser for CH but solve it via com.bakdata.conquery.sql.execution.ResultSetProcessor
+
+		if (daterange == null) {
+			return Collections.emptyList();
+		}
+
+		String[] dates = daterange.split(DATE_SEPARATOR);
+		Preconditions.checkArgument(dates.length == 2, "Dateranges must have a start and end.");
+
+		// the dateranges have always an included start date marked by a [
+		String startDateExpression = dates[0];
+		int startDate;
+		if (startDateExpression.equals(ClickhouseFunctionProvider.MIN_DATE_VALUE)) {
+			startDate = CDateRange.NEGATIVE_INFINITY;
+		}
+		else {
+			LocalDate dateValue = Date.valueOf(startDateExpression).toLocalDate();
+			startDate = CDate.ofLocalDate(dateValue);
+		}
+
+		String endDateExpression = dates[1];
+		int endDate;
+		if (endDateExpression.equals(ClickhouseFunctionProvider.MAX_DATE_VALUE)) {
+			endDate = CDateRange.POSITIVE_INFINITY;
+		}
+		else {
+			LocalDate dateValue = Date.valueOf(endDateExpression).toLocalDate();
+			endDate = CDate.ofLocalDate(dateValue);
+		}
+
+		return List.of(startDate, endDate);
+	}
+
+}

backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDialectBundle.java

@@ -0,0 +1,101 @@
+package com.bakdata.conquery.sql.conversion.dialect.clickhouse;
+
+import java.util.List;
+
+import com.bakdata.conquery.models.config.ConqueryConfig;
+import com.bakdata.conquery.models.config.Dialect;
+import com.bakdata.conquery.models.events.MajorTypeId;
+import com.bakdata.conquery.models.query.Visitable;
+import com.bakdata.conquery.sql.conversion.NodeConverter;
+import com.bakdata.conquery.sql.conversion.cqelement.aggregation.AnsiSqlDateAggregator;
+import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.AnsiSqlIntervalPacker;
+import com.bakdata.conquery.sql.conversion.dialect.DialectBundle;
+import com.bakdata.conquery.sql.conversion.dialect.IntervalPacker;
+import com.bakdata.conquery.sql.conversion.dialect.SqlDateAggregator;
+import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider;
+import com.bakdata.conquery.sql.conversion.dialect.hana.HanaStratificationFunctions;
+import com.bakdata.conquery.sql.conversion.forms.StratificationFunctions;
+import com.bakdata.conquery.sql.execution.ResultSetProcessor;
+import com.bakdata.conquery.sql.execution.SqlCDateSetParser;
+import lombok.extern.slf4j.Slf4j;
+import org.jooq.DSLContext;
+import org.jooq.Field;
+import org.jooq.SQLDialect;
+
+@Slf4j
+public class ClickhouseDialectBundle implements DialectBundle {
+
+	private final SqlFunctionProvider functionProvider;
+	private final IntervalPacker intervalPacker;
+	private final SqlDateAggregator dateAggregator;
+	private final SqlCDateSetParser dateSetParser;
+
+	public ClickhouseDialectBundle() {
+		this.functionProvider = new ClickhouseFunctionProvider();
+		this.intervalPacker = new AnsiSqlIntervalPacker();
+		this.dateAggregator = new AnsiSqlDateAggregator(this.intervalPacker, this.functionProvider);
+		this.dateSetParser = new ClickhouseCDateSetParser(); // TODO => ArrayCDateSetParser
+	}
+
+	@Override
+	public Dialect getDialect() {
+		return Dialect.CLICKHOUSE;
+	}
+
+	@Override
+	public int getNameMaxLength() {
+		return 64;
+	}
+
+	@Override
+	public String getConnectionTestString() {
+		return "SELECT 1;";
+	}
+
+	@Override
+	public SQLDialect getJooqDialect() {
+		return SQLDialect.CLICKHOUSE;
+	}
+
+	@Override
+	public SqlCDateSetParser getCDateSetParser() {
+		return this.dateSetParser;
+	}
+
+	@Override
+	public List<NodeConverter<? extends Visitable>> getNodeConverters(DSLContext dslContext) {
+		return getDefaultNodeConverters(dslContext);
+	}
+
+	@Override
+	public StratificationFunctions getStratificationFunctions() {
+		return new ClickhouseStratificationFunctions(getFunctionProvider());
+	}
+
+	@Override
+	public boolean isTypeCompatible(Field<?> field, MajorTypeId type) {
+
+		return true; //TODO CLickhouse integration is terrible here. We always receive just Object.
+	}
+
+	@Override
+	public SqlFunctionProvider getFunctionProvider() {
+		return this.functionProvider;
+	}
+
+	@Override
+	public IntervalPacker getIntervalPacker() {
+		return this.intervalPacker;
+	}
+
+	@Override
+	public SqlDateAggregator getDateAggregator() {
+		return this.dateAggregator;
+	}
+
+	@Override
+	public ResultSetProcessor getResultSetProcessor(ConqueryConfig config) {
+		return new ClickhouseResultSetProcessor(config, getCDateSetParser());
+	}
+
+}