HCK-12398: add support for iceberg tables

adapter/0.2.18.json

@@ -0,0 +1,56 @@
+/**
+ * Copyright © 2016-2020 by IntegrIT S.A. dba Hackolade.  All rights reserved.
+ *
+ * The copyright to the computer software herein is the property of IntegrIT S.A.
+ * The software may be used and/or copied only with the written permission of
+ * IntegrIT S.A. or in accordance with the terms and conditions stipulated in
+ * the agreement/contract under which the software has been supplied.
+ *
+ * {
+ * 		"add": {
+ * 			"entity": [<names of new property>],
+ * 			"container": [<names of new property>],
+ * 			"model": [<names of new property>],
+ * 			"view": [<names of new property>],
+ *			"field": {
+ *				"<type>": [<names of new property>]
+ *			}
+ * 		},
+ * 		"remove": {
+ * 			"entity": [<names of new property>],
+ * 			"container": [<names of new property>],
+ * 			"model": [<names of new property>],
+ * 			"view": [<names of new property>],
+ *			"field": {
+ *				"<type>": [<names of new property>]
+ *			}
+ * 		},
+ * 		"modify": {
+ *	 		"entity": [
+ *	 			{
+ *					"from": { <properties that identify record> },
+ *					"to": { <properties that need to be changed> }
+ *				}
+ *			],
+ *			"container": [],
+ *			"model": [],
+ *			"view": [],
+ *			"field": []
+ * 		},
+ * }
+ */
+
+{
+	"modify": {
+		"entity": [
+			"assignProperties",
+			{
+				"key": "tableFormat",
+				"exist": false
+			},
+			{
+				"tableFormat": "Standard"
+			}
+		]
+	}
+}

forward_engineering/helpers/awsCliScriptHelpers/glueTableHelper.js

@@ -1,4 +1,5 @@
 const { CLI, CREATE_TABLE } = require('./cliConstants');
+const { TABLE_FORMAT } = require('../../../shared/constants');
 const {
 	getGlueTableColumns,
 	getGluePartitionKeyTableColumns,
@@ -25,9 +26,12 @@ const getGlueTableCreateStatement = (tableSchema, databaseName) => {
 				StoredAsSubDirectories: tableSchema.StoredAsSubDirectories,
 			},
 			Parameters: mapTableParameters(tableSchema),
-			PartitionKeys: getGluePartitionKeyTableColumns(tableSchema.properties),
+			PartitionKeys: handleParameterByTableFormat(tableSchema, () =>
+				getGluePartitionKeyTableColumns(tableSchema.properties),
+			),
 			TableType: tableSchema.externalTable ? 'EXTERNAL_TABLE' : '',
 		},
+		...getTableFormatParameters(tableSchema),
 	};
 
 	const cliStatement = `${CLI} ${CREATE_TABLE} '${JSON.stringify(tableParameters, null, 2)}'`;
@@ -75,6 +79,29 @@ const mapTableParameters = tableSchema => {
 	}
 };
 
+const handleParameterByTableFormat = (tableSchema, getParameter) => {
+	if (tableSchema.tableFormat === TABLE_FORMAT.iceberg) {
+		return;
+	}
+
+	return getParameter();
+};
+
+const getTableFormatParameters = tableSchema => {
+	if (tableSchema.tableFormat === TABLE_FORMAT.iceberg) {
+		return {
+			OpenTableFormatInput: {
+				IcebergInput: {
+					MetadataOperation: 'CREATE',
+					Version: `${tableSchema.icebergVersion}`,
+				},
+			},
+		};
+	}
+
+	return {};
+};
+
 module.exports = {
 	getGlueTableCreateStatement,
 };

forward_engineering/helpers/tableHelper.js

@@ -13,6 +13,7 @@ const {
 } = require('./generalHelper');
 const { getColumnsStatement, getColumnStatement, getColumns } = require('./columnHelper');
 const keyHelper = require('./keyHelper');
+const { TABLE_FORMAT } = require('../../shared/constants');
 
 const getCreateStatement = ({
 	dbName,
@@ -225,11 +226,14 @@ const getStoredAsStatement = tableData => {
 	return `STORED AS ${tableData.storedAsTable.toUpperCase()}`;
 };
 
-const getTableProperties = properties => {
-	if (!properties) {
-		return '';
-	}
-	return `(${properties.map(prop => `"${prop.tablePropKey}"="${prop.tablePropValue}"`).join(', ')})`;
+const getTableProperties = tableData => {
+	const icebergTableProperty = tableData.tableFormat === TABLE_FORMAT.iceberg ? '"table_type"="ICEBERG", ' : '';
+	const tableProperties = (tableData.tableProperties ?? [])
+		.map(prop => `"${prop.tablePropKey}"="${prop.tablePropValue}"`)
+		.join(', ');
+	const tablePropertiesClause = icebergTableProperty + tableProperties;
+
+	return tablePropertiesClause ? `(${tablePropertiesClause})` : '';
 };
 
 const isNumBucketsValid = numBuckets => {
@@ -275,7 +279,7 @@ const getTableStatement = (containerData, entityData, jsonSchema, definitions, f
 		rowFormatStatement: getRowFormat(tableData),
 		storedAsStatement: getStoredAsStatement(tableData),
 		location: tableData.location,
-		tableProperties: getTableProperties(tableData.tableProperties),
+		tableProperties: getTableProperties(tableData),
 		selectStatement: '',
 		isActivated: isTableActivated,
 	});

properties_pane/entity_level/entityLevelConfig.json

@@ -123,10 +123,47 @@ making sure that you maintain a proper JSON format.
 				"propertyKeyword": "code",
 				"propertyType": "text"
 			},
+			{
+				"propertyName": "Table format",
+				"propertyKeyword": "tableFormat",
+				"propertyTooltip": "Select from list of options",
+				"propertyType": "select",
+				"defaultValue": "Standard",
+				"options": ["Standard", "Iceberg"]
+			},
+			{
+				"propertyName": "Version",
+				"propertyKeyword": "icebergVersion",
+				"propertyValidate": true,
+				"propertyType": "numeric",
+				"valueType": "number",
+				"defaultValue": 2,
+				"minValue": 1,
+				"maxValue": 2,
+				"dependency": {
+					"key": "tableFormat",
+					"value": "Iceberg"
+				}
+			},
 			{
 				"propertyName": "External",
 				"propertyKeyword": "externalTable",
-				"propertyType": "checkbox"
+				"propertyType": "checkbox",
+				"dependency": {
+					"key": "tableFormat",
+					"value": "Standard"
+				}
+			},
+			{
+				"propertyName": "External",
+				"propertyKeyword": "externalTable",
+				"propertyType": "checkbox",
+				"defaultValue": true,
+				"disabled": true,
+				"dependency": {
+					"key": "tableFormat",
+					"value": "Iceberg"
+				}
 			},
 			{
 				"propertyName": "Stored as",
@@ -144,14 +181,22 @@ making sure that you maintain a proper JSON format.
 					"JSONfile",
 					"by",
 					"input/output format"
-				]
+				],
+				"dependency": {
+					"key": "tableFormat",
+					"value": "Standard"
+				}
 			},
 			{
 				"propertyName": "Row format",
 				"propertyKeyword": "rowFormat",
 				"propertyTooltip": "Select from list of options",
 				"propertyType": "select",
-				"options": ["", "delimited", "SerDe"]
+				"options": ["", "delimited", "SerDe"],
+				"dependency": {
+					"key": "tableFormat",
+					"value": "Standard"
+				}
 			},
 			{
 				"propertyName": "Fields terminated by",
@@ -268,7 +313,11 @@ making sure that you maintain a proper JSON format.
 			{
 				"propertyName": "Compressed",
 				"propertyKeyword": "compressed",
-				"propertyType": "checkbox"
+				"propertyType": "checkbox",
+				"dependency": {
+					"key": "tableFormat",
+					"value": "Standard"
+				}
 			},
 			{
 				"propertyName": "Parameter paths",
@@ -296,7 +345,11 @@ making sure that you maintain a proper JSON format.
 						"propertyTooltip": "",
 						"propertyType": "text"
 					}
-				]
+				],
+				"dependency": {
+					"key": "storedAsTable",
+					"value": "input/output format"
+				}
 			},
 			{
 				"propertyName": "Stored as sub-directories",
@@ -307,6 +360,11 @@ making sure that you maintain a proper JSON format.
 					"value": "input/output format"
 				}
 			},
+			{
+				"propertyName": "Location",
+				"propertyKeyword": "location",
+				"propertyType": "text"
+			},
 			{
 				"propertyName": "Partition key",
 				"propertyKeyword": "compositePartitionKey",
@@ -317,14 +375,22 @@ making sure that you maintain a proper JSON format.
 			{
 				"propertyName": "Disable No Validate",
 				"propertyKeyword": "disableNoValidate",
-				"propertyType": "checkbox"
+				"propertyType": "checkbox",
+				"dependency": {
+					"key": "tableFormat",
+					"value": "Standard"
+				}
 			},
 			{
 				"propertyName": "Clustering key",
 				"propertyKeyword": "compositeClusteringKey",
 				"propertyType": "primaryKeySetter",
 				"abbr": "CK",
-				"setPrimaryKey": false
+				"setPrimaryKey": false,
+				"dependency": {
+					"key": "tableFormat",
+					"value": "Standard"
+				}
 			},
 			{
 				"propertyName": "Sorted by",
@@ -334,42 +400,61 @@ making sure that you maintain a proper JSON format.
 				"attributeList": [
 					{ "name": "ascending", "abbr": "\u2191" },
 					{ "name": "descending", "abbr": "\u2193" }
-				]
+				],
+				"dependency": {
+					"key": "tableFormat",
+					"value": "Standard"
+				}
 			},
 			{
 				"propertyName": "Number of buckets",
 				"propertyKeyword": "numBuckets",
 				"propertyType": "numeric",
 				"valueType": "number",
-				"allowNegative": false
+				"allowNegative": false,
+				"dependency": {
+					"key": "tableFormat",
+					"value": "Standard"
+				}
 			},
 			{
 				"propertyName": "Skewed by",
 				"propertyKeyword": "skewedby",
 				"propertyType": "fieldList",
-				"template": "orderedList"
+				"template": "orderedList",
+				"dependency": {
+					"key": "tableFormat",
+					"value": "Standard"
+				}
 			},
 			{
 				"propertyName": "Skewed on",
 				"propertyKeyword": "skewedOn",
-				"propertyType": "text"
+				"propertyType": "text",
+				"dependency": {
+					"key": "tableFormat",
+					"value": "Standard"
+				}
 			},
 			{
 				"propertyName": "Skew stored as directories",
 				"propertyKeyword": "skewStoredAsDir",
-				"propertyType": "checkbox"
-			},
-			{
-				"propertyName": "Location",
-				"propertyKeyword": "location",
-				"propertyType": "text"
+				"propertyType": "checkbox",
+				"dependency": {
+					"key": "tableFormat",
+					"value": "Standard"
+				}
 			},
 			{
 				"propertyName": "Classification",
 				"propertyKeyword": "classification",
 				"propertyTooltip": "Select from list of options",
 				"propertyType": "select",
-				"options": ["Avro", "CSV", "JSON", "XML", "Parquet", "ORC"]
+				"options": ["Avro", "CSV", "JSON", "XML", "Parquet", "ORC"],
+				"dependency": {
+					"key": "tableFormat",
+					"value": "Standard"
+				}
 			},
 			{
 				"propertyName": "Table properties",

reverse_engineering/api.js

@@ -145,7 +145,7 @@ module.exports = {
 
 			const tree = parser.statements();
 
-			const hqlToCollectionsGenerator = new hqlToCollectionsVisitor();
+			const hqlToCollectionsGenerator = new hqlToCollectionsVisitor(logger);
 
 			const commands = tree.accept(hqlToCollectionsGenerator);
 			const { result, info, relationships } = commandsService.convertCommandsToReDocs(

reverse_engineering/helpers/tablePropertiesHelper.js

@@ -1,4 +1,5 @@
 const { get } = require('lodash');
+const { TABLE_FORMAT } = require('../../shared/constants');
 
 const mapSortColumns = (items = []) => {
 	return items.map(item => ({
@@ -46,7 +47,7 @@ const getClassification = (parameters = {}) => {
 
 const mapTableProperties = (parameters = {}) => {
 	return Object.entries(parameters).reduce((acc, [key, value]) => {
-		if (key === 'classification') {
+		if (['classification', 'table_type'].includes(key)) {
 			return acc;
 		}
 		return acc.concat({
@@ -99,6 +100,8 @@ const mapTableData = ({ tableData, logger }) => {
 			parameterPaths: mapSerDePaths(tableData.Table.StorageDescriptor?.SerdeInfo),
 			serDeParameters: mapSerDeParameters(tableData.Table.StorageDescriptor?.SerdeInfo?.Parameters),
 			classification: getClassification(tableData.Table.Parameters),
+			tableFormat:
+				tableData.Table.Parameters?.table_type === 'ICEBERG' ? TABLE_FORMAT.iceberg : TABLE_FORMAT.standard,
 		},
 		partitionKeys,
 		columns: mapColumns({ columns: tableData.Table.StorageDescriptor.Columns, logger }),
@@ -107,4 +110,5 @@ const mapTableData = ({ tableData, logger }) => {
 
 module.exports = {
 	mapTableData,
+	mapTableProperties,
 };