diff --git a/adapter/0.2.18.json b/adapter/0.2.18.json new file mode 100644 index 0000000..9c5505e --- /dev/null +++ b/adapter/0.2.18.json @@ -0,0 +1,56 @@ +/** + * Copyright © 2016-2020 by IntegrIT S.A. dba Hackolade. All rights reserved. + * + * The copyright to the computer software herein is the property of IntegrIT S.A. + * The software may be used and/or copied only with the written permission of + * IntegrIT S.A. or in accordance with the terms and conditions stipulated in + * the agreement/contract under which the software has been supplied. + * + * { + * "add": { + * "entity": [], + * "container": [], + * "model": [], + * "view": [], + * "field": { + * "": [] + * } + * }, + * "remove": { + * "entity": [], + * "container": [], + * "model": [], + * "view": [], + * "field": { + * "": [] + * } + * }, + * "modify": { + * "entity": [ + * { + * "from": { }, + * "to": { } + * } + * ], + * "container": [], + * "model": [], + * "view": [], + * "field": [] + * }, + * } + */ + +{ + "modify": { + "entity": [ + "assignProperties", + { + "key": "tableFormat", + "exist": false + }, + { + "tableFormat": "Standard" + } + ] + } +} diff --git a/forward_engineering/helpers/awsCliScriptHelpers/glueTableHelper.js b/forward_engineering/helpers/awsCliScriptHelpers/glueTableHelper.js index 9ecd7c1..25f8cd1 100644 --- a/forward_engineering/helpers/awsCliScriptHelpers/glueTableHelper.js +++ b/forward_engineering/helpers/awsCliScriptHelpers/glueTableHelper.js @@ -1,4 +1,5 @@ const { CLI, CREATE_TABLE } = require('./cliConstants'); +const { TABLE_FORMAT } = require('../../../shared/constants'); const { getGlueTableColumns, getGluePartitionKeyTableColumns, @@ -25,9 +26,12 @@ const getGlueTableCreateStatement = (tableSchema, databaseName) => { StoredAsSubDirectories: tableSchema.StoredAsSubDirectories, }, Parameters: mapTableParameters(tableSchema), - PartitionKeys: getGluePartitionKeyTableColumns(tableSchema.properties), + PartitionKeys: handleParameterByTableFormat(tableSchema, () => + getGluePartitionKeyTableColumns(tableSchema.properties), + ), TableType: tableSchema.externalTable ? 'EXTERNAL_TABLE' : '', }, + ...getTableFormatParameters(tableSchema), }; const cliStatement = `${CLI} ${CREATE_TABLE} '${JSON.stringify(tableParameters, null, 2)}'`; @@ -75,6 +79,29 @@ const mapTableParameters = tableSchema => { } }; +const handleParameterByTableFormat = (tableSchema, getParameter) => { + if (tableSchema.tableFormat === TABLE_FORMAT.iceberg) { + return; + } + + return getParameter(); +}; + +const getTableFormatParameters = tableSchema => { + if (tableSchema.tableFormat === TABLE_FORMAT.iceberg) { + return { + OpenTableFormatInput: { + IcebergInput: { + MetadataOperation: 'CREATE', + Version: `${tableSchema.icebergVersion}`, + }, + }, + }; + } + + return {}; +}; + module.exports = { getGlueTableCreateStatement, }; diff --git a/forward_engineering/helpers/tableHelper.js b/forward_engineering/helpers/tableHelper.js index 2d2edbd..04f6abb 100644 --- a/forward_engineering/helpers/tableHelper.js +++ b/forward_engineering/helpers/tableHelper.js @@ -13,6 +13,7 @@ const { } = require('./generalHelper'); const { getColumnsStatement, getColumnStatement, getColumns } = require('./columnHelper'); const keyHelper = require('./keyHelper'); +const { TABLE_FORMAT } = require('../../shared/constants'); const getCreateStatement = ({ dbName, @@ -225,11 +226,14 @@ const getStoredAsStatement = tableData => { return `STORED AS ${tableData.storedAsTable.toUpperCase()}`; }; -const getTableProperties = properties => { - if (!properties) { - return ''; - } - return `(${properties.map(prop => `"${prop.tablePropKey}"="${prop.tablePropValue}"`).join(', ')})`; +const getTableProperties = tableData => { + const icebergTableProperty = tableData.tableFormat === TABLE_FORMAT.iceberg ? '"table_type"="ICEBERG", ' : ''; + const tableProperties = (tableData.tableProperties ?? []) + .map(prop => `"${prop.tablePropKey}"="${prop.tablePropValue}"`) + .join(', '); + const tablePropertiesClause = icebergTableProperty + tableProperties; + + return tablePropertiesClause ? `(${tablePropertiesClause})` : ''; }; const isNumBucketsValid = numBuckets => { @@ -275,7 +279,7 @@ const getTableStatement = (containerData, entityData, jsonSchema, definitions, f rowFormatStatement: getRowFormat(tableData), storedAsStatement: getStoredAsStatement(tableData), location: tableData.location, - tableProperties: getTableProperties(tableData.tableProperties), + tableProperties: getTableProperties(tableData), selectStatement: '', isActivated: isTableActivated, }); diff --git a/properties_pane/entity_level/entityLevelConfig.json b/properties_pane/entity_level/entityLevelConfig.json index 18a01cf..08014b5 100644 --- a/properties_pane/entity_level/entityLevelConfig.json +++ b/properties_pane/entity_level/entityLevelConfig.json @@ -123,10 +123,47 @@ making sure that you maintain a proper JSON format. "propertyKeyword": "code", "propertyType": "text" }, + { + "propertyName": "Table format", + "propertyKeyword": "tableFormat", + "propertyTooltip": "Select from list of options", + "propertyType": "select", + "defaultValue": "Standard", + "options": ["Standard", "Iceberg"] + }, + { + "propertyName": "Version", + "propertyKeyword": "icebergVersion", + "propertyValidate": true, + "propertyType": "numeric", + "valueType": "number", + "defaultValue": 2, + "minValue": 1, + "maxValue": 2, + "dependency": { + "key": "tableFormat", + "value": "Iceberg" + } + }, { "propertyName": "External", "propertyKeyword": "externalTable", - "propertyType": "checkbox" + "propertyType": "checkbox", + "dependency": { + "key": "tableFormat", + "value": "Standard" + } + }, + { + "propertyName": "External", + "propertyKeyword": "externalTable", + "propertyType": "checkbox", + "defaultValue": true, + "disabled": true, + "dependency": { + "key": "tableFormat", + "value": "Iceberg" + } }, { "propertyName": "Stored as", @@ -144,14 +181,22 @@ making sure that you maintain a proper JSON format. "JSONfile", "by", "input/output format" - ] + ], + "dependency": { + "key": "tableFormat", + "value": "Standard" + } }, { "propertyName": "Row format", "propertyKeyword": "rowFormat", "propertyTooltip": "Select from list of options", "propertyType": "select", - "options": ["", "delimited", "SerDe"] + "options": ["", "delimited", "SerDe"], + "dependency": { + "key": "tableFormat", + "value": "Standard" + } }, { "propertyName": "Fields terminated by", @@ -268,7 +313,11 @@ making sure that you maintain a proper JSON format. { "propertyName": "Compressed", "propertyKeyword": "compressed", - "propertyType": "checkbox" + "propertyType": "checkbox", + "dependency": { + "key": "tableFormat", + "value": "Standard" + } }, { "propertyName": "Parameter paths", @@ -296,7 +345,11 @@ making sure that you maintain a proper JSON format. "propertyTooltip": "", "propertyType": "text" } - ] + ], + "dependency": { + "key": "storedAsTable", + "value": "input/output format" + } }, { "propertyName": "Stored as sub-directories", @@ -307,6 +360,11 @@ making sure that you maintain a proper JSON format. "value": "input/output format" } }, + { + "propertyName": "Location", + "propertyKeyword": "location", + "propertyType": "text" + }, { "propertyName": "Partition key", "propertyKeyword": "compositePartitionKey", @@ -317,14 +375,22 @@ making sure that you maintain a proper JSON format. { "propertyName": "Disable No Validate", "propertyKeyword": "disableNoValidate", - "propertyType": "checkbox" + "propertyType": "checkbox", + "dependency": { + "key": "tableFormat", + "value": "Standard" + } }, { "propertyName": "Clustering key", "propertyKeyword": "compositeClusteringKey", "propertyType": "primaryKeySetter", "abbr": "CK", - "setPrimaryKey": false + "setPrimaryKey": false, + "dependency": { + "key": "tableFormat", + "value": "Standard" + } }, { "propertyName": "Sorted by", @@ -334,42 +400,61 @@ making sure that you maintain a proper JSON format. "attributeList": [ { "name": "ascending", "abbr": "\u2191" }, { "name": "descending", "abbr": "\u2193" } - ] + ], + "dependency": { + "key": "tableFormat", + "value": "Standard" + } }, { "propertyName": "Number of buckets", "propertyKeyword": "numBuckets", "propertyType": "numeric", "valueType": "number", - "allowNegative": false + "allowNegative": false, + "dependency": { + "key": "tableFormat", + "value": "Standard" + } }, { "propertyName": "Skewed by", "propertyKeyword": "skewedby", "propertyType": "fieldList", - "template": "orderedList" + "template": "orderedList", + "dependency": { + "key": "tableFormat", + "value": "Standard" + } }, { "propertyName": "Skewed on", "propertyKeyword": "skewedOn", - "propertyType": "text" + "propertyType": "text", + "dependency": { + "key": "tableFormat", + "value": "Standard" + } }, { "propertyName": "Skew stored as directories", "propertyKeyword": "skewStoredAsDir", - "propertyType": "checkbox" - }, - { - "propertyName": "Location", - "propertyKeyword": "location", - "propertyType": "text" + "propertyType": "checkbox", + "dependency": { + "key": "tableFormat", + "value": "Standard" + } }, { "propertyName": "Classification", "propertyKeyword": "classification", "propertyTooltip": "Select from list of options", "propertyType": "select", - "options": ["Avro", "CSV", "JSON", "XML", "Parquet", "ORC"] + "options": ["Avro", "CSV", "JSON", "XML", "Parquet", "ORC"], + "dependency": { + "key": "tableFormat", + "value": "Standard" + } }, { "propertyName": "Table properties", diff --git a/reverse_engineering/api.js b/reverse_engineering/api.js index e404254..0c4c473 100644 --- a/reverse_engineering/api.js +++ b/reverse_engineering/api.js @@ -145,7 +145,7 @@ module.exports = { const tree = parser.statements(); - const hqlToCollectionsGenerator = new hqlToCollectionsVisitor(); + const hqlToCollectionsGenerator = new hqlToCollectionsVisitor(logger); const commands = tree.accept(hqlToCollectionsGenerator); const { result, info, relationships } = commandsService.convertCommandsToReDocs( diff --git a/reverse_engineering/helpers/tablePropertiesHelper.js b/reverse_engineering/helpers/tablePropertiesHelper.js index d111dcf..0af4c72 100644 --- a/reverse_engineering/helpers/tablePropertiesHelper.js +++ b/reverse_engineering/helpers/tablePropertiesHelper.js @@ -1,4 +1,5 @@ const { get } = require('lodash'); +const { TABLE_FORMAT } = require('../../shared/constants'); const mapSortColumns = (items = []) => { return items.map(item => ({ @@ -46,7 +47,7 @@ const getClassification = (parameters = {}) => { const mapTableProperties = (parameters = {}) => { return Object.entries(parameters).reduce((acc, [key, value]) => { - if (key === 'classification') { + if (['classification', 'table_type'].includes(key)) { return acc; } return acc.concat({ @@ -99,6 +100,8 @@ const mapTableData = ({ tableData, logger }) => { parameterPaths: mapSerDePaths(tableData.Table.StorageDescriptor?.SerdeInfo), serDeParameters: mapSerDeParameters(tableData.Table.StorageDescriptor?.SerdeInfo?.Parameters), classification: getClassification(tableData.Table.Parameters), + tableFormat: + tableData.Table.Parameters?.table_type === 'ICEBERG' ? TABLE_FORMAT.iceberg : TABLE_FORMAT.standard, }, partitionKeys, columns: mapColumns({ columns: tableData.Table.StorageDescriptor.Columns, logger }), @@ -107,4 +110,5 @@ const mapTableData = ({ tableData, logger }) => { module.exports = { mapTableData, + mapTableProperties, }; diff --git a/reverse_engineering/hqlToCollectionsVisitor.js b/reverse_engineering/hqlToCollectionsVisitor.js index 9f6a31d..43d5054 100644 --- a/reverse_engineering/hqlToCollectionsVisitor.js +++ b/reverse_engineering/hqlToCollectionsVisitor.js @@ -30,6 +30,8 @@ const { } = require('./commandsService'); const schemaHelper = require('./schemaHelper'); +const { mapTableProperties } = require('./helpers/tablePropertiesHelper'); +const { TABLE_FORMAT } = require('../shared/constants'); const ALLOWED_COMMANDS = [ HiveParser.RULE_createTableStatement, @@ -49,6 +51,11 @@ const ALLOWED_COMMANDS = [ ]; class Visitor extends HiveParserVisitor { + constructor(logger) { + super(); + this.logger = logger; + } + visitStatement(ctx) { const execStatement = ctx.execStatement(); if (execStatement) { @@ -77,7 +84,8 @@ class Visitor extends HiveParserVisitor { const tableRowFormat = this.visitWhenExists(ctx, 'tableRowFormat', {}); const description = this.visitWhenExists(ctx, 'tableComment'); const location = this.visitWhenExists(ctx, 'tableLocation'); - const tableProperties = this.visitWhenExists(ctx, 'tablePropertiesPrefixed'); + const tablePropertiesPrefixed = this.visitWhenExists(ctx, 'tablePropertiesPrefixed'); + const { tableFormat, tableProperties } = getTableProperties(tablePropertiesPrefixed, this.logger); const temporaryTable = Boolean(ctx.KW_TEMPORARY()); const externalTable = Boolean(ctx.KW_EXTERNAL()); const storedAsTable = this.visitWhenExists(ctx, 'tableFileFormat', {}); @@ -116,6 +124,7 @@ class Visitor extends HiveParserVisitor { skewedOn, skewStoredAsDir, location, + tableFormat, tableProperties, ...storedAsTable, ...tableRowFormat, @@ -1443,4 +1452,26 @@ const getMappingType = ctx => { } }; +const getTableProperties = (tablePropertiesPrefixed, logger) => { + try { + const properties = tablePropertiesPrefixed.replace(/^\(/, '{').replace(/\)$/, '}'); + const parsedProperties = JSON.parse(properties); + const { table_type, ...restTableProperties } = parsedProperties; + const tableFormat = table_type === 'ICEBERG' ? TABLE_FORMAT.iceberg : TABLE_FORMAT.standard; + const tableProperties = mapTableProperties(restTableProperties); + + return { + tableFormat, + tableProperties, + }; + } catch (error) { + logger.log('error', error, 'getTableProperties'); + + return { + tableFormat: TABLE_FORMAT.standard, + tableProperties: [], + }; + } +}; + module.exports = Visitor; diff --git a/shared/constants.js b/shared/constants.js new file mode 100644 index 0000000..08cc50e --- /dev/null +++ b/shared/constants.js @@ -0,0 +1,8 @@ +const TABLE_FORMAT = { + iceberg: 'Iceberg', + standard: 'Standard', +}; + +module.exports = { + TABLE_FORMAT, +};