From 9461e01175d8e3396a0615865fe5714dabe3b679 Mon Sep 17 00:00:00 2001 From: Pascal Desmarets Date: Sun, 17 Aug 2025 11:19:53 +0200 Subject: [PATCH 01/10] Added Iceberg table properties in entity config --- .../entity_level/entityLevelConfig.json | 120 +++++++++++++++--- 1 file changed, 102 insertions(+), 18 deletions(-) diff --git a/properties_pane/entity_level/entityLevelConfig.json b/properties_pane/entity_level/entityLevelConfig.json index 18a01cf..526a675 100644 --- a/properties_pane/entity_level/entityLevelConfig.json +++ b/properties_pane/entity_level/entityLevelConfig.json @@ -123,10 +123,46 @@ making sure that you maintain a proper JSON format. "propertyKeyword": "code", "propertyType": "text" }, + { + "propertyName": "Table format", + "propertyKeyword": "tableFormat", + "propertyTooltip": "Select from list of options", + "propertyType": "select", + "defaultValue": "Standard", + "options": ["Standard", "Iceberg"] + }, + { + "propertyName": "Version", + "propertyKeyword": "icebergVersion", + "propertyValidate": true, + "propertyType": "numeric", + "valueType": "number", + "defaultValue": 2, + "minValue": 1, + "maxValue": 2, + "dependency": { + "key": "tableFormat", + "value": "Iceberg" + } + }, { "propertyName": "External", "propertyKeyword": "externalTable", - "propertyType": "checkbox" + "propertyType": "checkbox", + "dependency": { + "key": "tableFormat", + "value": "Standard" + } + }, + { + "propertyName": "External", + "propertyKeyword": "externalTable", + "propertyType": "checkbox", + "defaultValue": true, + "dependency": { + "key": "tableFormat", + "value": "Iceberg" + } }, { "propertyName": "Stored as", @@ -144,14 +180,22 @@ making sure that you maintain a proper JSON format. "JSONfile", "by", "input/output format" - ] + ], + "dependency": { + "key": "tableFormat", + "value": "Standard" + } }, { "propertyName": "Row format", "propertyKeyword": "rowFormat", "propertyTooltip": "Select from list of options", "propertyType": "select", - "options": ["", "delimited", "SerDe"] + "options": ["", "delimited", "SerDe"], + "dependency": { + "key": "tableFormat", + "value": "Standard" + } }, { "propertyName": "Fields terminated by", @@ -268,7 +312,11 @@ making sure that you maintain a proper JSON format. { "propertyName": "Compressed", "propertyKeyword": "compressed", - "propertyType": "checkbox" + "propertyType": "checkbox", + "dependency": { + "key": "tableFormat", + "value": "Standard" + } }, { "propertyName": "Parameter paths", @@ -296,7 +344,11 @@ making sure that you maintain a proper JSON format. "propertyTooltip": "", "propertyType": "text" } - ] + ], + "dependency": { + "key": "storedAsTable", + "value": "input/output format" + } }, { "propertyName": "Stored as sub-directories", @@ -307,6 +359,11 @@ making sure that you maintain a proper JSON format. "value": "input/output format" } }, + { + "propertyName": "Location", + "propertyKeyword": "location", + "propertyType": "text" + }, { "propertyName": "Partition key", "propertyKeyword": "compositePartitionKey", @@ -317,14 +374,22 @@ making sure that you maintain a proper JSON format. { "propertyName": "Disable No Validate", "propertyKeyword": "disableNoValidate", - "propertyType": "checkbox" + "propertyType": "checkbox", + "dependency": { + "key": "tableFormat", + "value": "Standard" + } }, { "propertyName": "Clustering key", "propertyKeyword": "compositeClusteringKey", "propertyType": "primaryKeySetter", "abbr": "CK", - "setPrimaryKey": false + "setPrimaryKey": false, + "dependency": { + "key": "tableFormat", + "value": "Standard" + } }, { "propertyName": "Sorted by", @@ -334,42 +399,61 @@ making sure that you maintain a proper JSON format. "attributeList": [ { "name": "ascending", "abbr": "\u2191" }, { "name": "descending", "abbr": "\u2193" } - ] + ], + "dependency": { + "key": "tableFormat", + "value": "Standard" + } }, { "propertyName": "Number of buckets", "propertyKeyword": "numBuckets", "propertyType": "numeric", "valueType": "number", - "allowNegative": false + "allowNegative": false, + "dependency": { + "key": "tableFormat", + "value": "Standard" + } }, { "propertyName": "Skewed by", "propertyKeyword": "skewedby", "propertyType": "fieldList", - "template": "orderedList" + "template": "orderedList", + "dependency": { + "key": "tableFormat", + "value": "Standard" + } }, { "propertyName": "Skewed on", "propertyKeyword": "skewedOn", - "propertyType": "text" + "propertyType": "text", + "dependency": { + "key": "tableFormat", + "value": "Standard" + } }, { "propertyName": "Skew stored as directories", "propertyKeyword": "skewStoredAsDir", - "propertyType": "checkbox" - }, - { - "propertyName": "Location", - "propertyKeyword": "location", - "propertyType": "text" + "propertyType": "checkbox", + "dependency": { + "key": "tableFormat", + "value": "Standard" + } }, { "propertyName": "Classification", "propertyKeyword": "classification", "propertyTooltip": "Select from list of options", "propertyType": "select", - "options": ["Avro", "CSV", "JSON", "XML", "Parquet", "ORC"] + "options": ["Avro", "CSV", "JSON", "XML", "Parquet", "ORC"], + "dependency": { + "key": "tableFormat", + "value": "Standard" + } }, { "propertyName": "Table properties", From df43dfc301f8abafecccfe905389e173b0129f80 Mon Sep 17 00:00:00 2001 From: Serhii Filonenko Date: Thu, 21 Aug 2025 07:41:31 +0300 Subject: [PATCH 02/10] HCK-12398: disable external table property for iceberg table --- properties_pane/entity_level/entityLevelConfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/properties_pane/entity_level/entityLevelConfig.json b/properties_pane/entity_level/entityLevelConfig.json index 526a675..08014b5 100644 --- a/properties_pane/entity_level/entityLevelConfig.json +++ b/properties_pane/entity_level/entityLevelConfig.json @@ -159,6 +159,7 @@ making sure that you maintain a proper JSON format. "propertyKeyword": "externalTable", "propertyType": "checkbox", "defaultValue": true, + "disabled": true, "dependency": { "key": "tableFormat", "value": "Iceberg" From ce2e307a47e7ff624868b770af653b9907c21257 Mon Sep 17 00:00:00 2001 From: Serhii Filonenko Date: Thu, 21 Aug 2025 07:41:49 +0300 Subject: [PATCH 03/10] HCK-12398: add an adapter --- adapter/0.2.18.json | 56 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 adapter/0.2.18.json diff --git a/adapter/0.2.18.json b/adapter/0.2.18.json new file mode 100644 index 0000000..9c5505e --- /dev/null +++ b/adapter/0.2.18.json @@ -0,0 +1,56 @@ +/** + * Copyright © 2016-2020 by IntegrIT S.A. dba Hackolade. All rights reserved. + * + * The copyright to the computer software herein is the property of IntegrIT S.A. + * The software may be used and/or copied only with the written permission of + * IntegrIT S.A. or in accordance with the terms and conditions stipulated in + * the agreement/contract under which the software has been supplied. + * + * { + * "add": { + * "entity": [], + * "container": [], + * "model": [], + * "view": [], + * "field": { + * "": [] + * } + * }, + * "remove": { + * "entity": [], + * "container": [], + * "model": [], + * "view": [], + * "field": { + * "": [] + * } + * }, + * "modify": { + * "entity": [ + * { + * "from": { }, + * "to": { } + * } + * ], + * "container": [], + * "model": [], + * "view": [], + * "field": [] + * }, + * } + */ + +{ + "modify": { + "entity": [ + "assignProperties", + { + "key": "tableFormat", + "exist": false + }, + { + "tableFormat": "Standard" + } + ] + } +} From 6624f4fcad9387214f6611e915135e1a25d127e8 Mon Sep 17 00:00:00 2001 From: Serhii Filonenko Date: Thu, 21 Aug 2025 07:42:17 +0300 Subject: [PATCH 04/10] HCK-12398: expand aws create table script --- .../awsCliScriptHelpers/glueTableHelper.js | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/forward_engineering/helpers/awsCliScriptHelpers/glueTableHelper.js b/forward_engineering/helpers/awsCliScriptHelpers/glueTableHelper.js index 9ecd7c1..1cb6536 100644 --- a/forward_engineering/helpers/awsCliScriptHelpers/glueTableHelper.js +++ b/forward_engineering/helpers/awsCliScriptHelpers/glueTableHelper.js @@ -28,6 +28,7 @@ const getGlueTableCreateStatement = (tableSchema, databaseName) => { PartitionKeys: getGluePartitionKeyTableColumns(tableSchema.properties), TableType: tableSchema.externalTable ? 'EXTERNAL_TABLE' : '', }, + ...getTableFormatParameters(tableSchema), }; const cliStatement = `${CLI} ${CREATE_TABLE} '${JSON.stringify(tableParameters, null, 2)}'`; @@ -75,6 +76,21 @@ const mapTableParameters = tableSchema => { } }; +const getTableFormatParameters = tableSchema => { + if (tableSchema.tableFormat === 'Iceberg') { + return { + OpenTableFormatInput: { + IcebergInput: { + MetadataOperation: 'CREATE', + Version: tableSchema.icebergVersion, + }, + }, + }; + } + + return {}; +}; + module.exports = { getGlueTableCreateStatement, }; From b5b810f22be2f047b6c420bb3dc2ca5114187204 Mon Sep 17 00:00:00 2001 From: Serhii Filonenko Date: Thu, 21 Aug 2025 07:58:15 +0300 Subject: [PATCH 05/10] HCK-12398: expand hive create table script --- forward_engineering/helpers/tableHelper.js | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/forward_engineering/helpers/tableHelper.js b/forward_engineering/helpers/tableHelper.js index 2d2edbd..5910aa0 100644 --- a/forward_engineering/helpers/tableHelper.js +++ b/forward_engineering/helpers/tableHelper.js @@ -225,11 +225,14 @@ const getStoredAsStatement = tableData => { return `STORED AS ${tableData.storedAsTable.toUpperCase()}`; }; -const getTableProperties = properties => { - if (!properties) { - return ''; - } - return `(${properties.map(prop => `"${prop.tablePropKey}"="${prop.tablePropValue}"`).join(', ')})`; +const getTableProperties = tableData => { + const icebergTableProperty = tableData.tableFormat === 'Iceberg' ? '"table_type"="ICEBERG", ' : ''; + const tableProperties = (tableData.tableProperties ?? []) + .map(prop => `"${prop.tablePropKey}"="${prop.tablePropValue}"`) + .join(', '); + const tablePropertiesClause = icebergTableProperty + tableProperties; + + return tablePropertiesClause ? `(${tablePropertiesClause})` : ''; }; const isNumBucketsValid = numBuckets => { @@ -275,7 +278,7 @@ const getTableStatement = (containerData, entityData, jsonSchema, definitions, f rowFormatStatement: getRowFormat(tableData), storedAsStatement: getStoredAsStatement(tableData), location: tableData.location, - tableProperties: getTableProperties(tableData.tableProperties), + tableProperties: getTableProperties(tableData), selectStatement: '', isActivated: isTableActivated, }); From c4bc6f849f551357bc053d7cf1a471e541c2d426 Mon Sep 17 00:00:00 2001 From: Serhii Filonenko Date: Thu, 21 Aug 2025 08:30:27 +0300 Subject: [PATCH 06/10] HCK-12398: adjust RE from hql file --- .../helpers/tablePropertiesHelper.js | 4 ++- .../hqlToCollectionsVisitor.js | 25 ++++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/reverse_engineering/helpers/tablePropertiesHelper.js b/reverse_engineering/helpers/tablePropertiesHelper.js index d111dcf..b5d4cf5 100644 --- a/reverse_engineering/helpers/tablePropertiesHelper.js +++ b/reverse_engineering/helpers/tablePropertiesHelper.js @@ -46,7 +46,7 @@ const getClassification = (parameters = {}) => { const mapTableProperties = (parameters = {}) => { return Object.entries(parameters).reduce((acc, [key, value]) => { - if (key === 'classification') { + if (['classification', 'table_type'].includes(key)) { return acc; } return acc.concat({ @@ -99,6 +99,7 @@ const mapTableData = ({ tableData, logger }) => { parameterPaths: mapSerDePaths(tableData.Table.StorageDescriptor?.SerdeInfo), serDeParameters: mapSerDeParameters(tableData.Table.StorageDescriptor?.SerdeInfo?.Parameters), classification: getClassification(tableData.Table.Parameters), + tableFormat: tableData.Table.Parameters?.table_type === 'ICEBERG' ? 'Iceberg' : 'Standard', }, partitionKeys, columns: mapColumns({ columns: tableData.Table.StorageDescriptor.Columns, logger }), @@ -107,4 +108,5 @@ const mapTableData = ({ tableData, logger }) => { module.exports = { mapTableData, + mapTableProperties, }; diff --git a/reverse_engineering/hqlToCollectionsVisitor.js b/reverse_engineering/hqlToCollectionsVisitor.js index 9f6a31d..7dbb4f9 100644 --- a/reverse_engineering/hqlToCollectionsVisitor.js +++ b/reverse_engineering/hqlToCollectionsVisitor.js @@ -30,6 +30,7 @@ const { } = require('./commandsService'); const schemaHelper = require('./schemaHelper'); +const { mapTableProperties } = require('./helpers/tablePropertiesHelper'); const ALLOWED_COMMANDS = [ HiveParser.RULE_createTableStatement, @@ -77,7 +78,8 @@ class Visitor extends HiveParserVisitor { const tableRowFormat = this.visitWhenExists(ctx, 'tableRowFormat', {}); const description = this.visitWhenExists(ctx, 'tableComment'); const location = this.visitWhenExists(ctx, 'tableLocation'); - const tableProperties = this.visitWhenExists(ctx, 'tablePropertiesPrefixed'); + const tablePropertiesPrefixed = this.visitWhenExists(ctx, 'tablePropertiesPrefixed'); + const { tableFormat, tableProperties } = getTableProperties(tablePropertiesPrefixed); const temporaryTable = Boolean(ctx.KW_TEMPORARY()); const externalTable = Boolean(ctx.KW_EXTERNAL()); const storedAsTable = this.visitWhenExists(ctx, 'tableFileFormat', {}); @@ -116,6 +118,7 @@ class Visitor extends HiveParserVisitor { skewedOn, skewStoredAsDir, location, + tableFormat, tableProperties, ...storedAsTable, ...tableRowFormat, @@ -1443,4 +1446,24 @@ const getMappingType = ctx => { } }; +const getTableProperties = tablePropertiesPrefixed => { + try { + const properties = tablePropertiesPrefixed.replace(/^\(/, '{').replace(/\)$/, '}').replace(/=/g, ':'); + const parsedProperties = JSON.parse(properties); + const { table_type, ...restTableProperties } = parsedProperties; + const tableFormat = table_type === 'ICEBERG' ? 'Iceberg' : 'Standard'; + const tableProperties = mapTableProperties(restTableProperties); + + return { + tableFormat, + tableProperties, + }; + } catch (err) { + return { + tableFormat: 'Standard', + tableProperties: [], + }; + } +}; + module.exports = Visitor; From ff86c57f8bf0eed2cec2244b93ac416dd8a3de77 Mon Sep 17 00:00:00 2001 From: Serhii Filonenko Date: Thu, 21 Aug 2025 08:37:32 +0300 Subject: [PATCH 07/10] HCK-12398: add constant --- .../helpers/awsCliScriptHelpers/glueTableHelper.js | 3 ++- forward_engineering/helpers/tableHelper.js | 2 +- reverse_engineering/helpers/tablePropertiesHelper.js | 4 +++- reverse_engineering/hqlToCollectionsVisitor.js | 5 +++-- shared/constants.js | 8 ++++++++ 5 files changed, 17 insertions(+), 5 deletions(-) create mode 100644 shared/constants.js diff --git a/forward_engineering/helpers/awsCliScriptHelpers/glueTableHelper.js b/forward_engineering/helpers/awsCliScriptHelpers/glueTableHelper.js index 1cb6536..7c9580e 100644 --- a/forward_engineering/helpers/awsCliScriptHelpers/glueTableHelper.js +++ b/forward_engineering/helpers/awsCliScriptHelpers/glueTableHelper.js @@ -1,4 +1,5 @@ const { CLI, CREATE_TABLE } = require('./cliConstants'); +const { TABLE_FORMAT } = require('../../../shared/constants'); const { getGlueTableColumns, getGluePartitionKeyTableColumns, @@ -77,7 +78,7 @@ const mapTableParameters = tableSchema => { }; const getTableFormatParameters = tableSchema => { - if (tableSchema.tableFormat === 'Iceberg') { + if (tableSchema.tableFormat === TABLE_FORMAT.iceberg) { return { OpenTableFormatInput: { IcebergInput: { diff --git a/forward_engineering/helpers/tableHelper.js b/forward_engineering/helpers/tableHelper.js index 5910aa0..7ea8cb9 100644 --- a/forward_engineering/helpers/tableHelper.js +++ b/forward_engineering/helpers/tableHelper.js @@ -226,7 +226,7 @@ const getStoredAsStatement = tableData => { }; const getTableProperties = tableData => { - const icebergTableProperty = tableData.tableFormat === 'Iceberg' ? '"table_type"="ICEBERG", ' : ''; + const icebergTableProperty = tableData.tableFormat === TABLE_F ? '"table_type"="ICEBERG", ' : ''; const tableProperties = (tableData.tableProperties ?? []) .map(prop => `"${prop.tablePropKey}"="${prop.tablePropValue}"`) .join(', '); diff --git a/reverse_engineering/helpers/tablePropertiesHelper.js b/reverse_engineering/helpers/tablePropertiesHelper.js index b5d4cf5..0af4c72 100644 --- a/reverse_engineering/helpers/tablePropertiesHelper.js +++ b/reverse_engineering/helpers/tablePropertiesHelper.js @@ -1,4 +1,5 @@ const { get } = require('lodash'); +const { TABLE_FORMAT } = require('../../shared/constants'); const mapSortColumns = (items = []) => { return items.map(item => ({ @@ -99,7 +100,8 @@ const mapTableData = ({ tableData, logger }) => { parameterPaths: mapSerDePaths(tableData.Table.StorageDescriptor?.SerdeInfo), serDeParameters: mapSerDeParameters(tableData.Table.StorageDescriptor?.SerdeInfo?.Parameters), classification: getClassification(tableData.Table.Parameters), - tableFormat: tableData.Table.Parameters?.table_type === 'ICEBERG' ? 'Iceberg' : 'Standard', + tableFormat: + tableData.Table.Parameters?.table_type === 'ICEBERG' ? TABLE_FORMAT.iceberg : TABLE_FORMAT.standard, }, partitionKeys, columns: mapColumns({ columns: tableData.Table.StorageDescriptor.Columns, logger }), diff --git a/reverse_engineering/hqlToCollectionsVisitor.js b/reverse_engineering/hqlToCollectionsVisitor.js index 7dbb4f9..70749ea 100644 --- a/reverse_engineering/hqlToCollectionsVisitor.js +++ b/reverse_engineering/hqlToCollectionsVisitor.js @@ -31,6 +31,7 @@ const { const schemaHelper = require('./schemaHelper'); const { mapTableProperties } = require('./helpers/tablePropertiesHelper'); +const { TABLE_FORMAT } = require('../shared/constants'); const ALLOWED_COMMANDS = [ HiveParser.RULE_createTableStatement, @@ -1451,7 +1452,7 @@ const getTableProperties = tablePropertiesPrefixed => { const properties = tablePropertiesPrefixed.replace(/^\(/, '{').replace(/\)$/, '}').replace(/=/g, ':'); const parsedProperties = JSON.parse(properties); const { table_type, ...restTableProperties } = parsedProperties; - const tableFormat = table_type === 'ICEBERG' ? 'Iceberg' : 'Standard'; + const tableFormat = table_type === 'ICEBERG' ? TABLE_FORMAT.iceberg : TABLE_FORMAT.standard; const tableProperties = mapTableProperties(restTableProperties); return { @@ -1460,7 +1461,7 @@ const getTableProperties = tablePropertiesPrefixed => { }; } catch (err) { return { - tableFormat: 'Standard', + tableFormat: TABLE_FORMAT.standard, tableProperties: [], }; } diff --git a/shared/constants.js b/shared/constants.js new file mode 100644 index 0000000..08cc50e --- /dev/null +++ b/shared/constants.js @@ -0,0 +1,8 @@ +const TABLE_FORMAT = { + iceberg: 'Iceberg', + standard: 'Standard', +}; + +module.exports = { + TABLE_FORMAT, +}; From f2ed1afe304f51e1abfbcf285eab561e02709e2b Mon Sep 17 00:00:00 2001 From: Serhii Filonenko Date: Thu, 21 Aug 2025 09:15:52 +0300 Subject: [PATCH 08/10] HCK-12398: fix typo --- forward_engineering/helpers/tableHelper.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/forward_engineering/helpers/tableHelper.js b/forward_engineering/helpers/tableHelper.js index 7ea8cb9..04f6abb 100644 --- a/forward_engineering/helpers/tableHelper.js +++ b/forward_engineering/helpers/tableHelper.js @@ -13,6 +13,7 @@ const { } = require('./generalHelper'); const { getColumnsStatement, getColumnStatement, getColumns } = require('./columnHelper'); const keyHelper = require('./keyHelper'); +const { TABLE_FORMAT } = require('../../shared/constants'); const getCreateStatement = ({ dbName, @@ -226,7 +227,7 @@ const getStoredAsStatement = tableData => { }; const getTableProperties = tableData => { - const icebergTableProperty = tableData.tableFormat === TABLE_F ? '"table_type"="ICEBERG", ' : ''; + const icebergTableProperty = tableData.tableFormat === TABLE_FORMAT.iceberg ? '"table_type"="ICEBERG", ' : ''; const tableProperties = (tableData.tableProperties ?? []) .map(prop => `"${prop.tablePropKey}"="${prop.tablePropValue}"`) .join(', '); From 9c59ceb9f9b1bef1f2fbe60f957c33bc41368897 Mon Sep 17 00:00:00 2001 From: Serhii Filonenko Date: Thu, 21 Aug 2025 10:41:56 +0300 Subject: [PATCH 09/10] HCK-12398: fix table parameter values --- .../helpers/awsCliScriptHelpers/glueTableHelper.js | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/forward_engineering/helpers/awsCliScriptHelpers/glueTableHelper.js b/forward_engineering/helpers/awsCliScriptHelpers/glueTableHelper.js index 7c9580e..25f8cd1 100644 --- a/forward_engineering/helpers/awsCliScriptHelpers/glueTableHelper.js +++ b/forward_engineering/helpers/awsCliScriptHelpers/glueTableHelper.js @@ -26,7 +26,9 @@ const getGlueTableCreateStatement = (tableSchema, databaseName) => { StoredAsSubDirectories: tableSchema.StoredAsSubDirectories, }, Parameters: mapTableParameters(tableSchema), - PartitionKeys: getGluePartitionKeyTableColumns(tableSchema.properties), + PartitionKeys: handleParameterByTableFormat(tableSchema, () => + getGluePartitionKeyTableColumns(tableSchema.properties), + ), TableType: tableSchema.externalTable ? 'EXTERNAL_TABLE' : '', }, ...getTableFormatParameters(tableSchema), @@ -77,13 +79,21 @@ const mapTableParameters = tableSchema => { } }; +const handleParameterByTableFormat = (tableSchema, getParameter) => { + if (tableSchema.tableFormat === TABLE_FORMAT.iceberg) { + return; + } + + return getParameter(); +}; + const getTableFormatParameters = tableSchema => { if (tableSchema.tableFormat === TABLE_FORMAT.iceberg) { return { OpenTableFormatInput: { IcebergInput: { MetadataOperation: 'CREATE', - Version: tableSchema.icebergVersion, + Version: `${tableSchema.icebergVersion}`, }, }, }; From 7c78369610f7c18f084762e558f8f326c06034ba Mon Sep 17 00:00:00 2001 From: Serhii Filonenko Date: Thu, 21 Aug 2025 11:03:43 +0300 Subject: [PATCH 10/10] HCK-12398: add log for exception --- reverse_engineering/api.js | 2 +- reverse_engineering/hqlToCollectionsVisitor.js | 15 +++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/reverse_engineering/api.js b/reverse_engineering/api.js index e404254..0c4c473 100644 --- a/reverse_engineering/api.js +++ b/reverse_engineering/api.js @@ -145,7 +145,7 @@ module.exports = { const tree = parser.statements(); - const hqlToCollectionsGenerator = new hqlToCollectionsVisitor(); + const hqlToCollectionsGenerator = new hqlToCollectionsVisitor(logger); const commands = tree.accept(hqlToCollectionsGenerator); const { result, info, relationships } = commandsService.convertCommandsToReDocs( diff --git a/reverse_engineering/hqlToCollectionsVisitor.js b/reverse_engineering/hqlToCollectionsVisitor.js index 70749ea..43d5054 100644 --- a/reverse_engineering/hqlToCollectionsVisitor.js +++ b/reverse_engineering/hqlToCollectionsVisitor.js @@ -51,6 +51,11 @@ const ALLOWED_COMMANDS = [ ]; class Visitor extends HiveParserVisitor { + constructor(logger) { + super(); + this.logger = logger; + } + visitStatement(ctx) { const execStatement = ctx.execStatement(); if (execStatement) { @@ -80,7 +85,7 @@ class Visitor extends HiveParserVisitor { const description = this.visitWhenExists(ctx, 'tableComment'); const location = this.visitWhenExists(ctx, 'tableLocation'); const tablePropertiesPrefixed = this.visitWhenExists(ctx, 'tablePropertiesPrefixed'); - const { tableFormat, tableProperties } = getTableProperties(tablePropertiesPrefixed); + const { tableFormat, tableProperties } = getTableProperties(tablePropertiesPrefixed, this.logger); const temporaryTable = Boolean(ctx.KW_TEMPORARY()); const externalTable = Boolean(ctx.KW_EXTERNAL()); const storedAsTable = this.visitWhenExists(ctx, 'tableFileFormat', {}); @@ -1447,9 +1452,9 @@ const getMappingType = ctx => { } }; -const getTableProperties = tablePropertiesPrefixed => { +const getTableProperties = (tablePropertiesPrefixed, logger) => { try { - const properties = tablePropertiesPrefixed.replace(/^\(/, '{').replace(/\)$/, '}').replace(/=/g, ':'); + const properties = tablePropertiesPrefixed.replace(/^\(/, '{').replace(/\)$/, '}'); const parsedProperties = JSON.parse(properties); const { table_type, ...restTableProperties } = parsedProperties; const tableFormat = table_type === 'ICEBERG' ? TABLE_FORMAT.iceberg : TABLE_FORMAT.standard; @@ -1459,7 +1464,9 @@ const getTableProperties = tablePropertiesPrefixed => { tableFormat, tableProperties, }; - } catch (err) { + } catch (error) { + logger.log('error', error, 'getTableProperties'); + return { tableFormat: TABLE_FORMAT.standard, tableProperties: [],