From 56b8cd6bb8c4ff048da964ff4d862e815ff0921f Mon Sep 17 00:00:00 2001 From: Ramit Gupta Date: Tue, 10 Mar 2026 18:43:25 +0530 Subject: [PATCH 1/2] HIVE-29413:Avoid code duplication by updating getPartCols method for iceberg tables --- .../mapreduce/TestHCatMultiOutputFormat.java | 2 +- .../hive/ql/ddl/table/AlterTableUtils.java | 2 +- .../column/show/ShowColumnsOperation.java | 5 +-- .../create/like/CreateTableLikeOperation.java | 2 +- .../table/info/desc/DescTableOperation.java | 2 +- .../formatter/TextDescTableFormatter.java | 4 +- .../JsonShowTableStatusFormatter.java | 2 +- .../TextShowTableStatusFormatter.java | 2 +- .../ddl/table/partition/PartitionUtils.java | 2 +- .../AlterTableExchangePartitionAnalyzer.java | 4 +- .../partition/show/ShowPartitionAnalyzer.java | 6 +-- .../archive/AlterTableArchiveOperation.java | 2 +- .../archive/AlterTableArchiveUtils.java | 2 +- .../archive/AlterTableUnarchiveOperation.java | 2 +- .../create/AbstractCreateViewAnalyzer.java | 2 +- .../hadoop/hive/ql/exec/ArchiveUtils.java | 6 +-- .../hadoop/hive/ql/exec/DDLPlanUtils.java | 4 +- .../apache/hadoop/hive/ql/exec/MoveTask.java | 2 +- .../apache/hadoop/hive/ql/exec/Utilities.java | 2 +- .../hive/ql/exec/repl/ReplLoadTask.java | 2 +- .../hive/ql/metadata/DummyPartition.java | 4 +- .../apache/hadoop/hive/ql/metadata/Hive.java | 16 ++++---- .../HiveMaterializedViewsRegistry.java | 2 +- .../hadoop/hive/ql/metadata/Partition.java | 11 +++--- .../apache/hadoop/hive/ql/metadata/Table.java | 37 ++++++++++++------- .../ql/optimizer/ColumnPrunerProcFactory.java | 3 +- .../hive/ql/optimizer/GenMapRedUtils.java | 2 +- .../ql/optimizer/ppr/PartitionPruner.java | 2 +- .../ql/parse/AcidExportSemanticAnalyzer.java | 3 +- .../hive/ql/parse/BaseSemanticAnalyzer.java | 2 +- .../hadoop/hive/ql/parse/CalcitePlanner.java | 2 +- .../parse/ColumnStatsAutoGatherContext.java | 2 +- .../ql/parse/ColumnStatsSemanticAnalyzer.java | 5 +-- .../hive/ql/parse/ImportSemanticAnalyzer.java | 2 +- .../hive/ql/parse/LoadSemanticAnalyzer.java | 4 +- .../hive/ql/parse/MergeSemanticAnalyzer.java | 4 +- .../hadoop/hive/ql/parse/ParseUtils.java | 5 +-- .../ql/parse/RewriteSemanticAnalyzer.java | 2 +- .../hive/ql/parse/SemanticAnalyzer.java | 6 +-- .../rewrite/CopyOnWriteMergeRewriter.java | 2 +- .../hive/ql/parse/rewrite/MergeRewriter.java | 4 +- .../ql/parse/rewrite/SplitMergeRewriter.java | 2 +- .../ql/parse/rewrite/SplitUpdateRewriter.java | 4 +- .../rewrite/sql/MultiInsertSqlGenerator.java | 6 +-- .../NativeAcidMultiInsertSqlGenerator.java | 6 +-- .../hive/ql/stats/ColStatsProcessor.java | 2 +- 46 files changed, 100 insertions(+), 97 deletions(-) diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java index d87158b23fae..b898785eaf5a 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java @@ -374,7 +374,7 @@ private List getTableData(String table, String database) throws Exceptio Hive hive = Hive.get(conf); org.apache.hadoop.hive.ql.metadata.Table tbl = hive.getTable(database, table); FetchWork work; - if (!tbl.getPartCols().isEmpty()) { + if (!tbl.getSupportedPartCols().isEmpty()) { List partitions = hive.getPartitions(tbl); List partDesc = new ArrayList(); List partLocs = new ArrayList(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableUtils.java index 17a964a44583..2f507d679aca 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableUtils.java @@ -75,7 +75,7 @@ public static boolean isSchemaEvolutionEnabled(Table table, Configuration conf) } public static boolean isFullPartitionSpec(Table table, Map partitionSpec) { - for (FieldSchema partitionCol : table.getPartCols()) { + for (FieldSchema partitionCol : table.getNativePartCols()) { if (partitionSpec.get(partitionCol.getName()) == null) { return false; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/show/ShowColumnsOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/show/ShowColumnsOperation.java index 289479b7ee79..6a2abd4ddfc2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/show/ShowColumnsOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/show/ShowColumnsOperation.java @@ -66,10 +66,7 @@ private List getColumnsByPattern() throws HiveException { private List getCols() throws HiveException { Table table = context.getDb().getTable(desc.getTableName()); - List allColumns = new ArrayList<>(); - allColumns.addAll(table.getCols()); - allColumns.addAll(table.getPartCols()); - return allColumns; + return new ArrayList<>(table.getAllCols()); } private Matcher getMatcher() { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/like/CreateTableLikeOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/like/CreateTableLikeOperation.java index 770724b90abf..38d95039dc66 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/like/CreateTableLikeOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/like/CreateTableLikeOperation.java @@ -100,7 +100,7 @@ private Table createViewLikeTable(Table oldTable) throws HiveException { setUserSpecifiedLocation(table); table.setFields(oldTable.getCols()); - table.setPartCols(oldTable.getPartCols()); + table.setPartCols(oldTable.getNativePartCols()); if (desc.getDefaultSerdeProps() != null) { for (Map.Entry e : desc.getDefaultSerdeProps().entrySet()) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java index b60126a8db15..d38157777482 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java @@ -131,7 +131,7 @@ private void getColumnsNoColumnPath(Table table, Partition partition, List partitionColumns = null; // TODO (HIVE-29413): Refactor to a generic getPartCols() implementation if (table.isPartitioned()) { - partitionColumns = table.hasNonNativePartitionSupport() ? - table.getStorageHandler().getPartitionKeys(table) : - table.getPartCols(); + partitionColumns = table.getSupportedPartCols(); } if (CollectionUtils.isNotEmpty(partitionColumns) && conf.getBoolVar(ConfVars.HIVE_DISPLAY_PARTITION_COLUMNS_SEPARATELY)) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/status/formatter/JsonShowTableStatusFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/status/formatter/JsonShowTableStatusFormatter.java index 073db26e756c..960d7fddb5ba 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/status/formatter/JsonShowTableStatusFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/status/formatter/JsonShowTableStatusFormatter.java @@ -69,7 +69,7 @@ private Map makeOneTableStatus(Table table, Hive db, HiveConf co builder.put("partitioned", table.isPartitioned()); if (table.isPartitioned()) { - builder.put("partitionColumns", JsonDescTableFormatter.createColumnsInfo(table.getPartCols(), + builder.put("partitionColumns", JsonDescTableFormatter.createColumnsInfo(table.getSupportedPartCols(), Collections.emptyList())); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/status/formatter/TextShowTableStatusFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/status/formatter/TextShowTableStatusFormatter.java index 552dc310465b..c6c6367812e6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/status/formatter/TextShowTableStatusFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/status/formatter/TextShowTableStatusFormatter.java @@ -73,7 +73,7 @@ private void writeStorageInfo(DataOutputStream out, Partition partition, Table t private void writeColumnsInfo(DataOutputStream out, Table table) throws IOException, UnsupportedEncodingException { String columns = MetaStoreUtils.getDDLFromFieldSchema("columns", table.getCols()); String partitionColumns = table.isPartitioned() ? - MetaStoreUtils.getDDLFromFieldSchema("partition_columns", table.getPartCols()) : ""; + MetaStoreUtils.getDDLFromFieldSchema("partition_columns", table.getSupportedPartCols()) : ""; out.write(Utilities.newLineCode); out.write(("columns:" + columns).getBytes(StandardCharsets.UTF_8)); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/PartitionUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/PartitionUtils.java index db7a5dfcd3d0..86f7b01a8bb6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/PartitionUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/PartitionUtils.java @@ -150,7 +150,7 @@ public static List getPartitionsWithSpecs(Hive db, Table table, GetPa } private static String tablePartitionColNames(Table table) { - List partCols = table.getPartCols(); + List partCols = table.getNativePartCols(); return String.join("/", partCols.toString()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/exchange/AlterTableExchangePartitionAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/exchange/AlterTableExchangePartitionAnalyzer.java index 6485627c7e6e..b16f5e522e5b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/exchange/AlterTableExchangePartitionAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/exchange/AlterTableExchangePartitionAnalyzer.java @@ -84,7 +84,7 @@ protected void analyzeCommand(TableName tableName, Map partition if (AcidUtils.isTransactionalTable(sourceTable) || AcidUtils.isTransactionalTable(destTable)) { throw new SemanticException(ErrorMsg.EXCHANGE_PARTITION_NOT_ALLOWED_WITH_TRANSACTIONAL_TABLES.getMsg()); } - List sourceProjectFilters = MetaStoreUtils.getPvals(sourceTable.getPartCols(), partitionSpecs); + List sourceProjectFilters = MetaStoreUtils.getPvals(sourceTable.getNativePartCols(), partitionSpecs); // check if source partition exists GetPartitionsFilterSpec sourcePartitionsFilterSpec = new GetPartitionsFilterSpec(); @@ -106,7 +106,7 @@ protected void analyzeCommand(TableName tableName, Map partition throw new SemanticException(ErrorMsg.PARTITION_VALUE_NOT_CONTINUOUS.getMsg(partitionSpecs.toString())); } - List destProjectFilters = MetaStoreUtils.getPvals(destTable.getPartCols(), partitionSpecs); + List destProjectFilters = MetaStoreUtils.getPvals(destTable.getNativePartCols(), partitionSpecs); // check if dest partition exists GetPartitionsFilterSpec getDestPartitionsFilterSpec = new GetPartitionsFilterSpec(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/show/ShowPartitionAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/show/ShowPartitionAnalyzer.java index c0bffcebdb23..3b639ace0d74 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/show/ShowPartitionAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/show/ShowPartitionAnalyzer.java @@ -102,7 +102,7 @@ ExprNodeDesc getShowPartitionsFilter(Table table, ASTNode command) throws Semant if (astChild.getType() == HiveParser.TOK_WHERE) { RowResolver rwsch = new RowResolver(); Map colTypes = new HashMap(); - for (FieldSchema fs : table.getPartCols()) { + for (FieldSchema fs : table.getSupportedPartCols()) { rwsch.put(table.getTableName(), fs.getName(), new ColumnInfo(fs.getName(), TypeInfoFactory.stringTypeInfo, null, true)); colTypes.put(fs.getName().toLowerCase(), fs.getType()); @@ -202,8 +202,8 @@ private String getShowPartitionsOrder(Table table, ASTNode command) throws Seman if (astChild.getType() == HiveParser.TOK_ORDERBY) { Map poses = new HashMap(); RowResolver rwsch = new RowResolver(); - for (int i = 0; i < table.getPartCols().size(); i++) { - FieldSchema fs = table.getPartCols().get(i); + for (int i = 0; i < table.getSupportedPartCols().size(); i++) { + FieldSchema fs = table.getSupportedPartCols().get(i); rwsch.put(table.getTableName(), fs.getName(), new ColumnInfo(fs.getName(), TypeInfoFactory.getPrimitiveTypeInfo(fs.getType()), null, true)); poses.put(fs.getName().toLowerCase(), i); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/archive/AlterTableArchiveOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/archive/AlterTableArchiveOperation.java index e218e590a24e..56d79e01ecd1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/archive/AlterTableArchiveOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/archive/AlterTableArchiveOperation.java @@ -129,7 +129,7 @@ private Path getOriginalDir(Table table, PartSpecInfo partitionSpecInfo, List partitionColumns) throws SemanticException { - if (oldView.getPartCols().isEmpty() || oldView.getPartCols().equals(partitionColumns)) { + if (oldView.getNativePartCols().isEmpty() || oldView.getNativePartCols().equals(partitionColumns)) { return; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java index ebe8f2f52775..5485906d8c57 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java @@ -74,7 +74,7 @@ static public PartSpecInfo create(Table tbl, Map partSpec) // ARCHIVE PARTITION(hr='13') won't List prefixFields = new ArrayList(); List prefixValues = new ArrayList(); - List partCols = tbl.getPartCols(); + List partCols = tbl.getNativePartCols(); Iterator itrPsKeys = partSpec.keySet().iterator(); for (FieldSchema fs : partCols) { if (!itrPsKeys.hasNext()) { @@ -222,7 +222,7 @@ public static int getArchivingLevel(Partition p) throws HiveException { * @throws HiveException */ public static String getPartialName(Partition p, int level) throws HiveException { - List fields = p.getTable().getPartCols().subList(0, level); + List fields = p.getTable().getNativePartCols().subList(0, level); List values = p.getValues().subList(0, level); try { return Warehouse.makePartName(fields, values); @@ -273,7 +273,7 @@ public static String conflictingArchiveNameOrNull(Hive db, Table tbl, Map spec = new HashMap(partSpec); List reversedKeys = new ArrayList(); - for (FieldSchema fs : tbl.getPartCols()) { + for (FieldSchema fs : tbl.getNativePartCols()) { if (spec.containsKey(fs.getName())) { reversedKeys.add(fs.getName()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java index a5bc66733f46..2bc7bf9718a6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java @@ -294,7 +294,7 @@ public String getPartitionActualName(Partition pt) { */ private Map getPartitionColumnToPrimitiveCategory(Partition pt) { Map resultMap = new HashMap<>(); - for (FieldSchema schema: pt.getTable().getPartCols()) { + for (FieldSchema schema: pt.getTable().getSupportedPartCols()) { resultMap.put( schema.getName(), ((PrimitiveTypeInfo) TypeInfoUtils.getTypeInfoFromTypeString(schema.getType())).getPrimitiveCategory() @@ -976,7 +976,7 @@ private String getComment(Table table) { } private String getPartitionsForView(Table table) { - List partitionKeys = table.getPartCols(); + List partitionKeys = table.getSupportedPartCols(); if (partitionKeys.isEmpty()) { return ""; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java index 3eca5531f127..7c040c4ba230 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java @@ -633,7 +633,7 @@ public void logMessage(LoadTableDesc tbd) { private DataContainer handleStaticParts(Hive db, Table table, LoadTableDesc tbd, TaskInformation ti) throws HiveException, IOException, InvalidOperationException { - List partVals = MetaStoreUtils.getPvals(table.getPartCols(), tbd.getPartitionSpec()); + List partVals = MetaStoreUtils.getPvals(table.getNativePartCols(), tbd.getPartitionSpec()); db.validatePartitionNameCharacters(partVals); if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { Utilities.FILE_OP_LOGGER.trace("loadPartition called from " + tbd.getSourcePath() diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index 37e91652fb88..b8c087eb1333 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -4317,7 +4317,7 @@ public static void setPartitionColumnNames(Configuration conf, TableScanOperator if (metadata == null) { return; } - List partCols = metadata.getPartCols(); + List partCols = metadata.getNativePartCols(); if (partCols != null && !partCols.isEmpty()) { conf.set(serdeConstants.LIST_PARTITION_COLUMNS, MetaStoreUtils.getColumnNamesFromFieldSchema(partCols)); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java index 9287fd75e766..69bf706242ed 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java @@ -548,7 +548,7 @@ public static Task createViewTask(MetaData metaData, String dbNameToLoadIn, H } CreateViewDesc desc = new CreateViewDesc(dbDotView, table.getCols(), null, table.getParameters(), - table.getPartColNames(), false, false, viewOriginalText, viewExpandedText, table.getPartCols()); + table.getPartColNames(), false, false, viewOriginalText, viewExpandedText, table.getNativePartCols()); desc.setReplicationSpec(metaData.getReplicationSpec()); desc.setOwnerName(table.getOwner()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java index c188eb09fdcf..be7c07019f02 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java @@ -91,9 +91,7 @@ public List getValues() { values = new ArrayList<>(); // TODO (HIVE-29413): Refactor to a generic getPartCols() implementation - for (FieldSchema fs : table.hasNonNativePartitionSupport() - ? table.getStorageHandler().getPartitionKeys(table) - : table.getPartCols()) { + for (FieldSchema fs : table.getSupportedPartCols()) { String val = partSpec.get(fs.getName()); values.add(val); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 0a63aea1478c..5d9fe44661e8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -868,7 +868,7 @@ public void createTable(String tableName, List columns, List par FieldSchema part = new FieldSchema(); part.setName(partCol); part.setType(STRING_TYPE_NAME); // default partition key - tbl.getPartCols().add(part); + tbl.getNativePartCols().add(part); } } tbl.setSerializationLib(LazySimpleSerDe.class.getName()); @@ -1245,8 +1245,8 @@ public void renamePartition(Table tbl, Map oldPartSpec, Partitio throws HiveException { try { Map newPartSpec = newPart.getSpec(); - if (oldPartSpec.keySet().size() != tbl.getPartCols().size() - || newPartSpec.keySet().size() != tbl.getPartCols().size()) { + if (oldPartSpec.keySet().size() != tbl.getNativePartCols().size() + || newPartSpec.keySet().size() != tbl.getNativePartCols().size()) { throw new HiveException("Unable to rename partition to the same name: number of partition cols don't match. "); } if (!oldPartSpec.keySet().equals(newPartSpec.keySet())){ @@ -1254,7 +1254,7 @@ public void renamePartition(Table tbl, Map oldPartSpec, Partitio } List pvals = new ArrayList(); - for (FieldSchema field : tbl.getPartCols()) { + for (FieldSchema field : tbl.getNativePartCols()) { String val = oldPartSpec.get(field.getName()); if (val == null || val.length() == 0) { throw new HiveException("get partition: Value for key " @@ -3831,7 +3831,7 @@ public Partition getPartition(Table tbl, Map partSpec, boolean forceCreate, String partPath, boolean inheritTableSpecs) throws HiveException { tbl.validatePartColumnNames(partSpec, true); List pvals = new ArrayList(); - for (FieldSchema field : tbl.getPartCols()) { + for (FieldSchema field : tbl.getNativePartCols()) { String val = partSpec.get(field.getName()); // enable dynamic partitioning if ((val == null && !HiveConf.getBoolVar(conf, HiveConf.ConfVars.DYNAMIC_PARTITIONING)) @@ -4220,7 +4220,7 @@ public List getPartitionNames(Table tbl, Map partSpec, s if (tbl.hasNonNativePartitionSupport()) { return tbl.getStorageHandler().getPartitionNames(tbl, partSpec); } - List pvals = MetaStoreUtils.getPvals(tbl.getPartCols(), partSpec); + List pvals = MetaStoreUtils.getPvals(tbl.getNativePartCols(), partSpec); return getPartitionNamesByPartitionVals(tbl, pvals, max); } @@ -4462,7 +4462,7 @@ private List getPartitionsWithAuth(Table tbl, Map par throw new HiveException(ErrorMsg.TABLE_NOT_PARTITIONED, tbl.getTableName()); } - List partialPvals = MetaStoreUtils.getPvals(tbl.getPartCols(), partialPartSpec); + List partialPvals = MetaStoreUtils.getPvals(tbl.getNativePartCols(), partialPartSpec); List partitions = null; try { @@ -4771,7 +4771,7 @@ static List convertFromPartSpec(Iterator iterator, Tab || partitionWithoutSD.getRelativePath().isEmpty()) { if (tbl.getDataLocation() != null) { Path partPath = new Path(tbl.getDataLocation(), - Warehouse.makePartName(tbl.getPartCols(), + Warehouse.makePartName(tbl.getNativePartCols(), partitionWithoutSD.getValues())); partitionLocation = partPath.toString(); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java index 0cf02a95e392..cdd165fa1bcd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java @@ -430,7 +430,7 @@ private static RelNode createMaterializedViewScan(HiveConf conf, Table viewTable // 1.2 Add column info corresponding to partition columns ArrayList partitionColumns = new ArrayList(); - for (FieldSchema part_col : viewTable.getPartCols()) { + for (FieldSchema part_col : viewTable.getNativePartCols()) { colName = part_col.getName(); colInfo = new ColumnInfo(colName, TypeInfoFactory.getPrimitiveTypeInfo(part_col.getType()), null, true); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java index 736e6e8c9f1a..1043ac74f61b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java @@ -125,7 +125,7 @@ public Partition(Table tbl, Map partSpec, Path location) throws public static org.apache.hadoop.hive.metastore.api.Partition createMetaPartitionObject( Table tbl, Map partSpec, Path location) throws HiveException { List pvals = new ArrayList(); - for (FieldSchema field : tbl.getPartCols()) { + for (FieldSchema field : tbl.getNativePartCols()) { String val = partSpec.get(field.getName()); if (val == null || val.isEmpty()) { throw new HiveException("partition spec is invalid; field " @@ -173,7 +173,8 @@ protected void initialize(Table table, // set default if location is not set and this is a physical // table partition (not a view partition) if (table.getDataLocation() != null) { - Path partPath = new Path(table.getDataLocation(), Warehouse.makePartName(table.getPartCols(), tPartition.getValues())); + Path partPath = new Path(table.getDataLocation(), + Warehouse.makePartName(table.getNativePartCols(), tPartition.getValues())); tPartition.getSd().setLocation(partPath.toString()); } } @@ -200,7 +201,7 @@ protected void initialize(Table table, public String getName() { try { - return Warehouse.makePartName(table.getPartCols(), tPartition.getValues()); + return Warehouse.makePartName(table.getSupportedPartCols(), tPartition.getValues()); } catch (MetaException e) { throw new RuntimeException(e); } @@ -543,7 +544,7 @@ public void setLocation(String location) { public void setValues(Map partSpec) throws HiveException { List pvals = new ArrayList(); - for (FieldSchema field : table.getPartCols()) { + for (FieldSchema field : table.getNativePartCols()) { String val = partSpec.get(field.getName()); if (val == null) { throw new HiveException( @@ -601,7 +602,7 @@ public Map, String> getSkewedColValueLocationMaps() { public void checkValidity() throws HiveException { if (!tPartition.getSd().equals(table.getSd())) { - Table.validateColumns(getCols(), table.getPartCols(), DDLUtils.isIcebergTable(table)); + Table.validateColumns(getCols(), table.getNativePartCols(), DDLUtils.isIcebergTable(table)); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java index f857e7d505f1..e33eb4b1d4a3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java @@ -270,7 +270,7 @@ public void checkValidity(Configuration conf) throws HiveException { if (getCols().isEmpty()) { throw new HiveException("at least one column must be specified for the table"); } - validateColumns(getCols(), getPartCols(), DDLUtils.isIcebergTable(this)); + validateColumns(getCols(), getNativePartCols(), DDLUtils.isIcebergTable(this)); if (!isView()) { if (null == getDeserializer(false)) { @@ -594,25 +594,30 @@ public boolean equals(Object obj) { && Objects.equals(snapshotRef, other.snapshotRef); } - public List getPartCols() { + public List getSupportedPartCols() { + return hasNonNativePartitionSupport() ? getStorageHandler().getPartitionKeys(this) : getNativePartCols(); + } + + public List getNativePartCols() { + List partKeys = tTable.getPartitionKeys(); + if (partKeys == null) { partKeys = new ArrayList<>(); tTable.setPartitionKeys(partKeys); } + return partKeys; } public FieldSchema getPartColByName(String colName) { - return getPartCols().stream() + return getNativePartCols().stream() .filter(key -> key.getName().toLowerCase().equals(colName)) .findFirst().orElse(null); } public List getPartColNames() { - List partCols = hasNonNativePartitionSupport() ? - getStorageHandler().getPartitionKeys(this) : getPartCols(); - return partCols.stream().map(FieldSchema::getName) + return getSupportedPartCols().stream().map(FieldSchema::getName) .collect(Collectors.toList()); } @@ -761,10 +766,16 @@ private List getColsInternal(boolean forMs) { * @return List<FieldSchema> */ public List getAllCols() { - ArrayList f_list = new ArrayList(); - f_list.addAll(getCols()); - f_list.addAll(getPartCols()); - return f_list; + ArrayList allCols = new ArrayList<>(getCols()); + Set colNames = allCols.stream() + .map(FieldSchema::getName) + .collect(Collectors.toSet()); + for (FieldSchema col : getSupportedPartCols()) { + if (!colNames.contains(col.getName())) { + allCols.add(col); + } + } + return allCols; } public void setPartCols(List partCols) { @@ -812,8 +823,8 @@ public void setOutputFormatClass(String name) throws HiveException { } public boolean isPartitioned() { - return hasNonNativePartitionSupport() ? getStorageHandler().isPartitioned(this) : - CollectionUtils.isNotEmpty(getPartCols()); + return hasNonNativePartitionSupport() ? getStorageHandler().isPartitioned(this) : + CollectionUtils.isNotEmpty(getNativePartCols()); } public void setFields(List fields) { @@ -1011,7 +1022,7 @@ public boolean isMaterializedView() { public LinkedHashMap createSpec( org.apache.hadoop.hive.metastore.api.Partition tp) { - List fsl = getPartCols(); + List fsl = getNativePartCols(); List tpl = tp.getValues(); LinkedHashMap spec = new LinkedHashMap(fsl.size()); for (int i = 0; i < fsl.size(); i++) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java index 3d3e4ce7663f..ec0096b96ea9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java @@ -807,8 +807,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx ctx, for (FieldNode col : cols) { int index = originalOutputColumnNames.indexOf(col.getFieldName()); Table tab = cppCtx.getParseContext().getViewProjectToTableSchema().get(op); - List fullFieldList = new ArrayList(tab.getCols()); - fullFieldList.addAll(tab.getPartCols()); + List fullFieldList = new ArrayList(tab.getAllCols()); cppCtx.getParseContext().getColumnAccessInfo() .add(tab.getCompleteName(), fullFieldList.get(index).getName()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java index bd1be003a512..15db24343b82 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java @@ -2149,7 +2149,7 @@ static void usePartitionColumns(Properties properties, Table table, List if (properties.containsKey(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS)) { usePartitionColumns(properties, partColNames); } else { - List partCols = table.getPartCols(); + List partCols = table.getNativePartCols(); String partNames = partCols.stream().map(FieldSchema::getName).collect(Collectors.joining("/")); String partTypes = partCols.stream().map(FieldSchema::getType).collect(Collectors.joining(":")); properties.setProperty( diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java index b1bc9eaf0a75..1083c16ecab5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java @@ -525,7 +525,7 @@ static private boolean pruneBySequentialScan(Table tab, List partitio } private static List extractPartColTypes(Table tab) { - List pCols = tab.getPartCols(); + List pCols = tab.getNativePartCols(); List partColTypeInfos = new ArrayList<>(pCols.size()); for (FieldSchema pCol : pCols) { partColTypeInfos.add(TypeInfoFactory.getPrimitiveTypeInfo(pCol.getType())); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java index 06912a1b3226..6c17d4d25e03 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java @@ -175,7 +175,8 @@ private void analyzeAcidExport(ASTNode ast, Table exportTable, ASTNode tokRefOrN //now generate insert statement //insert into newTableName select * from ts StringBuilder rewrittenQueryStr = generateExportQuery( - newTable.getPartCols(), tokRefOrNameExportTable, (ASTNode) tokRefOrNameExportTable.parent, newTableName); + newTable.getNativePartCols(), + tokRefOrNameExportTable, (ASTNode) tokRefOrNameExportTable.parent, newTableName); ReparseResult rr = ParseUtils.parseRewrittenQuery(ctx, rewrittenQueryStr); Context rewrittenCtx = rr.rewrittenCtx; rewrittenCtx.setIsUpdateDeleteMerge(false); //it's set in parseRewrittenQuery() diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index 986dcb7fcbbb..2a8b4bc62d9c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -1201,7 +1201,7 @@ public TableSpec(Table tableHandle, List partitions) { if (partitions != null && !partitions.isEmpty()) { this.specType = SpecType.STATIC_PARTITION; this.partitions = partitions; - List partCols = this.tableHandle.getPartCols(); + List partCols = this.tableHandle.getNativePartCols(); this.partSpec = new LinkedHashMap<>(); for (FieldSchema partCol : partCols) { partSpec.put(partCol.getName(), null); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index d5c683daa303..988e1b6a1209 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -3063,7 +3063,7 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc ArrayList partitionColumns = new ArrayList(); // 3.2 Add column info corresponding to partition columns - for (FieldSchema part_col : tabMetaData.getPartCols()) { + for (FieldSchema part_col : tabMetaData.getNativePartCols()) { colName = part_col.getName(); colInfo = new ColumnInfo(colName, TypeInfoFactory.getPrimitiveTypeInfo(part_col.getType()), diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java index 1b6f73ea264f..c0ee440e0d63 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java @@ -83,7 +83,7 @@ public ColumnStatsAutoGatherContext(SemanticAnalyzer sa, HiveConf conf, this.isInsertInto = isInsertInto; this.origCtx = ctx; columns = tbl.getCols(); - partitionColumns = tbl.getPartCols(); + partitionColumns = tbl.getNativePartCols(); } public List getLoadFileWork() { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java index 023934d9eb24..4250f185e277 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java @@ -193,8 +193,7 @@ private static CharSequence genPartitionClause(Table tbl, List pa private static String getColTypeOf(Table tbl, String partKey) { - for (FieldSchema fs : tbl.hasNonNativePartitionSupport() ? - tbl.getStorageHandler().getPartitionKeys(tbl) : tbl.getPartitionKeys()) { + for (FieldSchema fs : tbl.getNativePartCols()) { if (partKey.equalsIgnoreCase(fs.getName())) { return fs.getType().toLowerCase(); } @@ -278,7 +277,7 @@ private static String genRewrittenQuery(Table tbl, List colNames, List existingTablePartCols = table.getPartCols(); + List existingTablePartCols = table.getNativePartCols(); List importedTablePartCols = tableDesc.getPartCols(); if (!EximUtil.schemaCompare(importedTablePartCols, existingTablePartCols)) { throw new SemanticException( diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java index eb4a73f1e5e9..4e3115618253 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java @@ -511,7 +511,7 @@ private void reparseAndSuperAnalyze(Table table, URI fromURI) throws SemanticExc // Partition spec was already validated by caller when create TableSpec object. // So, need not validate inpPartSpec here. - List parts = table.getPartCols(); + List parts = table.getNativePartCols(); if (tableTree.getChildCount() >= 2) { ASTNode partSpecNode = (ASTNode) tableTree.getChild(1); inpPartSpec = new HashMap<>(partSpecNode.getChildCount()); @@ -561,7 +561,7 @@ private void reparseAndSuperAnalyze(Table table, URI fromURI) throws SemanticExc } rewrittenQueryStr.append(getFullTableNameForSQL((ASTNode)(tableTree.getChild(0)))); - addPartitionColsToInsert(table.getPartCols(), inpPartSpec, rewrittenQueryStr); + addPartitionColsToInsert(table.getNativePartCols(), inpPartSpec, rewrittenQueryStr); rewrittenQueryStr.append(" select * from "); rewrittenQueryStr.append(tempTblName); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/MergeSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/MergeSemanticAnalyzer.java index 882840ffef5a..751e398a1c5f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/MergeSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/MergeSemanticAnalyzer.java @@ -230,7 +230,7 @@ private MergeStatement.UpdateClause handleUpdate(ASTNode whenMatchedUpdateClause String deleteExtraPredicate) throws SemanticException { assert whenMatchedUpdateClause.getType() == HiveParser.TOK_MATCHED; assert getWhenClauseOperation(whenMatchedUpdateClause).getType() == HiveParser.TOK_UPDATE; - Map newValuesMap = new HashMap<>(targetTable.getCols().size() + targetTable.getPartCols().size()); + Map newValuesMap = new HashMap<>(targetTable.getAllCols().size()); ASTNode setClause = (ASTNode)getWhenClauseOperation(whenMatchedUpdateClause).getChild(0); //columns being updated -> update expressions; "setRCols" (last param) is null because we use actual expressions //before re-parsing, i.e. they are known to SemanticAnalyzer logic @@ -432,7 +432,7 @@ private static final class OnClauseAnalyzer { HiveConf conf, String onClauseAsString) { this.onClause = onClause; allTargetTableColumns.addAll(targetTable.getCols()); - allTargetTableColumns.addAll(targetTable.getPartCols()); + allTargetTableColumns.addAll(targetTable.getNativePartCols()); this.targetTableNameInSourceQuery = unescapeIdentifier(targetTableNameInSourceQuery); this.conf = conf; this.onClauseAsString = onClauseAsString; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java index 9964b9369065..9b48ce03456e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java @@ -581,8 +581,7 @@ public static Map> getFullPartitionSpecs( CommonTree ast, Table table, Configuration conf, boolean canGroupExprs) throws SemanticException { String defaultPartitionName = HiveConf.getVar(conf, HiveConf.ConfVars.DEFAULT_PARTITION_NAME); Map colTypes = new HashMap<>(); - List partitionKeys = table.hasNonNativePartitionSupport() ? - table.getStorageHandler().getPartitionKeys(table) : table.getPartitionKeys(); + List partitionKeys = table.getSupportedPartCols(); for (FieldSchema fs : partitionKeys) { colTypes.put(fs.getName().toLowerCase(), fs.getType()); } @@ -692,7 +691,7 @@ public static Map> getFullPartitionSpecs( */ private static int calculatePartPrefix(Table tbl, Set partSpecKeys) { int partPrefixToDrop = 0; - for (FieldSchema fs : tbl.getPartCols()) { + for (FieldSchema fs : tbl.getNativePartCols()) { if (!partSpecKeys.contains(fs.getName())) { break; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/RewriteSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/RewriteSemanticAnalyzer.java index 101f6b1fc3d8..fb118ba8ba5b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/RewriteSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/RewriteSemanticAnalyzer.java @@ -119,7 +119,7 @@ protected void checkValidSetClauseTarget(ASTNode colName, Table targetTable) thr String columnName = normalizeColName(colName.getText()); // Make sure this isn't one of the partitioning columns, that's not supported. - for (FieldSchema fschema : targetTable.getPartCols()) { + for (FieldSchema fschema : targetTable.getNativePartCols()) { if (fschema.getName().equalsIgnoreCase(columnName)) { throw new SemanticException(ErrorMsg.UPDATE_CANNOT_UPDATE_PART_VALUE.getMsg()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 5c4f049f0350..92488161d5f8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -2209,7 +2209,7 @@ private void handleInsertStatementSpecPhase1(ASTNode ast, QBParseInfo qbp, Phase } } else { // partition spec is not specified but column schema can have partitions specified - for(FieldSchema f : targetTable.getPartCols()) { + for(FieldSchema f : targetTable.getNativePartCols()) { //parser only allows foo(a,b), not foo(foo.a, foo.b) targetColumns.remove(f.getName()); } @@ -12038,7 +12038,7 @@ private Operator genTablePlan(String alias, QB qb) throws SemanticException { } // Hack!! - refactor once the metadata APIs with types are ready // Finally add the partitioning columns - for (FieldSchema part_col : tab.getPartCols()) { + for (FieldSchema part_col : tab.getNativePartCols()) { LOG.trace("Adding partition col: " + part_col); rwsch.put(alias, part_col.getName(), new ColumnInfo(part_col.getName(), TypeInfoFactory.getPrimitiveTypeInfo(part_col.getType()), alias, true)); @@ -12304,7 +12304,7 @@ private void setupStats(TableScanDesc tsDesc, QBParseInfo qbp, Table tab, String if (tab.isPartitioned() && !tab.hasNonNativePartitionSupport()) { List cols = new ArrayList(); if (qbp.getAnalyzeRewrite() != null) { - List partitionCols = tab.getPartCols(); + List partitionCols = tab.getNativePartCols(); for (FieldSchema fs : partitionCols) { cols.add(fs.getName()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/CopyOnWriteMergeRewriter.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/CopyOnWriteMergeRewriter.java index b72f2496d938..b7335473da85 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/CopyOnWriteMergeRewriter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/CopyOnWriteMergeRewriter.java @@ -202,7 +202,7 @@ public void appendWhenMatchedUpdateClause(MergeStatement.UpdateClause updateClau sqlGenerator.append(hintStr); hintStr = null; } - List values = new ArrayList<>(targetTable.getCols().size() + targetTable.getPartCols().size()); + List values = new ArrayList<>(targetTable.getAllCols().size()); values.addAll(sqlGenerator.getDeleteValues(Context.Operation.MERGE)); addValues(targetTable, targetAlias, updateClause.getNewValuesMap(), values); addValuesForRowLineageForCopyOnMerge(isRowLineageSupported, values, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/MergeRewriter.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/MergeRewriter.java index 3ec2e580f046..0b3b4e504d70 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/MergeRewriter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/MergeRewriter.java @@ -224,7 +224,7 @@ public void appendWhenMatchedUpdateClause(MergeStatement.UpdateClause updateClau sqlGenerator.append(" -- update clause").append("\n"); List valuesAndAcidSortKeys = new ArrayList<>( - targetTable.getCols().size() + targetTable.getPartCols().size() + 1); + targetTable.getAllCols().size() + 1); valuesAndAcidSortKeys.addAll(sqlGenerator.getSortKeys(Operation.MERGE)); addValues(targetTable, targetAlias, updateClause.getNewValuesMap(), valuesAndAcidSortKeys); sqlGenerator.appendInsertBranch(hintStr, valuesAndAcidSortKeys); @@ -250,7 +250,7 @@ protected void addValues(Table targetTable, String targetAlias, Map values.add( + targetTable.getNativePartCols().forEach(fieldSchema -> values.add( formatter.apply(fieldSchema.getName()))); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitMergeRewriter.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitMergeRewriter.java index 84fcf186f6b7..06edaca90f0f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitMergeRewriter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitMergeRewriter.java @@ -58,7 +58,7 @@ public void appendWhenMatchedUpdateClause(MergeStatement.UpdateClause updateClau String onClauseAsString = mergeStatement.getOnClauseAsText(); sqlGenerator.append(" -- update clause (insert part)\n"); - List values = new ArrayList<>(targetTable.getCols().size() + targetTable.getPartCols().size()); + List values = new ArrayList<>(targetTable.getAllCols().size()); addValues(targetTable, targetAlias, updateClause.getNewValuesMap(), values); addRowLineageColumnsForWhenMatchedUpdateClause(isRowLineageSupported, values, targetAlias, conf); sqlGenerator.appendInsertBranch(hintStr, values); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitUpdateRewriter.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitUpdateRewriter.java index d14ddc7eb485..7e93e9a024b3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitUpdateRewriter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitUpdateRewriter.java @@ -98,8 +98,8 @@ public ParseUtils.ReparseResult rewrite(Context context, UpdateStatement updateB insertValues.add(sqlGenerator.qualify(identifier)); } - if (updateBlock.getTargetTable().getPartCols() != null) { - updateBlock.getTargetTable().getPartCols().forEach( + if (updateBlock.getTargetTable().getNativePartCols() != null) { + updateBlock.getTargetTable().getNativePartCols().forEach( fieldSchema -> insertValues.add(sqlGenerator.qualify(HiveUtils.unparseIdentifier(fieldSchema.getName(), conf)))); } addRowLineageColumnsForUpdate(updateBlock.getTargetTable(), sqlGenerator, insertValues, conf); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/MultiInsertSqlGenerator.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/MultiInsertSqlGenerator.java index 7587daf13055..2bf818c1fd69 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/MultiInsertSqlGenerator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/MultiInsertSqlGenerator.java @@ -111,7 +111,7 @@ public void appendPartitionColsOfTarget() { */ public void appendPartitionCols(Table table) { // If the table is partitioned we have to put the partition() clause in - List partCols = table.getPartCols(); + List partCols = table.getNativePartCols(); if (partCols == null || partCols.isEmpty()) { return; } @@ -148,11 +148,11 @@ public void removeLastChar() { } public void appendPartColsOfTargetTableWithComma(String alias) { - if (targetTable.getPartCols() == null || targetTable.getPartCols().isEmpty()) { + if (targetTable.getNativePartCols() == null || targetTable.getNativePartCols().isEmpty()) { return; } queryStr.append(','); - appendCols(targetTable.getPartCols(), alias, null, FieldSchema::getName); + appendCols(targetTable.getNativePartCols(), alias, null, FieldSchema::getName); } public void appendAllColsOfTargetTable(String prefix) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/NativeAcidMultiInsertSqlGenerator.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/NativeAcidMultiInsertSqlGenerator.java index 87e426800442..c45ef18bdb96 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/NativeAcidMultiInsertSqlGenerator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/NativeAcidMultiInsertSqlGenerator.java @@ -36,7 +36,7 @@ public NativeAcidMultiInsertSqlGenerator(Table table, String targetTableFullName @Override public void appendAcidSelectColumns(Operation operation) { queryStr.append("ROW__ID,"); - for (FieldSchema fieldSchema : targetTable.getPartCols()) { + for (FieldSchema fieldSchema : targetTable.getNativePartCols()) { String identifier = HiveUtils.unparseIdentifier(fieldSchema.getName(), this.conf); queryStr.append(identifier); queryStr.append(","); @@ -45,9 +45,9 @@ public void appendAcidSelectColumns(Operation operation) { @Override public List getDeleteValues(Operation operation) { - List deleteValues = new ArrayList<>(1 + targetTable.getPartCols().size()); + List deleteValues = new ArrayList<>(1 + targetTable.getNativePartCols().size()); deleteValues.add(qualify("ROW__ID")); - for (FieldSchema fieldSchema : targetTable.getPartCols()) { + for (FieldSchema fieldSchema : targetTable.getNativePartCols()) { deleteValues.add(qualify(HiveUtils.unparseIdentifier(fieldSchema.getName(), conf))); } return deleteValues; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java index be62d94019ed..6c005bf1af61 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java @@ -156,7 +156,7 @@ private boolean constructColumnStatsFromPackedRows(Table tbl, List Date: Mon, 23 Mar 2026 14:01:10 +0530 Subject: [PATCH 2/2] updated api usage --- .../mapreduce/TestHCatMultiOutputFormat.java | 2 +- .../hive/ql/ddl/table/AlterTableUtils.java | 2 +- .../create/like/CreateTableLikeOperation.java | 2 +- .../ddl/table/info/desc/DescTableOperation.java | 2 +- .../formatter/JsonShowTableStatusFormatter.java | 2 +- .../formatter/TextShowTableStatusFormatter.java | 2 +- .../ql/ddl/table/partition/PartitionUtils.java | 3 +-- .../AlterTableExchangePartitionAnalyzer.java | 4 ++-- .../partition/show/ShowPartitionAnalyzer.java | 6 +++--- .../archive/AlterTableArchiveOperation.java | 2 +- .../storage/archive/AlterTableArchiveUtils.java | 2 +- .../archive/AlterTableUnarchiveOperation.java | 2 +- .../view/create/AbstractCreateViewAnalyzer.java | 2 +- .../hadoop/hive/ql/exec/ArchiveUtils.java | 6 +++--- .../hadoop/hive/ql/exec/DDLPlanUtils.java | 4 ++-- .../apache/hadoop/hive/ql/exec/MoveTask.java | 2 +- .../apache/hadoop/hive/ql/exec/Utilities.java | 2 +- .../hadoop/hive/ql/exec/repl/ReplLoadTask.java | 2 +- .../apache/hadoop/hive/ql/metadata/Hive.java | 16 ++++++++-------- .../metadata/HiveMaterializedViewsRegistry.java | 2 +- .../hadoop/hive/ql/metadata/Partition.java | 10 +++++----- .../apache/hadoop/hive/ql/metadata/Table.java | 17 +++++++---------- .../hive/ql/optimizer/GenMapRedUtils.java | 2 +- .../hive/ql/optimizer/ppr/PartitionPruner.java | 2 +- .../ql/parse/AcidExportSemanticAnalyzer.java | 2 +- .../hive/ql/parse/BaseSemanticAnalyzer.java | 2 +- .../hadoop/hive/ql/parse/CalcitePlanner.java | 2 +- .../ql/parse/ColumnStatsAutoGatherContext.java | 3 +-- .../ql/parse/ColumnStatsSemanticAnalyzer.java | 4 ++-- .../hive/ql/parse/ImportSemanticAnalyzer.java | 4 +--- .../hive/ql/parse/LoadSemanticAnalyzer.java | 4 ++-- .../hive/ql/parse/MergeSemanticAnalyzer.java | 3 +-- .../apache/hadoop/hive/ql/parse/ParseUtils.java | 2 +- .../hive/ql/parse/RewriteSemanticAnalyzer.java | 2 +- .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 6 +++--- .../hive/ql/parse/rewrite/MergeRewriter.java | 2 +- .../ql/parse/rewrite/SplitUpdateRewriter.java | 4 ++-- .../rewrite/sql/MultiInsertSqlGenerator.java | 6 +++--- .../sql/NativeAcidMultiInsertSqlGenerator.java | 6 +++--- .../hadoop/hive/ql/stats/ColStatsProcessor.java | 2 +- 40 files changed, 72 insertions(+), 80 deletions(-) diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java index b898785eaf5a..d87158b23fae 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java @@ -374,7 +374,7 @@ private List getTableData(String table, String database) throws Exceptio Hive hive = Hive.get(conf); org.apache.hadoop.hive.ql.metadata.Table tbl = hive.getTable(database, table); FetchWork work; - if (!tbl.getSupportedPartCols().isEmpty()) { + if (!tbl.getPartCols().isEmpty()) { List partitions = hive.getPartitions(tbl); List partDesc = new ArrayList(); List partLocs = new ArrayList(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableUtils.java index 2f507d679aca..17a964a44583 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableUtils.java @@ -75,7 +75,7 @@ public static boolean isSchemaEvolutionEnabled(Table table, Configuration conf) } public static boolean isFullPartitionSpec(Table table, Map partitionSpec) { - for (FieldSchema partitionCol : table.getNativePartCols()) { + for (FieldSchema partitionCol : table.getPartCols()) { if (partitionSpec.get(partitionCol.getName()) == null) { return false; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/like/CreateTableLikeOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/like/CreateTableLikeOperation.java index 38d95039dc66..770724b90abf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/like/CreateTableLikeOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/like/CreateTableLikeOperation.java @@ -100,7 +100,7 @@ private Table createViewLikeTable(Table oldTable) throws HiveException { setUserSpecifiedLocation(table); table.setFields(oldTable.getCols()); - table.setPartCols(oldTable.getNativePartCols()); + table.setPartCols(oldTable.getPartCols()); if (desc.getDefaultSerdeProps() != null) { for (Map.Entry e : desc.getDefaultSerdeProps().entrySet()) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java index d38157777482..b60126a8db15 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java @@ -131,7 +131,7 @@ private void getColumnsNoColumnPath(Table table, Partition partition, List makeOneTableStatus(Table table, Hive db, HiveConf co builder.put("partitioned", table.isPartitioned()); if (table.isPartitioned()) { - builder.put("partitionColumns", JsonDescTableFormatter.createColumnsInfo(table.getSupportedPartCols(), + builder.put("partitionColumns", JsonDescTableFormatter.createColumnsInfo(table.getPartCols(), Collections.emptyList())); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/status/formatter/TextShowTableStatusFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/status/formatter/TextShowTableStatusFormatter.java index c6c6367812e6..552dc310465b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/status/formatter/TextShowTableStatusFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/status/formatter/TextShowTableStatusFormatter.java @@ -73,7 +73,7 @@ private void writeStorageInfo(DataOutputStream out, Partition partition, Table t private void writeColumnsInfo(DataOutputStream out, Table table) throws IOException, UnsupportedEncodingException { String columns = MetaStoreUtils.getDDLFromFieldSchema("columns", table.getCols()); String partitionColumns = table.isPartitioned() ? - MetaStoreUtils.getDDLFromFieldSchema("partition_columns", table.getSupportedPartCols()) : ""; + MetaStoreUtils.getDDLFromFieldSchema("partition_columns", table.getPartCols()) : ""; out.write(Utilities.newLineCode); out.write(("columns:" + columns).getBytes(StandardCharsets.UTF_8)); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/PartitionUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/PartitionUtils.java index 86f7b01a8bb6..5882e4616506 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/PartitionUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/PartitionUtils.java @@ -23,7 +23,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.Map.Entry; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; @@ -150,7 +149,7 @@ public static List getPartitionsWithSpecs(Hive db, Table table, GetPa } private static String tablePartitionColNames(Table table) { - List partCols = table.getNativePartCols(); + List partCols = table.getPartCols(); return String.join("/", partCols.toString()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/exchange/AlterTableExchangePartitionAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/exchange/AlterTableExchangePartitionAnalyzer.java index b16f5e522e5b..6485627c7e6e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/exchange/AlterTableExchangePartitionAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/exchange/AlterTableExchangePartitionAnalyzer.java @@ -84,7 +84,7 @@ protected void analyzeCommand(TableName tableName, Map partition if (AcidUtils.isTransactionalTable(sourceTable) || AcidUtils.isTransactionalTable(destTable)) { throw new SemanticException(ErrorMsg.EXCHANGE_PARTITION_NOT_ALLOWED_WITH_TRANSACTIONAL_TABLES.getMsg()); } - List sourceProjectFilters = MetaStoreUtils.getPvals(sourceTable.getNativePartCols(), partitionSpecs); + List sourceProjectFilters = MetaStoreUtils.getPvals(sourceTable.getPartCols(), partitionSpecs); // check if source partition exists GetPartitionsFilterSpec sourcePartitionsFilterSpec = new GetPartitionsFilterSpec(); @@ -106,7 +106,7 @@ protected void analyzeCommand(TableName tableName, Map partition throw new SemanticException(ErrorMsg.PARTITION_VALUE_NOT_CONTINUOUS.getMsg(partitionSpecs.toString())); } - List destProjectFilters = MetaStoreUtils.getPvals(destTable.getNativePartCols(), partitionSpecs); + List destProjectFilters = MetaStoreUtils.getPvals(destTable.getPartCols(), partitionSpecs); // check if dest partition exists GetPartitionsFilterSpec getDestPartitionsFilterSpec = new GetPartitionsFilterSpec(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/show/ShowPartitionAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/show/ShowPartitionAnalyzer.java index 3b639ace0d74..c0bffcebdb23 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/show/ShowPartitionAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/show/ShowPartitionAnalyzer.java @@ -102,7 +102,7 @@ ExprNodeDesc getShowPartitionsFilter(Table table, ASTNode command) throws Semant if (astChild.getType() == HiveParser.TOK_WHERE) { RowResolver rwsch = new RowResolver(); Map colTypes = new HashMap(); - for (FieldSchema fs : table.getSupportedPartCols()) { + for (FieldSchema fs : table.getPartCols()) { rwsch.put(table.getTableName(), fs.getName(), new ColumnInfo(fs.getName(), TypeInfoFactory.stringTypeInfo, null, true)); colTypes.put(fs.getName().toLowerCase(), fs.getType()); @@ -202,8 +202,8 @@ private String getShowPartitionsOrder(Table table, ASTNode command) throws Seman if (astChild.getType() == HiveParser.TOK_ORDERBY) { Map poses = new HashMap(); RowResolver rwsch = new RowResolver(); - for (int i = 0; i < table.getSupportedPartCols().size(); i++) { - FieldSchema fs = table.getSupportedPartCols().get(i); + for (int i = 0; i < table.getPartCols().size(); i++) { + FieldSchema fs = table.getPartCols().get(i); rwsch.put(table.getTableName(), fs.getName(), new ColumnInfo(fs.getName(), TypeInfoFactory.getPrimitiveTypeInfo(fs.getType()), null, true)); poses.put(fs.getName().toLowerCase(), i); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/archive/AlterTableArchiveOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/archive/AlterTableArchiveOperation.java index 56d79e01ecd1..e218e590a24e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/archive/AlterTableArchiveOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/archive/AlterTableArchiveOperation.java @@ -129,7 +129,7 @@ private Path getOriginalDir(Table table, PartSpecInfo partitionSpecInfo, List partitionColumns) throws SemanticException { - if (oldView.getNativePartCols().isEmpty() || oldView.getNativePartCols().equals(partitionColumns)) { + if (oldView.getPartCols().isEmpty() || oldView.getPartCols().equals(partitionColumns)) { return; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java index 5485906d8c57..ebe8f2f52775 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java @@ -74,7 +74,7 @@ static public PartSpecInfo create(Table tbl, Map partSpec) // ARCHIVE PARTITION(hr='13') won't List prefixFields = new ArrayList(); List prefixValues = new ArrayList(); - List partCols = tbl.getNativePartCols(); + List partCols = tbl.getPartCols(); Iterator itrPsKeys = partSpec.keySet().iterator(); for (FieldSchema fs : partCols) { if (!itrPsKeys.hasNext()) { @@ -222,7 +222,7 @@ public static int getArchivingLevel(Partition p) throws HiveException { * @throws HiveException */ public static String getPartialName(Partition p, int level) throws HiveException { - List fields = p.getTable().getNativePartCols().subList(0, level); + List fields = p.getTable().getPartCols().subList(0, level); List values = p.getValues().subList(0, level); try { return Warehouse.makePartName(fields, values); @@ -273,7 +273,7 @@ public static String conflictingArchiveNameOrNull(Hive db, Table tbl, Map spec = new HashMap(partSpec); List reversedKeys = new ArrayList(); - for (FieldSchema fs : tbl.getNativePartCols()) { + for (FieldSchema fs : tbl.getPartCols()) { if (spec.containsKey(fs.getName())) { reversedKeys.add(fs.getName()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java index 2bc7bf9718a6..a5bc66733f46 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java @@ -294,7 +294,7 @@ public String getPartitionActualName(Partition pt) { */ private Map getPartitionColumnToPrimitiveCategory(Partition pt) { Map resultMap = new HashMap<>(); - for (FieldSchema schema: pt.getTable().getSupportedPartCols()) { + for (FieldSchema schema: pt.getTable().getPartCols()) { resultMap.put( schema.getName(), ((PrimitiveTypeInfo) TypeInfoUtils.getTypeInfoFromTypeString(schema.getType())).getPrimitiveCategory() @@ -976,7 +976,7 @@ private String getComment(Table table) { } private String getPartitionsForView(Table table) { - List partitionKeys = table.getSupportedPartCols(); + List partitionKeys = table.getPartCols(); if (partitionKeys.isEmpty()) { return ""; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java index 7c040c4ba230..3eca5531f127 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java @@ -633,7 +633,7 @@ public void logMessage(LoadTableDesc tbd) { private DataContainer handleStaticParts(Hive db, Table table, LoadTableDesc tbd, TaskInformation ti) throws HiveException, IOException, InvalidOperationException { - List partVals = MetaStoreUtils.getPvals(table.getNativePartCols(), tbd.getPartitionSpec()); + List partVals = MetaStoreUtils.getPvals(table.getPartCols(), tbd.getPartitionSpec()); db.validatePartitionNameCharacters(partVals); if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { Utilities.FILE_OP_LOGGER.trace("loadPartition called from " + tbd.getSourcePath() diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index b8c087eb1333..37e91652fb88 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -4317,7 +4317,7 @@ public static void setPartitionColumnNames(Configuration conf, TableScanOperator if (metadata == null) { return; } - List partCols = metadata.getNativePartCols(); + List partCols = metadata.getPartCols(); if (partCols != null && !partCols.isEmpty()) { conf.set(serdeConstants.LIST_PARTITION_COLUMNS, MetaStoreUtils.getColumnNamesFromFieldSchema(partCols)); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java index 69bf706242ed..9287fd75e766 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java @@ -548,7 +548,7 @@ public static Task createViewTask(MetaData metaData, String dbNameToLoadIn, H } CreateViewDesc desc = new CreateViewDesc(dbDotView, table.getCols(), null, table.getParameters(), - table.getPartColNames(), false, false, viewOriginalText, viewExpandedText, table.getNativePartCols()); + table.getPartColNames(), false, false, viewOriginalText, viewExpandedText, table.getPartCols()); desc.setReplicationSpec(metaData.getReplicationSpec()); desc.setOwnerName(table.getOwner()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 5d9fe44661e8..0a63aea1478c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -868,7 +868,7 @@ public void createTable(String tableName, List columns, List par FieldSchema part = new FieldSchema(); part.setName(partCol); part.setType(STRING_TYPE_NAME); // default partition key - tbl.getNativePartCols().add(part); + tbl.getPartCols().add(part); } } tbl.setSerializationLib(LazySimpleSerDe.class.getName()); @@ -1245,8 +1245,8 @@ public void renamePartition(Table tbl, Map oldPartSpec, Partitio throws HiveException { try { Map newPartSpec = newPart.getSpec(); - if (oldPartSpec.keySet().size() != tbl.getNativePartCols().size() - || newPartSpec.keySet().size() != tbl.getNativePartCols().size()) { + if (oldPartSpec.keySet().size() != tbl.getPartCols().size() + || newPartSpec.keySet().size() != tbl.getPartCols().size()) { throw new HiveException("Unable to rename partition to the same name: number of partition cols don't match. "); } if (!oldPartSpec.keySet().equals(newPartSpec.keySet())){ @@ -1254,7 +1254,7 @@ public void renamePartition(Table tbl, Map oldPartSpec, Partitio } List pvals = new ArrayList(); - for (FieldSchema field : tbl.getNativePartCols()) { + for (FieldSchema field : tbl.getPartCols()) { String val = oldPartSpec.get(field.getName()); if (val == null || val.length() == 0) { throw new HiveException("get partition: Value for key " @@ -3831,7 +3831,7 @@ public Partition getPartition(Table tbl, Map partSpec, boolean forceCreate, String partPath, boolean inheritTableSpecs) throws HiveException { tbl.validatePartColumnNames(partSpec, true); List pvals = new ArrayList(); - for (FieldSchema field : tbl.getNativePartCols()) { + for (FieldSchema field : tbl.getPartCols()) { String val = partSpec.get(field.getName()); // enable dynamic partitioning if ((val == null && !HiveConf.getBoolVar(conf, HiveConf.ConfVars.DYNAMIC_PARTITIONING)) @@ -4220,7 +4220,7 @@ public List getPartitionNames(Table tbl, Map partSpec, s if (tbl.hasNonNativePartitionSupport()) { return tbl.getStorageHandler().getPartitionNames(tbl, partSpec); } - List pvals = MetaStoreUtils.getPvals(tbl.getNativePartCols(), partSpec); + List pvals = MetaStoreUtils.getPvals(tbl.getPartCols(), partSpec); return getPartitionNamesByPartitionVals(tbl, pvals, max); } @@ -4462,7 +4462,7 @@ private List getPartitionsWithAuth(Table tbl, Map par throw new HiveException(ErrorMsg.TABLE_NOT_PARTITIONED, tbl.getTableName()); } - List partialPvals = MetaStoreUtils.getPvals(tbl.getNativePartCols(), partialPartSpec); + List partialPvals = MetaStoreUtils.getPvals(tbl.getPartCols(), partialPartSpec); List partitions = null; try { @@ -4771,7 +4771,7 @@ static List convertFromPartSpec(Iterator iterator, Tab || partitionWithoutSD.getRelativePath().isEmpty()) { if (tbl.getDataLocation() != null) { Path partPath = new Path(tbl.getDataLocation(), - Warehouse.makePartName(tbl.getNativePartCols(), + Warehouse.makePartName(tbl.getPartCols(), partitionWithoutSD.getValues())); partitionLocation = partPath.toString(); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java index cdd165fa1bcd..0cf02a95e392 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java @@ -430,7 +430,7 @@ private static RelNode createMaterializedViewScan(HiveConf conf, Table viewTable // 1.2 Add column info corresponding to partition columns ArrayList partitionColumns = new ArrayList(); - for (FieldSchema part_col : viewTable.getNativePartCols()) { + for (FieldSchema part_col : viewTable.getPartCols()) { colName = part_col.getName(); colInfo = new ColumnInfo(colName, TypeInfoFactory.getPrimitiveTypeInfo(part_col.getType()), null, true); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java index 1043ac74f61b..330f37d1ef1c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java @@ -125,7 +125,7 @@ public Partition(Table tbl, Map partSpec, Path location) throws public static org.apache.hadoop.hive.metastore.api.Partition createMetaPartitionObject( Table tbl, Map partSpec, Path location) throws HiveException { List pvals = new ArrayList(); - for (FieldSchema field : tbl.getNativePartCols()) { + for (FieldSchema field : tbl.getPartCols()) { String val = partSpec.get(field.getName()); if (val == null || val.isEmpty()) { throw new HiveException("partition spec is invalid; field " @@ -174,7 +174,7 @@ protected void initialize(Table table, // table partition (not a view partition) if (table.getDataLocation() != null) { Path partPath = new Path(table.getDataLocation(), - Warehouse.makePartName(table.getNativePartCols(), tPartition.getValues())); + Warehouse.makePartName(table.getPartCols(), tPartition.getValues())); tPartition.getSd().setLocation(partPath.toString()); } } @@ -201,7 +201,7 @@ protected void initialize(Table table, public String getName() { try { - return Warehouse.makePartName(table.getSupportedPartCols(), tPartition.getValues()); + return Warehouse.makePartName(table.getPartCols(), tPartition.getValues()); } catch (MetaException e) { throw new RuntimeException(e); } @@ -544,7 +544,7 @@ public void setLocation(String location) { public void setValues(Map partSpec) throws HiveException { List pvals = new ArrayList(); - for (FieldSchema field : table.getNativePartCols()) { + for (FieldSchema field : table.getPartCols()) { String val = partSpec.get(field.getName()); if (val == null) { throw new HiveException( @@ -602,7 +602,7 @@ public Map, String> getSkewedColValueLocationMaps() { public void checkValidity() throws HiveException { if (!tPartition.getSd().equals(table.getSd())) { - Table.validateColumns(getCols(), table.getNativePartCols(), DDLUtils.isIcebergTable(table)); + Table.validateColumns(getCols(), table.getPartCols(), DDLUtils.isIcebergTable(table)); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java index e33eb4b1d4a3..afed76d11d87 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java @@ -270,7 +270,7 @@ public void checkValidity(Configuration conf) throws HiveException { if (getCols().isEmpty()) { throw new HiveException("at least one column must be specified for the table"); } - validateColumns(getCols(), getNativePartCols(), DDLUtils.isIcebergTable(this)); + validateColumns(getCols(), getPartCols(), DDLUtils.isIcebergTable(this)); if (!isView()) { if (null == getDeserializer(false)) { @@ -595,23 +595,20 @@ public boolean equals(Object obj) { } public List getSupportedPartCols() { - return hasNonNativePartitionSupport() ? getStorageHandler().getPartitionKeys(this) : getNativePartCols(); + return hasNonNativePartitionSupport() ? getStorageHandler().getPartitionKeys(this) : getPartCols(); } - public List getNativePartCols() { - + public List getPartCols() { List partKeys = tTable.getPartitionKeys(); - if (partKeys == null) { partKeys = new ArrayList<>(); tTable.setPartitionKeys(partKeys); } - return partKeys; } public FieldSchema getPartColByName(String colName) { - return getNativePartCols().stream() + return getPartCols().stream() .filter(key -> key.getName().toLowerCase().equals(colName)) .findFirst().orElse(null); } @@ -766,7 +763,7 @@ private List getColsInternal(boolean forMs) { * @return List<FieldSchema> */ public List getAllCols() { - ArrayList allCols = new ArrayList<>(getCols()); + List allCols = new ArrayList<>(getCols()); Set colNames = allCols.stream() .map(FieldSchema::getName) .collect(Collectors.toSet()); @@ -824,7 +821,7 @@ public void setOutputFormatClass(String name) throws HiveException { public boolean isPartitioned() { return hasNonNativePartitionSupport() ? getStorageHandler().isPartitioned(this) : - CollectionUtils.isNotEmpty(getNativePartCols()); + CollectionUtils.isNotEmpty(getPartCols()); } public void setFields(List fields) { @@ -1022,7 +1019,7 @@ public boolean isMaterializedView() { public LinkedHashMap createSpec( org.apache.hadoop.hive.metastore.api.Partition tp) { - List fsl = getNativePartCols(); + List fsl = getPartCols(); List tpl = tp.getValues(); LinkedHashMap spec = new LinkedHashMap(fsl.size()); for (int i = 0; i < fsl.size(); i++) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java index 15db24343b82..bd1be003a512 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java @@ -2149,7 +2149,7 @@ static void usePartitionColumns(Properties properties, Table table, List if (properties.containsKey(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS)) { usePartitionColumns(properties, partColNames); } else { - List partCols = table.getNativePartCols(); + List partCols = table.getPartCols(); String partNames = partCols.stream().map(FieldSchema::getName).collect(Collectors.joining("/")); String partTypes = partCols.stream().map(FieldSchema::getType).collect(Collectors.joining(":")); properties.setProperty( diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java index 1083c16ecab5..b1bc9eaf0a75 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java @@ -525,7 +525,7 @@ static private boolean pruneBySequentialScan(Table tab, List partitio } private static List extractPartColTypes(Table tab) { - List pCols = tab.getNativePartCols(); + List pCols = tab.getPartCols(); List partColTypeInfos = new ArrayList<>(pCols.size()); for (FieldSchema pCol : pCols) { partColTypeInfos.add(TypeInfoFactory.getPrimitiveTypeInfo(pCol.getType())); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java index 6c17d4d25e03..05f3b85f271f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java @@ -175,7 +175,7 @@ private void analyzeAcidExport(ASTNode ast, Table exportTable, ASTNode tokRefOrN //now generate insert statement //insert into newTableName select * from ts StringBuilder rewrittenQueryStr = generateExportQuery( - newTable.getNativePartCols(), + newTable.getPartCols(), tokRefOrNameExportTable, (ASTNode) tokRefOrNameExportTable.parent, newTableName); ReparseResult rr = ParseUtils.parseRewrittenQuery(ctx, rewrittenQueryStr); Context rewrittenCtx = rr.rewrittenCtx; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index 2a8b4bc62d9c..986dcb7fcbbb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -1201,7 +1201,7 @@ public TableSpec(Table tableHandle, List partitions) { if (partitions != null && !partitions.isEmpty()) { this.specType = SpecType.STATIC_PARTITION; this.partitions = partitions; - List partCols = this.tableHandle.getNativePartCols(); + List partCols = this.tableHandle.getPartCols(); this.partSpec = new LinkedHashMap<>(); for (FieldSchema partCol : partCols) { partSpec.put(partCol.getName(), null); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 988e1b6a1209..d5c683daa303 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -3063,7 +3063,7 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc ArrayList partitionColumns = new ArrayList(); // 3.2 Add column info corresponding to partition columns - for (FieldSchema part_col : tabMetaData.getNativePartCols()) { + for (FieldSchema part_col : tabMetaData.getPartCols()) { colName = part_col.getName(); colInfo = new ColumnInfo(colName, TypeInfoFactory.getPrimitiveTypeInfo(part_col.getType()), diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java index c0ee440e0d63..c7c74cb53282 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java @@ -24,7 +24,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -83,7 +82,7 @@ public ColumnStatsAutoGatherContext(SemanticAnalyzer sa, HiveConf conf, this.isInsertInto = isInsertInto; this.origCtx = ctx; columns = tbl.getCols(); - partitionColumns = tbl.getNativePartCols(); + partitionColumns = tbl.getPartCols(); } public List getLoadFileWork() { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java index 4250f185e277..6f1a116a524b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java @@ -193,7 +193,7 @@ private static CharSequence genPartitionClause(Table tbl, List pa private static String getColTypeOf(Table tbl, String partKey) { - for (FieldSchema fs : tbl.getNativePartCols()) { + for (FieldSchema fs : tbl.getSupportedPartCols()) { if (partKey.equalsIgnoreCase(fs.getName())) { return fs.getType().toLowerCase(); } @@ -277,7 +277,7 @@ private static String genRewrittenQuery(Table tbl, List colNames, List existingTablePartCols = table.getNativePartCols(); + List existingTablePartCols = table.getPartCols(); List importedTablePartCols = tableDesc.getPartCols(); if (!EximUtil.schemaCompare(importedTablePartCols, existingTablePartCols)) { throw new SemanticException( diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java index 4e3115618253..eb4a73f1e5e9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java @@ -511,7 +511,7 @@ private void reparseAndSuperAnalyze(Table table, URI fromURI) throws SemanticExc // Partition spec was already validated by caller when create TableSpec object. // So, need not validate inpPartSpec here. - List parts = table.getNativePartCols(); + List parts = table.getPartCols(); if (tableTree.getChildCount() >= 2) { ASTNode partSpecNode = (ASTNode) tableTree.getChild(1); inpPartSpec = new HashMap<>(partSpecNode.getChildCount()); @@ -561,7 +561,7 @@ private void reparseAndSuperAnalyze(Table table, URI fromURI) throws SemanticExc } rewrittenQueryStr.append(getFullTableNameForSQL((ASTNode)(tableTree.getChild(0)))); - addPartitionColsToInsert(table.getNativePartCols(), inpPartSpec, rewrittenQueryStr); + addPartitionColsToInsert(table.getPartCols(), inpPartSpec, rewrittenQueryStr); rewrittenQueryStr.append(" select * from "); rewrittenQueryStr.append(tempTblName); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/MergeSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/MergeSemanticAnalyzer.java index 751e398a1c5f..ac0a7e8f12be 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/MergeSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/MergeSemanticAnalyzer.java @@ -29,7 +29,6 @@ import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.rewrite.MergeStatement; import org.apache.hadoop.hive.ql.parse.rewrite.RewriterFactory; -import org.apache.hadoop.hive.ql.plan.HiveOperation; import java.util.ArrayList; import java.util.HashMap; @@ -432,7 +431,7 @@ private static final class OnClauseAnalyzer { HiveConf conf, String onClauseAsString) { this.onClause = onClause; allTargetTableColumns.addAll(targetTable.getCols()); - allTargetTableColumns.addAll(targetTable.getNativePartCols()); + allTargetTableColumns.addAll(targetTable.getPartCols()); this.targetTableNameInSourceQuery = unescapeIdentifier(targetTableNameInSourceQuery); this.conf = conf; this.onClauseAsString = onClauseAsString; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java index 9b48ce03456e..ee27962aae1c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java @@ -691,7 +691,7 @@ public static Map> getFullPartitionSpecs( */ private static int calculatePartPrefix(Table tbl, Set partSpecKeys) { int partPrefixToDrop = 0; - for (FieldSchema fs : tbl.getNativePartCols()) { + for (FieldSchema fs : tbl.getPartCols()) { if (!partSpecKeys.contains(fs.getName())) { break; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/RewriteSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/RewriteSemanticAnalyzer.java index fb118ba8ba5b..101f6b1fc3d8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/RewriteSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/RewriteSemanticAnalyzer.java @@ -119,7 +119,7 @@ protected void checkValidSetClauseTarget(ASTNode colName, Table targetTable) thr String columnName = normalizeColName(colName.getText()); // Make sure this isn't one of the partitioning columns, that's not supported. - for (FieldSchema fschema : targetTable.getNativePartCols()) { + for (FieldSchema fschema : targetTable.getPartCols()) { if (fschema.getName().equalsIgnoreCase(columnName)) { throw new SemanticException(ErrorMsg.UPDATE_CANNOT_UPDATE_PART_VALUE.getMsg()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 92488161d5f8..5c4f049f0350 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -2209,7 +2209,7 @@ private void handleInsertStatementSpecPhase1(ASTNode ast, QBParseInfo qbp, Phase } } else { // partition spec is not specified but column schema can have partitions specified - for(FieldSchema f : targetTable.getNativePartCols()) { + for(FieldSchema f : targetTable.getPartCols()) { //parser only allows foo(a,b), not foo(foo.a, foo.b) targetColumns.remove(f.getName()); } @@ -12038,7 +12038,7 @@ private Operator genTablePlan(String alias, QB qb) throws SemanticException { } // Hack!! - refactor once the metadata APIs with types are ready // Finally add the partitioning columns - for (FieldSchema part_col : tab.getNativePartCols()) { + for (FieldSchema part_col : tab.getPartCols()) { LOG.trace("Adding partition col: " + part_col); rwsch.put(alias, part_col.getName(), new ColumnInfo(part_col.getName(), TypeInfoFactory.getPrimitiveTypeInfo(part_col.getType()), alias, true)); @@ -12304,7 +12304,7 @@ private void setupStats(TableScanDesc tsDesc, QBParseInfo qbp, Table tab, String if (tab.isPartitioned() && !tab.hasNonNativePartitionSupport()) { List cols = new ArrayList(); if (qbp.getAnalyzeRewrite() != null) { - List partitionCols = tab.getNativePartCols(); + List partitionCols = tab.getPartCols(); for (FieldSchema fs : partitionCols) { cols.add(fs.getName()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/MergeRewriter.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/MergeRewriter.java index 0b3b4e504d70..c436e85a4eb6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/MergeRewriter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/MergeRewriter.java @@ -250,7 +250,7 @@ protected void addValues(Table targetTable, String targetAlias, Map values.add( + targetTable.getPartCols().forEach(fieldSchema -> values.add( formatter.apply(fieldSchema.getName()))); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitUpdateRewriter.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitUpdateRewriter.java index 7e93e9a024b3..d14ddc7eb485 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitUpdateRewriter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitUpdateRewriter.java @@ -98,8 +98,8 @@ public ParseUtils.ReparseResult rewrite(Context context, UpdateStatement updateB insertValues.add(sqlGenerator.qualify(identifier)); } - if (updateBlock.getTargetTable().getNativePartCols() != null) { - updateBlock.getTargetTable().getNativePartCols().forEach( + if (updateBlock.getTargetTable().getPartCols() != null) { + updateBlock.getTargetTable().getPartCols().forEach( fieldSchema -> insertValues.add(sqlGenerator.qualify(HiveUtils.unparseIdentifier(fieldSchema.getName(), conf)))); } addRowLineageColumnsForUpdate(updateBlock.getTargetTable(), sqlGenerator, insertValues, conf); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/MultiInsertSqlGenerator.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/MultiInsertSqlGenerator.java index 2bf818c1fd69..7587daf13055 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/MultiInsertSqlGenerator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/MultiInsertSqlGenerator.java @@ -111,7 +111,7 @@ public void appendPartitionColsOfTarget() { */ public void appendPartitionCols(Table table) { // If the table is partitioned we have to put the partition() clause in - List partCols = table.getNativePartCols(); + List partCols = table.getPartCols(); if (partCols == null || partCols.isEmpty()) { return; } @@ -148,11 +148,11 @@ public void removeLastChar() { } public void appendPartColsOfTargetTableWithComma(String alias) { - if (targetTable.getNativePartCols() == null || targetTable.getNativePartCols().isEmpty()) { + if (targetTable.getPartCols() == null || targetTable.getPartCols().isEmpty()) { return; } queryStr.append(','); - appendCols(targetTable.getNativePartCols(), alias, null, FieldSchema::getName); + appendCols(targetTable.getPartCols(), alias, null, FieldSchema::getName); } public void appendAllColsOfTargetTable(String prefix) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/NativeAcidMultiInsertSqlGenerator.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/NativeAcidMultiInsertSqlGenerator.java index c45ef18bdb96..87e426800442 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/NativeAcidMultiInsertSqlGenerator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/NativeAcidMultiInsertSqlGenerator.java @@ -36,7 +36,7 @@ public NativeAcidMultiInsertSqlGenerator(Table table, String targetTableFullName @Override public void appendAcidSelectColumns(Operation operation) { queryStr.append("ROW__ID,"); - for (FieldSchema fieldSchema : targetTable.getNativePartCols()) { + for (FieldSchema fieldSchema : targetTable.getPartCols()) { String identifier = HiveUtils.unparseIdentifier(fieldSchema.getName(), this.conf); queryStr.append(identifier); queryStr.append(","); @@ -45,9 +45,9 @@ public void appendAcidSelectColumns(Operation operation) { @Override public List getDeleteValues(Operation operation) { - List deleteValues = new ArrayList<>(1 + targetTable.getNativePartCols().size()); + List deleteValues = new ArrayList<>(1 + targetTable.getPartCols().size()); deleteValues.add(qualify("ROW__ID")); - for (FieldSchema fieldSchema : targetTable.getNativePartCols()) { + for (FieldSchema fieldSchema : targetTable.getPartCols()) { deleteValues.add(qualify(HiveUtils.unparseIdentifier(fieldSchema.getName(), conf))); } return deleteValues; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java index 6c005bf1af61..be62d94019ed 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java @@ -156,7 +156,7 @@ private boolean constructColumnStatsFromPackedRows(Table tbl, List