HIVE-29559: materializeCTE to query Analyzer from DDLSemanticAnalyzerFactory

ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java

@@ -31,7 +31,6 @@
 import java.util.Optional;
 import java.util.function.Function;
 import java.util.regex.Pattern;
-import org.antlr.runtime.ClassicToken;
 import org.antlr.runtime.CommonToken;
 import org.antlr.runtime.tree.Tree;
 import org.antlr.runtime.tree.TreeVisitor;
@@ -144,7 +143,6 @@
 import org.apache.hadoop.hive.ql.ErrorMsg;
 import org.apache.hadoop.hive.ql.QueryProperties;
 import org.apache.hadoop.hive.ql.QueryState;
-import org.apache.hadoop.hive.ql.ddl.table.create.CreateTableAnalyzer;
 import org.apache.hadoop.hive.ql.exec.ColumnInfo;
 import org.apache.hadoop.hive.ql.exec.FunctionInfo;
 import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
@@ -319,7 +317,6 @@
 import org.apache.hadoop.hive.ql.parse.type.TypeCheckProcFactory;
 import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
 import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
-import org.apache.hadoop.hive.ql.plan.HiveOperation;
 import org.apache.hadoop.hive.ql.plan.SelectDesc;
 import org.apache.hadoop.hive.ql.plan.mapper.EmptyStatsSource;
 import org.apache.hadoop.hive.ql.plan.mapper.StatsSource;
@@ -338,7 +335,6 @@
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
 import org.joda.time.Interval;
-import java.io.IOException;
 import java.lang.reflect.Field;
 import java.lang.reflect.InvocationTargetException;
 import java.math.BigDecimal;
@@ -1051,51 +1047,6 @@ boolean continueJoinMerge() {
     return !(runCBO && disableSemJoinReordering);
   }
 
-  @Override
-  Table materializeCTE(String cteName, CTEClause cte) throws HiveException {
-
-    ASTNode createTable = new ASTNode(new ClassicToken(HiveParser.TOK_CREATETABLE));
-
-    ASTNode tableName = new ASTNode(new ClassicToken(HiveParser.TOK_TABNAME));
-    tableName.addChild(new ASTNode(new ClassicToken(HiveParser.Identifier, cteName)));
-
-    ASTNode temporary = new ASTNode(new ClassicToken(HiveParser.KW_TEMPORARY, MATERIALIZATION_MARKER));
-
-    createTable.addChild(tableName);
-    createTable.addChild(temporary);
-    createTable.addChild(cte.cteNode);
-
-    CreateTableAnalyzer analyzer = new CreateTableAnalyzer(queryState);
-    analyzer.initCtx(ctx);
-    analyzer.init(false);
-
-    // should share cte contexts
-    analyzer.aliasToCTEs.putAll(aliasToCTEs);
-
-    HiveOperation operation = queryState.getHiveOperation();
-    try {
-      analyzer.analyzeInternal(createTable);
-    } finally {
-      queryState.setCommandType(operation);
-    }
-
-    Table table = analyzer.tableDesc.toTable(conf);
-    Path location = table.getDataLocation();
-    try {
-      location.getFileSystem(conf).mkdirs(location);
-    } catch (IOException e) {
-      throw new HiveException(e);
-    }
-    table.setMaterializedTable(true);
-
-    LOG.info(cteName + " will be materialized into " + location);
-    cte.source = analyzer;
-
-    ctx.addMaterializedTable(cteName, table, getMaterializedTableStats(analyzer.getSinkOp()));
-
-    return table;
-  }
-
   @Override
   String fixCtasColumnName(String colName) {
     if (runCBO) {

ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java

@@ -117,6 +117,7 @@
 import org.apache.hadoop.hive.ql.ddl.DDLDescWithTableProperties;
 import org.apache.hadoop.hive.ql.ddl.DDLWork;
 import org.apache.hadoop.hive.ql.ddl.misc.hooks.InsertCommitHookDesc;
+import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory;
 import org.apache.hadoop.hive.ql.ddl.table.create.CreateTableDesc;
 import org.apache.hadoop.hive.ql.ddl.table.misc.preinsert.PreInsertTableDesc;
 import org.apache.hadoop.hive.ql.ddl.table.misc.properties.AlterTableUnsetPropertiesDesc;
@@ -1568,7 +1569,7 @@ Table materializeCTE(String cteName, CTEClause cte) throws HiveException {
     createTable.addChild(temporary);
     createTable.addChild(cte.cteNode);
 
-    SemanticAnalyzer analyzer = new SemanticAnalyzer(queryState);
+    SemanticAnalyzer analyzer = (SemanticAnalyzer) DDLSemanticAnalyzerFactory.getAnalyzer(createTable, queryState);
     analyzer.initCtx(ctx);
     analyzer.init(false);
 

ql/src/test/org/apache/hadoop/hive/ql/parse/TestSemanticAnalyzer.java

@@ -20,12 +20,15 @@
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.CALLS_REAL_METHODS;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.mockStatic;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.when;
 
@@ -46,6 +49,8 @@
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
 import org.apache.hadoop.hive.ql.Context;
+import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory;
+import org.apache.hadoop.hive.ql.ddl.table.create.CreateTableAnalyzer;
 import org.apache.hadoop.hive.ql.QueryProperties;
 import org.apache.hadoop.hive.ql.QueryProperties.QueryType;
 import org.apache.hadoop.hive.ql.QueryState;
@@ -65,6 +70,7 @@
 import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.mockito.MockedStatic;
 import org.mockito.stubbing.Answer;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -493,4 +499,51 @@ private void checkTablesUsed(String query, Set<String> tables) throws Exception
 
     Assert.assertEquals(new TreeSet<>(tables), new TreeSet<>(result));
   }
+
+  @Test
+  public void testMaterializeCTEWithCBODisabled() throws Exception {
+    testMaterializeCTEUsesDDLFactory(false);
+  }
+
+  @Test
+  public void testMaterializeCTEWithCBOEnabled() throws Exception {
+    testMaterializeCTEUsesDDLFactory(true);
+  }
+
+  private void testMaterializeCTEUsesDDLFactory(boolean cboEnabled) throws Exception {
+    HiveConf testConf = new HiveConf(conf);
+    testConf.setBoolVar(HiveConf.ConfVars.HIVE_CBO_ENABLED, cboEnabled);
+
+    SessionState.start(testConf);
+    Context ctx = new Context(testConf);
+
+    // Reference CTE 3 times to exceed default materialization threshold of 2
+    String query = "WITH cte AS (SELECT COUNT(*) as cnt FROM table1) " +
+                   "SELECT * FROM cte UNION ALL SELECT * FROM cte UNION ALL SELECT * FROM cte";
+
+    ASTNode astNode = ParseUtils.parse(query, ctx);
+    QueryState queryState = new QueryState.Builder().withHiveConf(testConf).build();
+    BaseSemanticAnalyzer analyzer = SemanticAnalyzerFactory.get(queryState, astNode);
+    analyzer.initCtx(ctx);
+
+    try (MockedStatic<DDLSemanticAnalyzerFactory> mocked =
+            mockStatic(DDLSemanticAnalyzerFactory.class, CALLS_REAL_METHODS)) {
+      BaseSemanticAnalyzer[] cteAnalyzer = new BaseSemanticAnalyzer[1];
+
+      mocked.when(() -> DDLSemanticAnalyzerFactory.getAnalyzer(any(ASTNode.class), any(QueryState.class)))
+          .thenAnswer(invocation -> {
+            BaseSemanticAnalyzer result = (BaseSemanticAnalyzer) invocation.callRealMethod();
+            if (invocation.getArgument(0, ASTNode.class).getType() == HiveParser.TOK_CREATETABLE) {
+              cteAnalyzer[0] = result;
+            }
+            return result;
+          });
+
+      analyzer.analyze(astNode, ctx);
+
+      assertNotNull("DDLSemanticAnalyzerFactory should be called for CTE materialization", cteAnalyzer[0]);
+      assertTrue("CTE materialization should use CreateTableAnalyzer",
+          cteAnalyzer[0] instanceof CreateTableAnalyzer);
+    }
+  }
 }

ql/src/test/queries/clientpositive/cte_materialize.q

@@ -0,0 +1,27 @@
+-- Test CTE materialization with both CBO enabled and disabled
+-- Verifies DDLSemanticAnalyzerFactory is used for CTE materialization
+-- Also ensures that an NPE is no longer triggered with CBO off (HIVE-28724 regression)
+
+-- Test with CBO enabled (default)
+explain
+WITH cte AS (
+    SELECT COUNT(*) as cnt FROM (SELECT 1 as id) t
+)
+SELECT * FROM cte
+UNION ALL
+SELECT * FROM cte
+UNION ALL
+SELECT * FROM cte;
+
+-- Test with CBO disabled
+set hive.cbo.enable=false;
+
+explain
+WITH cte AS (
+    SELECT COUNT(*) as cnt FROM (SELECT 1 as id) t
+)
+SELECT * FROM cte
+UNION ALL
+SELECT * FROM cte
+UNION ALL
+SELECT * FROM cte;