feat: Enable CTEs to reference temp tables

Temp tables (#tmp) are now first-class citizens that can be referenced
within CTEs, just like regular tables or other CTEs. This enables:

- Filtering temp tables in CTEs with WHERE clauses
- Window functions (PARTITION BY, ROW_NUMBER) over temp tables
- Nested CTEs that reference temp tables
- Complex multi-stage transformations using temp tables

Implementation:
- Added execute_with_temp_tables() to QueryEngine
- Temp tables are merged into CTE context at query start
- Threaded TempTableRegistry through QueryExecutionService
- Updated non_interactive.rs to pass temp registry to service

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

Author: TimelordUK

src/data/query_engine.rs

@@ -17,6 +17,7 @@ use crate::data::hash_join::HashJoinExecutor;
 use crate::data::recursive_where_evaluator::RecursiveWhereEvaluator;
 use crate::data::row_expanders::RowExpanderRegistry;
 use crate::data::subquery_executor::SubqueryExecutor;
+use crate::data::temp_table_registry::TempTableRegistry;
 use crate::execution_plan::{ExecutionPlan, ExecutionPlanBuilder, StepType};
 use crate::sql::aggregates::{contains_aggregate, is_aggregate_compatible};
 use crate::sql::parser::ast::ColumnRef;
@@ -216,14 +217,47 @@ impl QueryEngine {
         Ok(view)
     }
 
+    /// Execute a SQL query with optional temp table registry access
+    pub fn execute_with_temp_tables(
+        &self,
+        table: Arc<DataTable>,
+        sql: &str,
+        temp_tables: Option<&TempTableRegistry>,
+    ) -> Result<DataView> {
+        let (view, _plan) = self.execute_with_plan_and_temp_tables(table, sql, temp_tables)?;
+        Ok(view)
+    }
+
     /// Execute a parsed SelectStatement on a `DataTable` and return a `DataView`
     pub fn execute_statement(
         &self,
         table: Arc<DataTable>,
         statement: SelectStatement,
+    ) -> Result<DataView> {
+        self.execute_statement_with_temp_tables(table, statement, None)
+    }
+
+    /// Execute a parsed SelectStatement with optional temp table access
+    pub fn execute_statement_with_temp_tables(
+        &self,
+        table: Arc<DataTable>,
+        statement: SelectStatement,
+        temp_tables: Option<&TempTableRegistry>,
     ) -> Result<DataView> {
         // First process CTEs to build context
         let mut cte_context = HashMap::new();
+
+        // Add temp tables to CTE context if provided
+        if let Some(temp_registry) = temp_tables {
+            for table_name in temp_registry.list_tables() {
+                if let Some(temp_table) = temp_registry.get(&table_name) {
+                    debug!("Adding temp table {} to CTE context", table_name);
+                    let view = DataView::new(temp_table);
+                    cte_context.insert(table_name, Arc::new(view));
+                }
+            }
+        }
+
         for cte in &statement.ctes {
             debug!("QueryEngine: Pre-processing CTE '{}'...", cte.name);
             // Execute the CTE based on its type
@@ -349,6 +383,16 @@ impl QueryEngine {
         &self,
         table: Arc<DataTable>,
         sql: &str,
+    ) -> Result<(DataView, ExecutionPlan)> {
+        self.execute_with_plan_and_temp_tables(table, sql, None)
+    }
+
+    /// Execute a query with temp tables and return both the result and the execution plan
+    pub fn execute_with_plan_and_temp_tables(
+        &self,
+        table: Arc<DataTable>,
+        sql: &str,
+        temp_tables: Option<&TempTableRegistry>,
     ) -> Result<(DataView, ExecutionPlan)> {
         let mut plan_builder = ExecutionPlanBuilder::new();
         let start_time = Instant::now();
@@ -372,6 +416,17 @@ impl QueryEngine {
         // First process CTEs to build context
         let mut cte_context = HashMap::new();
 
+        // Add temp tables to CTE context if provided
+        if let Some(temp_registry) = temp_tables {
+            for table_name in temp_registry.list_tables() {
+                if let Some(temp_table) = temp_registry.get(&table_name) {
+                    debug!("Adding temp table {} to CTE context", table_name);
+                    let view = DataView::new(temp_table);
+                    cte_context.insert(table_name, Arc::new(view));
+                }
+            }
+        }
+
         if !statement.ctes.is_empty() {
             plan_builder.begin_step(
                 StepType::CTE,

src/non_interactive.rs

@@ -902,7 +902,12 @@ pub fn execute_script(config: NonInteractiveConfig) -> Result<()> {
 
         // Execute the statement
         let service = QueryExecutionService::new(config.case_insensitive, config.auto_hide_empty);
-        match service.execute(&executable_sql, Some(&dataview), None) {
+        match service.execute_with_temp_tables(
+            &executable_sql,
+            Some(&dataview),
+            None,
+            Some(&temp_tables),
+        ) {
             Ok(result) => {
                 let exec_time = stmt_start.elapsed().as_secs_f64() * 1000.0;
                 let final_view = result.dataview;

src/services/query_execution_service.rs

@@ -1,6 +1,7 @@
 use crate::config::config::BehaviorConfig;
 use crate::data::data_view::DataView;
 use crate::data::query_engine::QueryEngine;
+use crate::data::temp_table_registry::TempTableRegistry;
 use crate::debug_trace::{DebugContext, DebugLevel, QueryTrace, ScopedTimer};
 use crate::execution_plan::ExecutionPlan;
 use anyhow::Result;
@@ -94,13 +95,48 @@ impl QueryExecutionService {
         self.execute_with_debug(query, current_dataview, original_source, None)
     }
 
+    /// Execute a query with temp table support
+    pub fn execute_with_temp_tables(
+        &self,
+        query: &str,
+        current_dataview: Option<&DataView>,
+        original_source: Option<&crate::data::datatable::DataTable>,
+        temp_tables: Option<&TempTableRegistry>,
+    ) -> Result<QueryExecutionResult> {
+        self.execute_with_temp_tables_and_debug(
+            query,
+            current_dataview,
+            original_source,
+            temp_tables,
+            None,
+        )
+    }
+
     /// Execute a query with optional debug tracing
     pub fn execute_with_debug(
         &self,
         query: &str,
         current_dataview: Option<&DataView>,
         original_source: Option<&crate::data::datatable::DataTable>,
         debug_context: Option<DebugContext>,
+    ) -> Result<QueryExecutionResult> {
+        self.execute_with_temp_tables_and_debug(
+            query,
+            current_dataview,
+            original_source,
+            None,
+            debug_context,
+        )
+    }
+
+    /// Execute a query with temp tables and optional debug tracing
+    pub fn execute_with_temp_tables_and_debug(
+        &self,
+        query: &str,
+        current_dataview: Option<&DataView>,
+        original_source: Option<&crate::data::datatable::DataTable>,
+        temp_tables: Option<&TempTableRegistry>,
+        debug_context: Option<DebugContext>,
     ) -> Result<QueryExecutionResult> {
         // Check if debug context is enabled before moving it
         let debug_enabled = debug_context.is_some();
@@ -243,7 +279,11 @@ impl QueryExecutionService {
                 self.date_notation.clone(),
             )
         };
-        let mut new_dataview = engine.execute(table_arc, query)?;
+        let mut new_dataview = if temp_tables.is_some() {
+            engine.execute_with_temp_tables(table_arc, query, temp_tables)?
+        } else {
+            engine.execute(table_arc, query)?
+        };
         let query_engine_time = query_start.elapsed();
 
         trace.log_timed(

tests/sql_examples/test_cte_temp_tables.sql

@@ -0,0 +1,63 @@
+-- Test: CTEs can now reference temp tables
+-- This demonstrates that temp tables are now first-class citizens
+-- that can be used in CTEs just like regular tables
+
+-- Example 1: Simple CTE referencing temp table
+WITH data AS (
+    SELECT value as id, value * 10 as amount
+    FROM RANGE(1, 5)
+)
+SELECT * FROM data INTO #sample_data;
+GO
+
+WITH filtered AS (
+    SELECT * FROM #sample_data WHERE amount > 20
+)
+SELECT * FROM filtered ORDER BY id;
+GO
+
+-- Example 2: Window functions in CTEs over temp tables
+WITH partitioned AS (
+    SELECT
+        id,
+        amount,
+        ROW_NUMBER() OVER (ORDER BY amount DESC) as rank
+    FROM #sample_data
+)
+SELECT * FROM partitioned ORDER BY rank;
+GO
+
+-- Example 3: Multiple CTEs with temp table reference
+WITH doubled AS (
+    SELECT id, amount * 2 as doubled FROM #sample_data
+),
+tripled AS (
+    SELECT id, amount * 3 as tripled FROM #sample_data
+)
+SELECT
+    doubled.id,
+    doubled.doubled,
+    tripled.tripled
+FROM doubled
+JOIN tripled ON doubled.id = tripled.id
+ORDER BY doubled.id;
+GO
+
+-- Example 4: Temp table with string filtering
+WITH strings AS (
+    SELECT
+        value as id,
+        CASE
+            WHEN value % 2 = 0 THEN 'even.type'
+            ELSE 'odd.type'
+        END as category
+    FROM RANGE(1, 6)
+)
+SELECT * FROM strings INTO #categories;
+GO
+
+WITH filtered_cats AS (
+    SELECT * FROM #categories WHERE category.Contains('.')
+)
+SELECT * FROM filtered_cats ORDER BY id;
+GO