From 6c955af57474f29c5ef458b2668b7010f8b6d35b Mon Sep 17 00:00:00 2001 From: Alessandro Solimando Date: Wed, 29 Jul 2026 14:59:49 +0200 Subject: [PATCH 1/2] fix(physical-plan): preserve an exact zero when capping null_count and distinct_count at num_rows `cap_at_rows` demoted `null_count` and `distinct_count` to inexact unconditionally, so an exactly empty input came out of `FilterExec` with `Exact(0)` for `num_rows`, `total_byte_size` and `byte_size` but `Inexact(0)` for the two counts. --- datafusion/physical-plan/src/filter.rs | 43 ++++++++++++++++++++------ 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs index 511be0bbdd9e2..5ee10735cc1ed 100644 --- a/datafusion/physical-plan/src/filter.rs +++ b/datafusion/physical-plan/src/filter.rs @@ -1032,15 +1032,17 @@ fn interval_bound_to_precision( } /// Caps a row-bounded column statistic (a null count or distinct count) at the -/// filtered row estimate, since a column cannot have more nulls or distinct -/// values than it has rows. Known counts are demoted to inexact because the -/// filtered row count is itself an estimate. +/// filtered row count, since a column cannot have more nulls or distinct values +/// than it has rows. Known counts are demoted to inexact because a +/// filter-derived row bound is normally an estimate, the exception being an +/// exact zero, which proves the column is empty. fn cap_at_rows( value: Precision, filtered_num_rows: Precision, ) -> Precision { match filtered_num_rows { Precision::Absent => value.to_inexact(), + Precision::Exact(0) => Precision::Exact(0), rows => value.to_inexact().min(&rows), } } @@ -2904,16 +2906,29 @@ mod tests { #[tokio::test] async fn test_filter_statistics_preserves_exactly_empty_input() -> Result<()> { // A satisfiable predicate over an exactly empty input: the filter cannot - // produce rows, so the whole estimate stays exact. - let schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]); + // produce rows, so the whole estimate stays exact. Column `b` is not + // mentioned by the predicate, so its null and distinct counts go through + // the generic row cap. + let schema = Schema::new(vec![ + Field::new("a", DataType::Int32, true), + Field::new("b", DataType::Int32, true), + ]); let input_stats = Statistics { num_rows: Precision::Exact(0), total_byte_size: Precision::Exact(0), - column_statistics: vec![ColumnStatistics { - null_count: Precision::Exact(0), - byte_size: Precision::Exact(0), - ..Default::default() - }], + column_statistics: vec![ + ColumnStatistics { + null_count: Precision::Exact(0), + byte_size: Precision::Exact(0), + ..Default::default() + }, + ColumnStatistics { + null_count: Precision::Exact(3), + distinct_count: Precision::Exact(7), + byte_size: Precision::Exact(0), + ..Default::default() + }, + ], }; let predicate = Arc::new(BinaryExpr::new( Arc::new(Column::new("a", 0)), @@ -2933,6 +2948,14 @@ mod tests { statistics.column_statistics[0].byte_size, Precision::Exact(0) ); + assert_eq!( + statistics.column_statistics[1].null_count, + Precision::Exact(0) + ); + assert_eq!( + statistics.column_statistics[1].distinct_count, + Precision::Exact(0) + ); Ok(()) } From 7a0b6bb4f0c6e60d5403f21d694f7b54fdd4b690 Mon Sep 17 00:00:00 2001 From: Alessandro Solimando Date: Wed, 29 Jul 2026 16:21:06 +0200 Subject: [PATCH 2/2] fix(physical-plan): report an exact zero total_byte_size for a contradictory predicate A contradictory predicate already yielded `Exact(0)` for `num_rows` and per-column `byte_size`, but `total_byte_size` went through the selectivity estimate and came out inexact for any non-empty input. --- datafusion/physical-plan/src/filter.rs | 40 ++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs index 5ee10735cc1ed..f627f4917d08c 100644 --- a/datafusion/physical-plan/src/filter.rs +++ b/datafusion/physical-plan/src/filter.rs @@ -401,8 +401,11 @@ impl FilterExec { } }; - let total_byte_size = - input_total_byte_size.with_estimated_selectivity(selectivity); + let total_byte_size = if is_infeasible { + Precision::Exact(0) + } else { + input_total_byte_size.with_estimated_selectivity(selectivity) + }; Ok(Statistics { num_rows, @@ -2936,7 +2939,7 @@ mod tests { Arc::new(Literal::new(ScalarValue::Int32(Some(5)))), )); - let input = Arc::new(StatisticsExec::new(input_stats, schema)); + let input = Arc::new(StatisticsExec::new(input_stats, schema.clone())); let filter: Arc = Arc::new(FilterExec::try_new(predicate, input)?); let statistics = @@ -2957,6 +2960,37 @@ mod tests { Precision::Exact(0) ); + // A contradictory predicate (`a = 1 AND a = 2`) proves the same for an + // input of any size, so the byte size is exact too. + let input = Arc::new(StatisticsExec::new( + Statistics { + num_rows: Precision::Inexact(1000), + total_byte_size: Precision::Inexact(8000), + column_statistics: vec![ColumnStatistics::new_unknown(); 2], + }, + schema, + )); + let contradiction = Arc::new(BinaryExpr::new( + Arc::new(BinaryExpr::new( + Arc::new(Column::new("a", 0)), + Operator::Eq, + Arc::new(Literal::new(ScalarValue::Int32(Some(1)))), + )), + Operator::And, + Arc::new(BinaryExpr::new( + Arc::new(Column::new("a", 0)), + Operator::Eq, + Arc::new(Literal::new(ScalarValue::Int32(Some(2)))), + )), + )); + let filter: Arc = + Arc::new(FilterExec::try_new(contradiction, input)?); + let statistics = + StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?; + + assert_eq!(statistics.num_rows, Precision::Exact(0)); + assert_eq!(statistics.total_byte_size, Precision::Exact(0)); + Ok(()) }