From 217d0da1c8f3ad4e3c3929fb77bf064199c4e697 Mon Sep 17 00:00:00 2001 From: Michael Kleen Date: Wed, 1 Jul 2026 12:26:04 +0200 Subject: [PATCH 1/5] fix: Avoid panicing when stats are not available for a group split --- .../datasource/src/file_scan_config/mod.rs | 51 +++++++++++++++++++ datafusion/datasource/src/statistics.rs | 9 +++- 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/datafusion/datasource/src/file_scan_config/mod.rs b/datafusion/datasource/src/file_scan_config/mod.rs index b73d100e056f4..8d0a57b7e320b 100644 --- a/datafusion/datasource/src/file_scan_config/mod.rs +++ b/datafusion/datasource/src/file_scan_config/mod.rs @@ -1944,6 +1944,57 @@ mod tests { } } + #[test] + fn test_split_groups_by_statistics_missing_stats_no_partition_values() { + use arrow::compute::SortOptions; + + let table_schema = Arc::new(Schema::new(vec![ + Field::new("a", DataType::Float64, false), + Field::new("b", DataType::Float64, false), + ])); + + // Two files in a single group. Each file has statistics for only the + // first column "a" and no partition values and the statistics + // are missing for the second column "b" + let make_file = |name: &str, min: f64, max: f64| { + PartitionedFile::new(name.to_string(), 1024).with_statistics(Arc::new( + Statistics { + num_rows: Precision::Exact(100), + total_byte_size: Precision::Exact(1024), + column_statistics: vec![ColumnStatistics { + min_value: Precision::Exact(ScalarValue::Float64(Some(min))), + max_value: Precision::Exact(ScalarValue::Float64(Some(max))), + ..Default::default() + }], + }, + )) + }; + + let file_group = FileGroup::new(vec![ + make_file("file1.parquet", 0.0, 1.0), + make_file("file2.parquet", 2.0, 3.0), + ]); + + // Sort by column "b" with index 1, which has no statistics + let sort_order = LexOrdering::new(vec![PhysicalSortExpr::new( + Arc::new(Column::new("b", 1)), + SortOptions::default(), + )]) + .unwrap(); + + // Statistics are unavailable for this column, must not panic + let result = FileScanConfig::split_groups_by_statistics( + &table_schema, + std::slice::from_ref(&file_group), + &sort_order, + ); + + assert!( + result.is_err(), + "expected an error because statistics are missing, got: {result:?}" + ); + } + // sets default for configs that play no role in projections fn config_for_projection( file_schema: SchemaRef, diff --git a/datafusion/datasource/src/statistics.rs b/datafusion/datasource/src/statistics.rs index 781cf3dbced94..fd42dcef17304 100644 --- a/datafusion/datasource/src/statistics.rs +++ b/datafusion/datasource/src/statistics.rs @@ -97,8 +97,13 @@ impl MinMaxStatistics { .zip(s.column_statistics[i].max_value.get_value().cloned()) .ok_or_else(|| plan_datafusion_err!("statistics not found")) } else { - let partition_value = &pv[i - s.column_statistics.len()]; - Ok((partition_value.clone(), partition_value.clone())) + let partition_value_index = i - s.column_statistics.len(); + if pv.len() > partition_value_index { + let partition_value = &pv[partition_value_index]; + Ok((partition_value.clone(), partition_value.clone())) + } else { + Err(plan_datafusion_err!("statistics not found")) + } } }) .collect::>>()? From b065df4893dad4795073ec2513afe6bcff568fdb Mon Sep 17 00:00:00 2001 From: Michael Kleen Date: Fri, 3 Jul 2026 12:38:51 +0200 Subject: [PATCH 2/5] Clean-up --- datafusion/datasource/src/statistics.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/datafusion/datasource/src/statistics.rs b/datafusion/datasource/src/statistics.rs index fd42dcef17304..7b87b1e3162f0 100644 --- a/datafusion/datasource/src/statistics.rs +++ b/datafusion/datasource/src/statistics.rs @@ -97,12 +97,12 @@ impl MinMaxStatistics { .zip(s.column_statistics[i].max_value.get_value().cloned()) .ok_or_else(|| plan_datafusion_err!("statistics not found")) } else { - let partition_value_index = i - s.column_statistics.len(); - if pv.len() > partition_value_index { - let partition_value = &pv[partition_value_index]; + if let Some(partition_value) = + pv.get(i - s.column_statistics.len()) + { Ok((partition_value.clone(), partition_value.clone())) } else { - Err(plan_datafusion_err!("statistics not found")) + return Err(plan_datafusion_err!("statistics not found")); } } }) From 30aee46c07658e4f84a1fb6b9981fa4698aa8fd6 Mon Sep 17 00:00:00 2001 From: Michael Kleen Date: Fri, 3 Jul 2026 13:01:16 +0200 Subject: [PATCH 3/5] Move test to statistics --- .../datasource/src/file_scan_config/mod.rs | 51 ------------------- datafusion/datasource/src/statistics.rs | 26 ++++++++++ 2 files changed, 26 insertions(+), 51 deletions(-) diff --git a/datafusion/datasource/src/file_scan_config/mod.rs b/datafusion/datasource/src/file_scan_config/mod.rs index 8d0a57b7e320b..b73d100e056f4 100644 --- a/datafusion/datasource/src/file_scan_config/mod.rs +++ b/datafusion/datasource/src/file_scan_config/mod.rs @@ -1944,57 +1944,6 @@ mod tests { } } - #[test] - fn test_split_groups_by_statistics_missing_stats_no_partition_values() { - use arrow::compute::SortOptions; - - let table_schema = Arc::new(Schema::new(vec![ - Field::new("a", DataType::Float64, false), - Field::new("b", DataType::Float64, false), - ])); - - // Two files in a single group. Each file has statistics for only the - // first column "a" and no partition values and the statistics - // are missing for the second column "b" - let make_file = |name: &str, min: f64, max: f64| { - PartitionedFile::new(name.to_string(), 1024).with_statistics(Arc::new( - Statistics { - num_rows: Precision::Exact(100), - total_byte_size: Precision::Exact(1024), - column_statistics: vec![ColumnStatistics { - min_value: Precision::Exact(ScalarValue::Float64(Some(min))), - max_value: Precision::Exact(ScalarValue::Float64(Some(max))), - ..Default::default() - }], - }, - )) - }; - - let file_group = FileGroup::new(vec![ - make_file("file1.parquet", 0.0, 1.0), - make_file("file2.parquet", 2.0, 3.0), - ]); - - // Sort by column "b" with index 1, which has no statistics - let sort_order = LexOrdering::new(vec![PhysicalSortExpr::new( - Arc::new(Column::new("b", 1)), - SortOptions::default(), - )]) - .unwrap(); - - // Statistics are unavailable for this column, must not panic - let result = FileScanConfig::split_groups_by_statistics( - &table_schema, - std::slice::from_ref(&file_group), - &sort_order, - ); - - assert!( - result.is_err(), - "expected an error because statistics are missing, got: {result:?}" - ); - } - // sets default for configs that play no role in projections fn config_for_projection( file_schema: SchemaRef, diff --git a/datafusion/datasource/src/statistics.rs b/datafusion/datasource/src/statistics.rs index 7b87b1e3162f0..8b85e89c58934 100644 --- a/datafusion/datasource/src/statistics.rs +++ b/datafusion/datasource/src/statistics.rs @@ -887,4 +887,30 @@ mod tests { Ok(()) } + + #[test] + fn min_max_statistics_missing_column_stats_returns_error() { + let schema = test_schema(); + let sort_order = + [PhysicalSortExpr::new_default(Arc::new(Column::new("a", 0)))].into(); + let files = [ + file_with_stats("f1.parquet", Statistics::default()), + file_with_stats("f2.parquet", Statistics::default()), + ]; + + let err = match MinMaxStatistics::new_from_files( + &sort_order, + &schema, + None, + files.iter(), + ) { + Ok(_) => panic!("expected missing statistics error"), + Err(err) => err, + }; + + assert!( + err.to_string().contains("statistics not found"), + "unexpected error: {err:?}" + ); + } } From 52971d803b7c8d6d2eb28f553e8cbb62601b8d66 Mon Sep 17 00:00:00 2001 From: Michael Kleen Date: Fri, 3 Jul 2026 14:27:02 +0200 Subject: [PATCH 4/5] fix clippy --- datafusion/datasource/src/statistics.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/datasource/src/statistics.rs b/datafusion/datasource/src/statistics.rs index 8b85e89c58934..0eff19910c282 100644 --- a/datafusion/datasource/src/statistics.rs +++ b/datafusion/datasource/src/statistics.rs @@ -102,7 +102,7 @@ impl MinMaxStatistics { { Ok((partition_value.clone(), partition_value.clone())) } else { - return Err(plan_datafusion_err!("statistics not found")); + Err(plan_datafusion_err!("statistics not found")) } } }) From 279342b4e79a21cd0c02143daf76d4dfa1d46bfb Mon Sep 17 00:00:00 2001 From: Michael Kleen Date: Fri, 3 Jul 2026 16:12:35 +0200 Subject: [PATCH 5/5] Improve error message --- datafusion/datasource/src/statistics.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/datafusion/datasource/src/statistics.rs b/datafusion/datasource/src/statistics.rs index 0eff19910c282..491b4e54e8993 100644 --- a/datafusion/datasource/src/statistics.rs +++ b/datafusion/datasource/src/statistics.rs @@ -102,7 +102,10 @@ impl MinMaxStatistics { { Ok((partition_value.clone(), partition_value.clone())) } else { - Err(plan_datafusion_err!("statistics not found")) + Err(plan_datafusion_err!( + "statistics not found for partition, expected at most {}", + s.column_statistics.len() + )) } } }) @@ -909,7 +912,8 @@ mod tests { }; assert!( - err.to_string().contains("statistics not found"), + err.to_string() + .contains("statistics not found for partition"), "unexpected error: {err:?}" ); }