From 26928b9af67698a472ee247bc811b72fe2ec6dba Mon Sep 17 00:00:00 2001 From: buraksenn Date: Fri, 3 Jul 2026 16:39:02 +0300 Subject: [PATCH 1/2] fix: return execution error instead of capacity overflow panic in array_resize --- datafusion/functions-nested/src/resize.rs | 69 ++++++++++++++++++- .../test_files/array/array_resize.slt | 12 ++++ 2 files changed, 78 insertions(+), 3 deletions(-) diff --git a/datafusion/functions-nested/src/resize.rs b/datafusion/functions-nested/src/resize.rs index d11064bf7efd6..5f2066c0288b6 100644 --- a/datafusion/functions-nested/src/resize.rs +++ b/datafusion/functions-nested/src/resize.rs @@ -219,6 +219,14 @@ fn general_list_resize>( } } + if output_values_len > max_resize_values(&data_type) + || O::from_usize(output_values_len).is_none() + { + return exec_err!( + "array_resize: resulting array of {output_values_len} elements exceeds the maximum array size" + ); + } + // The fast path is valid when at least one row grows and every row would // use the same fill value. let use_bulk_fill = max_extra > 0 @@ -342,12 +350,27 @@ where )?)) } +/// Largest element count whose eager value buffer stays within `isize::MAX` +/// bytes, so `array_resize` rejects oversized results instead of panicking. +/// Only primitive and `FixedSizeBinary` leaves are byte-exact. +fn max_resize_values(value_type: &DataType) -> usize { + let element_width = match value_type { + DataType::FixedSizeBinary(size) if *size > 0 => { + (*size as usize).saturating_mul(u8::BITS as usize) + } + _ => value_type.primitive_width().unwrap_or(size_of::()), + }; + (isize::MAX as usize) / element_width.max(1) +} + #[cfg(test)] mod tests { use super::array_resize_inner; - use arrow::array::{ArrayRef, AsArray, Int64Array, ListArray}; - use arrow::buffer::{NullBuffer, ScalarBuffer}; - use arrow::datatypes::Int32Type; + use arrow::array::{ + ArrayRef, AsArray, FixedSizeBinaryArray, Int64Array, LargeListArray, ListArray, + }; + use arrow::buffer::{NullBuffer, OffsetBuffer, ScalarBuffer}; + use arrow::datatypes::{DataType, Field, Int32Type, Int64Type}; use datafusion_common::Result; use std::sync::Arc; @@ -373,4 +396,44 @@ mod tests { Ok(()) } + + #[test] + fn test_array_resize_large_size_errors_without_panicking() { + let array: ArrayRef = + Arc::new(ListArray::from_iter_primitive::(vec![ + Some(vec![Some(1)]), + ])); + let size: ArrayRef = Arc::new(Int64Array::from(vec![i64::MAX])); + let fill: ArrayRef = Arc::new(Int64Array::from(vec![0])); + + let err = array_resize_inner(&[array, size, fill]).unwrap_err(); + assert!( + err.to_string().contains("exceeds the maximum array size"), + "unexpected error: {err}" + ); + } + + #[test] + fn test_array_resize_fixed_size_binary_large_size_errors_without_panicking() { + // FixedSizeBinary is byte-accounted separately from the generic fallback. + let values = + FixedSizeBinaryArray::try_from_iter(vec![vec![0u8; 32]].into_iter()).unwrap(); + let elem_field = + Arc::new(Field::new_list_field(DataType::FixedSizeBinary(32), true)); + let offsets = OffsetBuffer::::new(vec![0i64, 1].into()); + let array: ArrayRef = Arc::new(LargeListArray::new( + elem_field, + offsets, + Arc::new(values) as ArrayRef, + None, + )); + // Passes the width-16 bound (isize::MAX / 16) but overflows at width 32. + let size: ArrayRef = Arc::new(Int64Array::from(vec![400_000_000_000_000_000i64])); + + let err = array_resize_inner(&[array, size]).unwrap_err(); + assert!( + err.to_string().contains("exceeds the maximum array size"), + "unexpected error: {err}" + ); + } } diff --git a/datafusion/sqllogictest/test_files/array/array_resize.slt b/datafusion/sqllogictest/test_files/array/array_resize.slt index cb2ffa3a7e0be..aa915b4fd9439 100644 --- a/datafusion/sqllogictest/test_files/array/array_resize.slt +++ b/datafusion/sqllogictest/test_files/array/array_resize.slt @@ -64,6 +64,18 @@ select array_resize(arrow_cast(make_array(1, 2, 3), 'LargeList(Int64)'), 5, 4); query error select array_resize(make_array(1, 2, 3), -5, 2); +# array_resize with a very large size should error instead of panicking (capacity overflow) +query error DataFusion error: Execution error: array_resize: resulting array of 9223372036854775807 elements exceeds the maximum array size +select array_resize(make_array(1), 9223372036854775807, 0); + +query error DataFusion error: Execution error: array_resize: resulting array of 9223372036854775807 elements exceeds the maximum array size +select array_resize(arrow_cast(make_array(1), 'LargeList(Int64)'), 9223372036854775807, 0); + +# List size above i32::MAX must error via the offset-type guard instead of +# silently truncating the offsets or attempting a multi-GB allocation +query error DataFusion error: Execution error: array_resize: resulting array of 3000000000 elements exceeds the maximum array size +select array_resize(make_array(1), 3000000000, 0); + # array_resize scalar function #5 query ? select array_resize(make_array(1.1, 2.2, 3.3), 10, 9.9); From 6e3fedaf3deab9264e799f7060f220286ffec616 Mon Sep 17 00:00:00 2001 From: buraksenn Date: Sun, 5 Jul 2026 15:51:28 +0300 Subject: [PATCH 2/2] address review and add extra tests for coverage --- datafusion/functions-nested/src/resize.rs | 36 ++++++++++++++++--- .../test_files/array/array_resize.slt | 5 +++ 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/datafusion/functions-nested/src/resize.rs b/datafusion/functions-nested/src/resize.rs index 5f2066c0288b6..e4fd8421fe6c3 100644 --- a/datafusion/functions-nested/src/resize.rs +++ b/datafusion/functions-nested/src/resize.rs @@ -355,11 +355,10 @@ where /// Only primitive and `FixedSizeBinary` leaves are byte-exact. fn max_resize_values(value_type: &DataType) -> usize { let element_width = match value_type { - DataType::FixedSizeBinary(size) if *size > 0 => { - (*size as usize).saturating_mul(u8::BITS as usize) - } + DataType::FixedSizeBinary(size) if *size > 0 => *size as usize, _ => value_type.primitive_width().unwrap_or(size_of::()), }; + (isize::MAX as usize) / element_width.max(1) } @@ -415,7 +414,6 @@ mod tests { #[test] fn test_array_resize_fixed_size_binary_large_size_errors_without_panicking() { - // FixedSizeBinary is byte-accounted separately from the generic fallback. let values = FixedSizeBinaryArray::try_from_iter(vec![vec![0u8; 32]].into_iter()).unwrap(); let elem_field = @@ -436,4 +434,34 @@ mod tests { "unexpected error: {err}" ); } + + #[test] + fn test_array_resize_accumulates_values_across_rows() { + // Each row's target (6e17) is individually under the width-8 cap + // (isize::MAX / 8), but their sum (1.2e18) exceeds it, so the guard + // must reject based on the accumulated total rather than per row. + let values = Int64Array::from(vec![1, 2]); + let offsets = OffsetBuffer::::new(vec![0i64, 1, 2].into()); + let elem_field = Arc::new(Field::new_list_field(DataType::Int64, true)); + let array: ArrayRef = Arc::new(LargeListArray::new( + elem_field, + offsets, + Arc::new(values) as ArrayRef, + None, + )); + let size: ArrayRef = Arc::new(Int64Array::from(vec![ + 600_000_000_000_000_000i64, + 600_000_000_000_000_000i64, + ])); + + let err = array_resize_inner(&[array, size]).unwrap_err(); + assert!( + err.to_string().contains("1200000000000000000"), + "expected accumulated total in error: {err}" + ); + assert!( + err.to_string().contains("exceeds the maximum array size"), + "unexpected error: {err}" + ); + } } diff --git a/datafusion/sqllogictest/test_files/array/array_resize.slt b/datafusion/sqllogictest/test_files/array/array_resize.slt index aa915b4fd9439..37f8f2c6935c6 100644 --- a/datafusion/sqllogictest/test_files/array/array_resize.slt +++ b/datafusion/sqllogictest/test_files/array/array_resize.slt @@ -76,6 +76,11 @@ select array_resize(arrow_cast(make_array(1), 'LargeList(Int64)'), 9223372036854 query error DataFusion error: Execution error: array_resize: resulting array of 3000000000 elements exceeds the maximum array size select array_resize(make_array(1), 3000000000, 0); +# Non-primitive element types (e.g. Utf8) fall back to a conservative byte +# width; a size above that cap must error instead of attempting a huge allocation. +query error DataFusion error: Execution error: array_resize: resulting array of 600000000000000000 elements exceeds the maximum array size +select array_resize(arrow_cast(make_array('a'), 'LargeList(Utf8)'), 600000000000000000); + # array_resize scalar function #5 query ? select array_resize(make_array(1.1, 2.2, 3.3), 10, 9.9);