From 62ca8c9c0fe2fea88884c7369d707a33e13a6069 Mon Sep 17 00:00:00 2001 From: buraksenn Date: Thu, 2 Jul 2026 16:34:48 +0300 Subject: [PATCH 1/3] chore: deprecate record_batch macro and use arrow::array::record_batch --- Cargo.lock | 3 +++ .../memory_pool_execution_plan.rs | 2 +- .../ffi/ffi_example_table_provider/Cargo.toml | 1 + .../ffi/ffi_example_table_provider/src/lib.rs | 4 ++-- datafusion/common/src/test_util.rs | 11 ++++++++++- datafusion/core/tests/macro_hygiene/mod.rs | 4 ++++ datafusion/datasource-parquet/src/opener/mod.rs | 4 ++-- datafusion/datasource/Cargo.toml | 1 + datafusion/datasource/src/projection.rs | 4 ++-- datafusion/ffi/src/record_batch_stream.rs | 2 +- datafusion/ffi/src/tests/catalog.rs | 2 +- datafusion/ffi/src/tests/mod.rs | 3 +-- datafusion/ffi/tests/ffi_udaf.rs | 3 +-- datafusion/ffi/tests/ffi_udf.rs | 3 +-- datafusion/physical-expr-adapter/Cargo.toml | 1 + .../physical-expr-adapter/src/schema_rewriter.rs | 4 ++-- 16 files changed, 34 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fa0f5fe50ece2..05f3090bd1cde 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1904,6 +1904,7 @@ name = "datafusion-datasource" version = "54.0.0" dependencies = [ "arrow", + "arrow-schema", "async-compression", "async-trait", "bytes", @@ -2394,6 +2395,7 @@ name = "datafusion-physical-expr-adapter" version = "54.0.0" dependencies = [ "arrow", + "arrow-schema", "datafusion-common", "datafusion-expr", "datafusion-functions", @@ -2951,6 +2953,7 @@ name = "ffi_example_table_provider" version = "0.1.0" dependencies = [ "arrow", + "arrow-schema", "datafusion", "datafusion-ffi", "ffi_module_interface", diff --git a/datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rs b/datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rs index eab813b7eedbd..ca765774d141f 100644 --- a/datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rs +++ b/datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rs @@ -26,9 +26,9 @@ //! - Handle memory pressure by spilling to disk //! - Release memory when done +use arrow::array::record_batch; use arrow::record_batch::RecordBatch; use arrow_schema::SchemaRef; -use datafusion::common::record_batch; use datafusion::common::{exec_datafusion_err, internal_err}; use datafusion::datasource::{DefaultTableSource, memory::MemTable}; use datafusion::error::Result; diff --git a/datafusion-examples/examples/ffi/ffi_example_table_provider/Cargo.toml b/datafusion-examples/examples/ffi/ffi_example_table_provider/Cargo.toml index 3cfa6dcf90f18..4192b751bca53 100644 --- a/datafusion-examples/examples/ffi/ffi_example_table_provider/Cargo.toml +++ b/datafusion-examples/examples/ffi/ffi_example_table_provider/Cargo.toml @@ -23,6 +23,7 @@ publish = false [dependencies] arrow = { workspace = true } +arrow-schema = { workspace = true } datafusion = { workspace = true } datafusion-ffi = { workspace = true } ffi_module_interface = { path = "../ffi_module_interface" } diff --git a/datafusion-examples/examples/ffi/ffi_example_table_provider/src/lib.rs b/datafusion-examples/examples/ffi/ffi_example_table_provider/src/lib.rs index 7894e97f3796d..75449867c5536 100644 --- a/datafusion-examples/examples/ffi/ffi_example_table_provider/src/lib.rs +++ b/datafusion-examples/examples/ffi/ffi_example_table_provider/src/lib.rs @@ -17,9 +17,9 @@ use std::sync::Arc; -use arrow::array::RecordBatch; +use arrow::array::{RecordBatch, record_batch}; use arrow::datatypes::{DataType, Field, Schema}; -use datafusion::{common::record_batch, datasource::MemTable}; +use datafusion::datasource::MemTable; use datafusion_ffi::proto::logical_extension_codec::FFI_LogicalExtensionCodec; use datafusion_ffi::table_provider::FFI_TableProvider; use ffi_module_interface::TableProviderModule; diff --git a/datafusion/common/src/test_util.rs b/datafusion/common/src/test_util.rs index f060704944233..90e0801a2efc8 100644 --- a/datafusion/common/src/test_util.rs +++ b/datafusion/common/src/test_util.rs @@ -364,15 +364,20 @@ macro_rules! create_array { /// Creates a record batch from literal slice of values, suitable for rapid /// testing and development. /// +/// **Deprecated**: prefer the upstream macro from `arrow`, +/// [`arrow::array::record_batch`], which now supports both the literal slice +/// form shown below and a variable/expression form. +/// /// Example: /// ``` -/// use datafusion_common::record_batch; +/// use arrow::array::record_batch; /// let batch = record_batch!( /// ("a", Int32, vec![1, 2, 3]), /// ("b", Float64, vec![Some(4.0), None, Some(5.0)]), /// ("c", Utf8, vec!["alpha", "beta", "gamma"]) /// ); /// ``` +#[deprecated(since = "54.0.0", note = "Use `arrow::array::record_batch` instead")] #[macro_export] macro_rules! record_batch { ($(($name: expr, $type: ident, $values: expr)),*) => { @@ -776,6 +781,10 @@ mod tests { } #[test] + #[expect( + deprecated, + reason = "testing the deprecated record_batch! macro itself" + )] fn test_create_record_batch() -> Result<()> { use arrow::array::Array; diff --git a/datafusion/core/tests/macro_hygiene/mod.rs b/datafusion/core/tests/macro_hygiene/mod.rs index 9fd60cd1f06f3..144062278cc10 100644 --- a/datafusion/core/tests/macro_hygiene/mod.rs +++ b/datafusion/core/tests/macro_hygiene/mod.rs @@ -41,6 +41,10 @@ mod plan_datafusion_err { } mod record_batch { + #![expect( + deprecated, + reason = "exercising hygiene of the deprecated `datafusion_common::record_batch!` while it is still exported" + )] // NO other imports! use datafusion_common::record_batch; diff --git a/datafusion/datasource-parquet/src/opener/mod.rs b/datafusion/datasource-parquet/src/opener/mod.rs index af50b8990130d..569552d1fa372 100644 --- a/datafusion/datasource-parquet/src/opener/mod.rs +++ b/datafusion/datasource-parquet/src/opener/mod.rs @@ -1695,12 +1695,12 @@ mod test { CachedParquetFileReaderFactory, DefaultParquetFileReaderFactory, ParquetFileReaderFactory, ParquetRowSelection, RowGroupAccess, }; - use arrow::array::RecordBatch; + use arrow::array::{RecordBatch, record_batch}; use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use bytes::{BufMut, BytesMut}; use datafusion_common::{ ColumnStatistics, ScalarValue, Statistics, assert_contains, internal_err, - record_batch, stats::Precision, + stats::Precision, }; use datafusion_datasource::morsel::{Morsel, Morselizer}; use datafusion_datasource::{PartitionedFile, TableSchema, TableSchemaBuilder}; diff --git a/datafusion/datasource/Cargo.toml b/datafusion/datasource/Cargo.toml index 2ac42ed900095..7b0fc83c5c088 100644 --- a/datafusion/datasource/Cargo.toml +++ b/datafusion/datasource/Cargo.toml @@ -73,6 +73,7 @@ url = { workspace = true } zstd = { workspace = true, optional = true } [dev-dependencies] +arrow-schema = { workspace = true } criterion = { workspace = true } datafusion-functions = { workspace = true } insta = { workspace = true } diff --git a/datafusion/datasource/src/projection.rs b/datafusion/datasource/src/projection.rs index 16207c086f7bc..d5fa9bf49b102 100644 --- a/datafusion/datasource/src/projection.rs +++ b/datafusion/datasource/src/projection.rs @@ -294,9 +294,9 @@ impl SplitProjection { mod test { use std::sync::Arc; - use arrow::array::{AsArray, RecordBatch}; + use arrow::array::{AsArray, RecordBatch, record_batch}; use arrow::datatypes::{DataType, Field, SchemaRef}; - use datafusion_common::{DFSchema, ScalarValue, config::ConfigOptions, record_batch}; + use datafusion_common::{DFSchema, ScalarValue, config::ConfigOptions}; use datafusion_expr::{Expr, ScalarUDF, col, execution_props::ExecutionProps}; use datafusion_functions::core::input_file_name::InputFileNameFunc; use datafusion_physical_expr::{ diff --git a/datafusion/ffi/src/record_batch_stream.rs b/datafusion/ffi/src/record_batch_stream.rs index 74709848cbb7f..5a92cbfe5fe78 100644 --- a/datafusion/ffi/src/record_batch_stream.rs +++ b/datafusion/ffi/src/record_batch_stream.rs @@ -218,8 +218,8 @@ impl Drop for FFI_RecordBatchStream { mod tests { use std::sync::Arc; + use arrow::array::record_batch; use arrow::datatypes::{DataType, Field, Schema}; - use datafusion::common::record_batch; use datafusion::error::Result; use datafusion::execution::SendableRecordBatchStream; use datafusion::test_util::bounded_stream; diff --git a/datafusion/ffi/src/tests/catalog.rs b/datafusion/ffi/src/tests/catalog.rs index 0c02de5d049ae..b0b0858a8a3d7 100644 --- a/datafusion/ffi/src/tests/catalog.rs +++ b/datafusion/ffi/src/tests/catalog.rs @@ -48,8 +48,8 @@ pub struct FixedSchemaProvider { } pub fn fruit_table() -> Arc { + use arrow::array::record_batch; use arrow::datatypes::{DataType, Field}; - use datafusion_common::record_batch; let schema = Arc::new(Schema::new(vec![ Field::new("units", DataType::Int32, true), diff --git a/datafusion/ffi/src/tests/mod.rs b/datafusion/ffi/src/tests/mod.rs index dcd0910ecb4e9..d372dcf9177e6 100644 --- a/datafusion/ffi/src/tests/mod.rs +++ b/datafusion/ffi/src/tests/mod.rs @@ -17,14 +17,13 @@ use std::sync::Arc; -use arrow::array::RecordBatch; +use arrow::array::{RecordBatch, record_batch}; use arrow_schema::{DataType, Field, Schema}; use async_provider::create_async_table_provider; use async_trait::async_trait; use catalog::create_catalog_provider; use datafusion_catalog::MemTable; use datafusion_catalog::{Session, TableProvider}; -use datafusion_common::record_batch; use datafusion_common::stats::Precision; use datafusion_common::{ColumnStatistics, Statistics}; use datafusion_common::{Result, ScalarValue}; diff --git a/datafusion/ffi/tests/ffi_udaf.rs b/datafusion/ffi/tests/ffi_udaf.rs index 7df3404d7421b..3234f6533df9c 100644 --- a/datafusion/ffi/tests/ffi_udaf.rs +++ b/datafusion/ffi/tests/ffi_udaf.rs @@ -21,8 +21,7 @@ mod tests { use std::sync::Arc; - use arrow::array::Float64Array; - use datafusion::common::record_batch; + use arrow::array::{Float64Array, record_batch}; use datafusion::error::Result; use datafusion::logical_expr::{AggregateUDF, AggregateUDFImpl}; use datafusion::prelude::{SessionContext, col}; diff --git a/datafusion/ffi/tests/ffi_udf.rs b/datafusion/ffi/tests/ffi_udf.rs index dffaf83c479b1..617cbc196b1ac 100644 --- a/datafusion/ffi/tests/ffi_udf.rs +++ b/datafusion/ffi/tests/ffi_udf.rs @@ -19,9 +19,8 @@ /// when the feature integration-tests is built #[cfg(feature = "integration-tests")] mod tests { - use arrow::array::{Array, AsArray}; + use arrow::array::{Array, AsArray, record_batch}; use arrow::datatypes::DataType; - use datafusion::common::record_batch; use datafusion::error::Result; use datafusion::logical_expr::{ExpressionPlacement, ScalarUDF, ScalarUDFImpl}; use datafusion::prelude::{SessionContext, col}; diff --git a/datafusion/physical-expr-adapter/Cargo.toml b/datafusion/physical-expr-adapter/Cargo.toml index 453c8bdaacb4a..0729eeea9d3b9 100644 --- a/datafusion/physical-expr-adapter/Cargo.toml +++ b/datafusion/physical-expr-adapter/Cargo.toml @@ -28,3 +28,4 @@ itertools = { workspace = true } workspace = true [dev-dependencies] +arrow-schema = { workspace = true } diff --git a/datafusion/physical-expr-adapter/src/schema_rewriter.rs b/datafusion/physical-expr-adapter/src/schema_rewriter.rs index d9eed669ba98f..0a71270ce9e83 100644 --- a/datafusion/physical-expr-adapter/src/schema_rewriter.rs +++ b/datafusion/physical-expr-adapter/src/schema_rewriter.rs @@ -627,10 +627,10 @@ mod tests { use super::*; use arrow::array::{ Array, BooleanArray, GenericListArray, Int32Array, Int64Array, RecordBatch, - RecordBatchOptions, StringArray, StringViewArray, StructArray, + RecordBatchOptions, StringArray, StringViewArray, StructArray, record_batch, }; use arrow::datatypes::{Field, Fields, Schema}; - use datafusion_common::{assert_contains, record_batch}; + use datafusion_common::assert_contains; use datafusion_expr::Operator; use datafusion_physical_expr::expressions::{Column, Literal, col}; From 56c46ba12b30239cb8e927eb42bd996c3ccef090 Mon Sep 17 00:00:00 2001 From: buraksenn Date: Thu, 2 Jul 2026 17:04:04 +0300 Subject: [PATCH 2/3] bump version for deprecation notice --- datafusion/common/src/test_util.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/common/src/test_util.rs b/datafusion/common/src/test_util.rs index 90e0801a2efc8..8ad1e086bc29b 100644 --- a/datafusion/common/src/test_util.rs +++ b/datafusion/common/src/test_util.rs @@ -377,7 +377,7 @@ macro_rules! create_array { /// ("c", Utf8, vec!["alpha", "beta", "gamma"]) /// ); /// ``` -#[deprecated(since = "54.0.0", note = "Use `arrow::array::record_batch` instead")] +#[deprecated(since = "55.0.0", note = "Use `arrow::array::record_batch` instead")] #[macro_export] macro_rules! record_batch { ($(($name: expr, $type: ident, $values: expr)),*) => { From b9f3b64d6d7490741f2f0f09784094ca68ce8fb9 Mon Sep 17 00:00:00 2001 From: buraksenn Date: Thu, 2 Jul 2026 22:37:35 +0300 Subject: [PATCH 3/3] fix false positive cargo machete --- .../examples/ffi/ffi_example_table_provider/Cargo.toml | 3 +++ datafusion/datasource/Cargo.toml | 3 +++ datafusion/physical-expr-adapter/Cargo.toml | 3 +++ 3 files changed, 9 insertions(+) diff --git a/datafusion-examples/examples/ffi/ffi_example_table_provider/Cargo.toml b/datafusion-examples/examples/ffi/ffi_example_table_provider/Cargo.toml index 4192b751bca53..d9571c9a0718e 100644 --- a/datafusion-examples/examples/ffi/ffi_example_table_provider/Cargo.toml +++ b/datafusion-examples/examples/ffi/ffi_example_table_provider/Cargo.toml @@ -28,6 +28,9 @@ datafusion = { workspace = true } datafusion-ffi = { workspace = true } ffi_module_interface = { path = "../ffi_module_interface" } +[package.metadata.cargo-machete] +ignored = ["arrow-schema"] + [lints] workspace = true diff --git a/datafusion/datasource/Cargo.toml b/datafusion/datasource/Cargo.toml index 7b0fc83c5c088..38a7db07e4e1f 100644 --- a/datafusion/datasource/Cargo.toml +++ b/datafusion/datasource/Cargo.toml @@ -79,6 +79,9 @@ datafusion-functions = { workspace = true } insta = { workspace = true } tempfile = { workspace = true } +[package.metadata.cargo-machete] +ignored = ["arrow-schema"] + # Note: add additional linter rules in lib.rs. # Rust does not support workspace + new linter rules in subcrates yet # https://github.com/rust-lang/cargo/issues/13157 diff --git a/datafusion/physical-expr-adapter/Cargo.toml b/datafusion/physical-expr-adapter/Cargo.toml index 0729eeea9d3b9..3c532c968be2a 100644 --- a/datafusion/physical-expr-adapter/Cargo.toml +++ b/datafusion/physical-expr-adapter/Cargo.toml @@ -29,3 +29,6 @@ workspace = true [dev-dependencies] arrow-schema = { workspace = true } + +[package.metadata.cargo-machete] +ignored = ["arrow-schema"]