Skip to content

Commit a5cbcd5

Browse files
committed
fix: find schema-adaptation casts by explicit target, not by name
Spark allows a Parquet column named "", so an empty target name does not mark a planner cast: an INT64 column named "" read as int skipped the read-conversion check and returned its values where Spark raises SchemaColumnConvertNotSupportedException. DataFusion's default adapter builds its schema casts with CastExpr::new_with_target_field, which sets the explicit-target flag that CastExpr::has_explicit_metadata() returns, also for an empty name, and with_new_children keeps it. Comet's planner builds the casts it puts in a predicate with CastExpr::new, which doesn't set it. replace_with_spark_cast now checks that flag, and is_schema_adaptation_cast is gone. Tests: an INT64 column named "" read as int is rejected under both type-promotion settings, the naming-convention test is replaced by one that pins the explicit-target flag, and the three substring filter tests are now one table-driven test on the existing scan_parquet_file helper, which still fails with the original INT32 -> bigint error without the fix.
1 parent a1350a7 commit a5cbcd5

1 file changed

Lines changed: 141 additions & 171 deletions

File tree

‎native/core/src/parquet/schema_adapter.rs‎

Lines changed: 141 additions & 171 deletions
Original file line numberDiff line numberDiff line change
@@ -429,23 +429,6 @@ fn is_string_or_binary(dt: &DataType) -> bool {
429429
)
430430
}
431431

432-
/// Whether `cast` reconciles a column's Parquet (file) type with its table-schema type, as opposed
433-
/// to being part of the pushed-down expression itself.
434-
///
435-
/// DataFusion's `DefaultPhysicalExprAdapter` builds the former with
436-
/// `CastExpr::new_with_target_field`, passing the table schema's field, so the target is named
437-
/// after the column (or nested field). Comet's planner builds the latter with `CastExpr::new`, for
438-
/// example to coerce a scalar function's arguments to the types the function requires:
439-
/// `substring(s, 1, 2)` casts its Int32 literals to Int64. `CastExpr::new` produces an unnamed
440-
/// target field.
441-
///
442-
/// Only the former are Parquet read conversions that Spark's vectorized reader validates. Applying
443-
/// those rules to the latter rejects valid queries, for example as a disallowed `INT32 -> bigint`
444-
/// promotion on Spark 3.x when row-level filter pushdown evaluates the predicate.
445-
fn is_schema_adaptation_cast(cast: &datafusion::physical_expr::expressions::CastExpr) -> bool {
446-
!cast.target_field().name().is_empty()
447-
}
448-
449432
/// Build a Spark-shaped `SchemaColumnConvertNotSupportedException` carrier for a
450433
/// rejected Parquet -> Spark conversion. `column` is the Spark-style column path (`a`, or
451434
/// `s, x` for a nested leaf); the bracketed wrapping mirrors
@@ -1405,17 +1388,21 @@ impl SparkPhysicalExprAdapter {
14051388

14061389
/// Replace CastExpr (DataFusion's cast) with Spark's Cast expression.
14071390
///
1408-
/// Only casts that reconcile a file column with its table-schema type are rewritten (see
1409-
/// [`is_schema_adaptation_cast`]). Any other `CastExpr` is part of the predicate itself and is
1410-
/// left as the planner built it.
1391+
/// Only the casts that reconcile a file column with its table-schema type are Parquet read
1392+
/// conversions, which Spark's vectorized reader validates. DataFusion's default adapter builds
1393+
/// them with `CastExpr::new_with_target_field`, so `has_explicit_metadata()` is true, also for a
1394+
/// column named `""`. Any other `CastExpr` is part of the predicate itself and is left as the
1395+
/// planner built it. Comet's planner uses `CastExpr::new`, for example to coerce `substring`'s
1396+
/// Int32 literals to Int64, and applying the read rules to that cast rejected valid queries as
1397+
/// a disallowed `INT32 -> bigint` promotion on Spark 3.x.
14111398
fn replace_with_spark_cast(
14121399
&self,
14131400
expr: Arc<dyn PhysicalExpr>,
14141401
) -> DataFusionResult<Transformed<Arc<dyn PhysicalExpr>>> {
14151402
// Check for CastExpr and replace with spark_expr::Cast
14161403
if let Some(cast) = expr.downcast_ref::<datafusion::physical_expr::expressions::CastExpr>()
14171404
{
1418-
if !is_schema_adaptation_cast(cast) {
1405+
if !cast.has_explicit_metadata() {
14191406
return Ok(Transformed::no(expr));
14201407
}
14211408

@@ -2094,6 +2081,36 @@ pub(crate) mod test {
20942081
Ok(err.to_string())
20952082
}
20962083

2084+
/// Spark allows a Parquet column named `""`, and reading one goes through the same conversion
2085+
/// check as any other column: an `INT64` column named `""` read as `int` is rejected, whether or
2086+
/// not type promotion is allowed, even when every value would fit.
2087+
#[tokio::test]
2088+
async fn empty_named_column_long_read_as_int_errors() -> Result<(), DataFusionError> {
2089+
let file_schema = Arc::new(Schema::new(vec![Field::new("", DataType::Int64, true)]));
2090+
let batch = RecordBatch::try_new(
2091+
Arc::clone(&file_schema),
2092+
vec![Arc::new(Int64Array::from(vec![1i64, 2]))],
2093+
)?;
2094+
let required_schema = Arc::new(Schema::new(vec![Field::new("", DataType::Int32, true)]));
2095+
for allow_type_promotion in [false, true] {
2096+
let mut options = default_options();
2097+
options.allow_type_promotion = allow_type_promotion;
2098+
let mut stream = scan_parquet(&batch, Arc::clone(&required_schema), options)?;
2099+
let msg = match stream.next().await {
2100+
Some(Err(e)) => e.to_string(),
2101+
other => panic!(
2102+
"allow_type_promotion={allow_type_promotion}: expected ParquetSchemaConvert, \
2103+
got {other:?}"
2104+
),
2105+
};
2106+
assert!(
2107+
msg.contains("Found: INT64") && msg.contains("Expected: int"),
2108+
"allow_type_promotion={allow_type_promotion}: {msg}"
2109+
);
2110+
}
2111+
Ok(())
2112+
}
2113+
20972114
/// `INT64 -> INT32` truncates to the lower 32 bits in DataFusion's cast.
20982115
/// Spark's vectorized reader rejects this. See #4297.
20992116
#[tokio::test]
@@ -2375,13 +2392,13 @@ pub(crate) mod test {
23752392
Ok(())
23762393
}
23772394

2378-
/// `is_schema_adaptation_cast` tells the casts DataFusion's default adapter inserts apart from
2379-
/// the casts Comet's planner puts in a predicate by whether the target field is named. If a
2380-
/// DataFusion upgrade changes how `CastExpr::new` names its target, this fails and the check
2381-
/// has to be revisited (newer DataFusion versions also expose an explicit-target marker).
2395+
/// `replace_with_spark_cast` tells the casts DataFusion's default adapter inserts apart from the
2396+
/// casts Comet's planner puts in a predicate by `CastExpr::has_explicit_metadata()`. This pins
2397+
/// what that relies on: `new_with_target_field` sets it, also for a field named `""`, which
2398+
/// Spark allows as a column name. `CastExpr::new` doesn't set it, and `with_new_children` keeps
2399+
/// it.
23822400
#[test]
2383-
fn schema_adaptation_cast_is_told_apart_by_target_field_name() {
2384-
use super::is_schema_adaptation_cast;
2401+
fn schema_casts_are_told_apart_by_explicit_target() -> Result<(), DataFusionError> {
23852402
use datafusion::physical_expr::expressions::CastExpr;
23862403

23872404
// Built by the planner, from a bare type.
@@ -2390,55 +2407,38 @@ pub(crate) mod test {
23902407
DataType::Int64,
23912408
None,
23922409
);
2393-
assert!(!is_schema_adaptation_cast(&planner_cast));
2410+
assert!(!planner_cast.has_explicit_metadata());
23942411

23952412
// Built by the default adapter, from the table schema's field.
2396-
let adapter_cast = CastExpr::new_with_target_field(
2397-
Arc::new(Column::new("a", 0)),
2398-
Arc::new(Field::new("a", DataType::Int64, true)),
2399-
None,
2400-
);
2401-
assert!(is_schema_adaptation_cast(&adapter_cast));
2413+
for name in ["a", ""] {
2414+
let adapter_cast: Arc<dyn PhysicalExpr> = Arc::new(CastExpr::new_with_target_field(
2415+
Arc::new(Column::new(name, 0)),
2416+
Arc::new(Field::new(name, DataType::Int64, true)),
2417+
None,
2418+
));
2419+
let rebuilt = Arc::clone(&adapter_cast)
2420+
.with_new_children(vec![Arc::new(Column::new(name, 0))])?;
2421+
for cast in [adapter_cast, rebuilt] {
2422+
let cast = cast.downcast_ref::<CastExpr>().expect("a CastExpr");
2423+
assert!(cast.has_explicit_metadata(), "target field {name:?}");
2424+
}
2425+
}
2426+
Ok(())
24022427
}
24032428

2404-
/// Scans a Parquet file with a pushed-down `substring(s, 1, 2) = 'ab'` data filter, the way
2405-
/// `CometNativeScan` sends it, and returns the values the scan produced.
2406-
///
2407-
/// The file schema is identical to the requested schema, so there is no schema evolution of any
2408-
/// kind. The predicate comes from the real planner, fed the proto `CometSubstring` emits (Int32
2409-
/// literals for the position and length). `SparkSubstring` takes Int64 positions, so the planner
2410-
/// coerces those literals with `CastExpr::new`.
2411-
async fn scan_with_substring_filter(
2412-
row_filter_pushdown: bool,
2413-
allow_type_promotion: bool,
2414-
) -> Result<Vec<String>, String> {
2429+
/// The pushed-down `substring(s, 1, 2) = 'ab'` data filter over `schema`, the way
2430+
/// `CometNativeScan` sends it: built by the real planner from the proto `CometSubstring` emits,
2431+
/// with Int32 literals for the position and length. `SparkSubstring` takes Int64 positions, so
2432+
/// the planner coerces those literals with `CastExpr::new`.
2433+
fn substring_filter(schema: SchemaRef) -> Result<Arc<dyn PhysicalExpr>, DataFusionError> {
24152434
use crate::execution::planner::PhysicalPlanner;
24162435
use datafusion::common::tree_node::{TreeNode, TreeNodeRecursion};
2417-
use datafusion::config::{ConfigOptions, TableParquetOptions};
2418-
use datafusion::datasource::physical_plan::FileSource;
24192436
use datafusion::logical_expr::ScalarUDF;
24202437
use datafusion::physical_expr::expressions::CastExpr;
2421-
use datafusion::physical_plan::common::collect;
24222438
use datafusion::prelude::SessionContext;
24232439
use datafusion_comet_proto::spark_expression::{self, expr::ExprStruct, literal, Expr};
24242440
use datafusion_spark::function::string::substring::SparkSubstring;
24252441

2426-
let schema = Arc::new(Schema::new(vec![Field::new("s", DataType::Utf8, true)]));
2427-
let batch = RecordBatch::try_new(
2428-
Arc::clone(&schema),
2429-
vec![Arc::new(StringArray::from(vec![
2430-
"abc", "xyz", "abd", "qrs",
2431-
]))],
2432-
)
2433-
.map_err(|e| e.to_string())?;
2434-
let filename = get_temp_filename();
2435-
let filename = filename.as_path().as_os_str().to_str().unwrap().to_string();
2436-
let file = File::create(&filename).map_err(|e| e.to_string())?;
2437-
let mut writer =
2438-
ArrowWriter::try_new(file, Arc::clone(&schema), None).map_err(|e| e.to_string())?;
2439-
writer.write(&batch).map_err(|e| e.to_string())?;
2440-
writer.close().map_err(|e| e.to_string())?;
2441-
24422442
let session_ctx = Arc::new(SessionContext::new());
24432443
session_ctx.register_udf(ScalarUDF::new_from_impl(SparkSubstring::default()));
24442444
let planner = PhysicalPlanner::new(Arc::clone(&session_ctx), 0);
@@ -2473,120 +2473,90 @@ pub(crate) mod test {
24732473
fail_on_error: false,
24742474
null_short_circuit: false,
24752475
}));
2476-
let predicate = planner
2477-
.create_expr(
2478-
&proto_expr(ExprStruct::Eq(Box::new(spark_expression::BinaryExpr {
2479-
left: Some(Box::new(substring)),
2480-
right: Some(Box::new(proto_expr(ExprStruct::Literal(
2481-
spark_expression::Literal {
2482-
value: Some(literal::Value::StringVal("ab".to_string())),
2483-
datatype: Some(proto_type(7)),
2484-
is_null: false,
2485-
},
2486-
)))),
2487-
}))),
2488-
Arc::clone(&schema),
2489-
)
2490-
.map_err(|e| e.to_string())?;
2476+
let predicate = planner.create_expr(
2477+
&proto_expr(ExprStruct::Eq(Box::new(spark_expression::BinaryExpr {
2478+
left: Some(Box::new(substring)),
2479+
right: Some(Box::new(proto_expr(ExprStruct::Literal(
2480+
spark_expression::Literal {
2481+
value: Some(literal::Value::StringVal("ab".to_string())),
2482+
datatype: Some(proto_type(7)),
2483+
is_null: false,
2484+
},
2485+
)))),
2486+
}))),
2487+
schema,
2488+
)?;
24912489

24922490
// Guard the premise: the planner really does leave DataFusion `CastExpr`s in the predicate.
24932491
let mut casts = 0;
2494-
predicate
2495-
.apply(|e| {
2496-
if e.downcast_ref::<CastExpr>().is_some() {
2497-
casts += 1;
2498-
}
2499-
Ok(TreeNodeRecursion::Continue)
2500-
})
2501-
.map_err(|e| e.to_string())?;
2492+
predicate.apply(|e| {
2493+
if e.downcast_ref::<CastExpr>().is_some() {
2494+
casts += 1;
2495+
}
2496+
Ok(TreeNodeRecursion::Continue)
2497+
})?;
25022498
assert!(casts >= 2, "expected planner-inserted casts in {predicate}");
2503-
2504-
// Configure the scan the way the native scan does when
2505-
// `spark.comet.parquet.rowFilterPushdown.enabled` is set, and push the filter the way
2506-
// `init_datasource_exec` does.
2507-
let mut table_options = TableParquetOptions::new();
2508-
table_options.global.pushdown_filters = row_filter_pushdown;
2509-
table_options.global.reorder_filters = row_filter_pushdown;
2510-
let mut config = ConfigOptions::default();
2511-
config.execution.parquet.pushdown_filters = row_filter_pushdown;
2512-
config.execution.parquet.reorder_filters = row_filter_pushdown;
2513-
let parquet_source =
2514-
ParquetSource::new(Arc::clone(&schema)).with_table_parquet_options(table_options);
2515-
let file_source: Arc<dyn FileSource> = parquet_source
2516-
.try_pushdown_filters(vec![predicate], &config)
2517-
.map_err(|e| e.to_string())?
2518-
.updated_node
2519-
.unwrap_or_else(|| Arc::new(parquet_source));
2520-
2521-
let mut spark_parquet_options = SparkParquetOptions::new(EvalMode::Legacy, "UTC");
2522-
spark_parquet_options.allow_type_promotion = allow_type_promotion;
2523-
let expr_adapter_factory: Arc<dyn PhysicalExprAdapterFactory> = Arc::new(
2524-
SparkPhysicalExprAdapterFactory::new(spark_parquet_options, None),
2525-
);
2526-
let files = FileGroup::new(vec![
2527-
PartitionedFile::from_path(filename).map_err(|e| e.to_string())?
2528-
]);
2529-
let file_scan_config =
2530-
FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)
2531-
.with_file_groups(vec![files])
2532-
.with_expr_adapter(Some(expr_adapter_factory))
2533-
.build();
2534-
2535-
let scan = DataSourceExec::new(Arc::new(file_scan_config));
2536-
let stream = scan
2537-
.execute(0, Arc::new(TaskContext::default()))
2538-
.map_err(|e| e.to_string())?;
2539-
let batches = collect(stream).await.map_err(|e| e.to_string())?;
2540-
let mut values = vec![];
2541-
for batch in &batches {
2542-
let column = arrow::compute::cast(batch.column(0), &DataType::Utf8)
2543-
.map_err(|e| e.to_string())?;
2544-
values.extend(
2545-
column
2546-
.as_string::<i32>()
2547-
.iter()
2548-
.flatten()
2549-
.map(str::to_string),
2550-
);
2551-
}
2552-
Ok(values)
2553-
}
2554-
2555-
/// Control: without row-level filter pushdown (the default) the predicate is only used for
2556-
/// row-group and page pruning, so it is never evaluated against a decoded batch. The scan
2557-
/// returns every row and leaves the filtering to the Spark `Filter` above it.
2558-
#[tokio::test]
2559-
async fn pushed_substring_filter_without_row_filter_pushdown() {
2560-
assert_eq!(
2561-
scan_with_substring_filter(false, false).await,
2562-
Ok(["abc", "xyz", "abd", "qrs"]
2563-
.iter()
2564-
.map(|s| s.to_string())
2565-
.collect::<Vec<_>>())
2566-
);
2567-
}
2568-
2569-
/// With `spark.comet.parquet.rowFilterPushdown.enabled=true` the predicate is evaluated on every
2570-
/// decoded batch. On Spark 3.x (`allow_type_promotion = false`) the planner's `Int32 -> Int64`
2571-
/// coercion of the `substring` literals must not be mistaken for a Parquet schema promotion,
2572-
/// which failed the scan with `ParquetSchemaConvert { column: "[]", physical_type: "INT32",
2573-
/// spark_type: "bigint" }`.
2574-
#[tokio::test]
2575-
async fn pushed_substring_filter_with_row_filter_pushdown() {
2576-
assert_eq!(
2577-
scan_with_substring_filter(true, false).await,
2578-
Ok(vec!["abc".to_string(), "abd".to_string()])
2579-
);
2499+
Ok(predicate)
25802500
}
25812501

2582-
/// Same as above for Spark 4.x (`allow_type_promotion = true`), which accepts `Int32 -> Int64`
2583-
/// and so never exposed the problem for this particular cast.
2502+
/// The casts Comet's planner puts in a pushed-down predicate are not Parquet read conversions.
2503+
/// The file's schema equals the requested schema, so there is no schema evolution of any kind.
25842504
#[tokio::test]
2585-
async fn pushed_substring_filter_with_row_filter_pushdown_and_type_promotion() {
2586-
assert_eq!(
2587-
scan_with_substring_filter(true, true).await,
2588-
Ok(vec!["abc".to_string(), "abd".to_string()])
2589-
);
2505+
async fn pushed_substring_filter_keeps_planner_casts() -> Result<(), DataFusionError> {
2506+
let schema = Arc::new(Schema::new(vec![Field::new("s", DataType::Utf8, true)]));
2507+
let batch = RecordBatch::try_new(
2508+
Arc::clone(&schema),
2509+
vec![Arc::new(StringArray::from(vec![
2510+
"abc", "xyz", "abd", "qrs",
2511+
]))],
2512+
)?;
2513+
let filename = get_temp_filename().to_str().unwrap().to_string();
2514+
let mut writer = ArrowWriter::try_new(File::create(&filename)?, Arc::clone(&schema), None)?;
2515+
writer.write(&batch)?;
2516+
writer.close()?;
2517+
let predicate = substring_filter(Arc::clone(&schema))?;
2518+
2519+
let every_row: &[&str] = &["abc", "xyz", "abd", "qrs"];
2520+
let matching: &[&str] = &["abc", "abd"];
2521+
for (row_filters, allow_type_promotion, expected) in [
2522+
// Control: without row-level filter pushdown (the default) the predicate is only used
2523+
// for row-group and page pruning, so the scan returns every row and leaves the
2524+
// filtering to the Spark `Filter` above it.
2525+
(false, false, every_row),
2526+
// With `spark.comet.parquet.rowFilterPushdown.enabled=true` the predicate is evaluated
2527+
// on every decoded batch. On Spark 3.x (`allow_type_promotion = false`) the planner's
2528+
// Int32 -> Int64 coercion was taken for a disallowed `INT32 -> bigint` promotion, which
2529+
// failed the scan with `ParquetSchemaConvert { column: "[]", .. }`.
2530+
(true, false, matching),
2531+
// Control: Spark 4.x allows that promotion, so this case passed before the fix too.
2532+
(true, true, matching),
2533+
] {
2534+
let mut options = default_options();
2535+
options.allow_type_promotion = allow_type_promotion;
2536+
let mut stream = scan_parquet_file(
2537+
filename.clone(),
2538+
Arc::clone(&schema),
2539+
options,
2540+
Some(Arc::clone(&predicate)),
2541+
row_filters,
2542+
)?;
2543+
let mut values: Vec<String> = vec![];
2544+
while let Some(batch) = stream.next().await {
2545+
let column = arrow::compute::cast(batch?.column(0), &DataType::Utf8)?;
2546+
values.extend(
2547+
column
2548+
.as_string::<i32>()
2549+
.iter()
2550+
.flatten()
2551+
.map(str::to_string),
2552+
);
2553+
}
2554+
assert_eq!(
2555+
values, expected,
2556+
"row_filters={row_filters} allow_type_promotion={allow_type_promotion}"
2557+
);
2558+
}
2559+
Ok(())
25902560
}
25912561

25922562
#[tokio::test]

0 commit comments

Comments
 (0)