Skip to content

Commit 0a14212

Browse files
authored
fix(core): allow safe_cast when merge widens a struct (#4917)
# Description Follow-up to #4889, from @ion-elgreco's [comment](#4889 (comment)). With `safe_cast` enabled (`error_on_type_mismatch=False` in Python), a merge with `merge_schema=True` that widens a struct fails during planning: ``` Unsupported TRY_CAST from Struct("x": Int64) to Struct("x": Int64, "y": Utf8) ``` DataFusion's `cast` checks struct casts by field name, but `try_cast` still plans with arrow's `can_cast_types`, which needs the same fields (`datafusion-physical-expr` 55.2 `try_cast.rs`, also on DataFusion `main`). The merged type is a superset of the target and source types, so this PR uses a strict `cast` for changed fields, in the CASE branches and in the CDF target preimage. This is the change Ion proposed. Tests: the struct-widening merge tests now run with and without `with_safe_cast(true)` (`rstest` `#[values(false, true)]`): - `test_merge_with_schema_merge_struct_added_field_keeps_copied_rows` - `test_merge_with_schema_merge_struct_added_field_keeps_unset_column` - `test_merge_with_schema_merge_and_pre_existing_struct_added_column` (the source struct only has the new field) - new `test_merge_with_schema_merge_struct_added_field_cdf`: CDF enabled, checks the preimage and postimage of the updated row. The four `safe_cast = true` cases fail on `main` with the error above. Testing (macOS arm64): - `cargo test -p deltalake-core --features datafusion --lib`: 1362 passed - `cargo fmt --check`: clean. `cargo clippy -p deltalake-core --features datafusion --tests --no-deps` reports nothing in `merge/mod.rs`. - Python bindings built from this branch: `tests/test_merge.py` 104 passed. With `error_on_type_mismatch=False`, a struct, a `list<struct>`, a source struct with only the new field, and a struct with CDF enabled all merge, and the unmatched row keeps `{x: 2, y: None}`. On `deltalake` 1.6.6, the first three drop the unmatched row's value to null and the CDF case fails with the same error. AI use: I used Claude Code to apply the change, write the tests, and run the test suites. I reviewed the diff and checked the DataFusion planning check and the test results myself. # Related Issue(s) - follow-up to #4889 / #4887 - upstream: apache/datafusion#26528 (`try_cast` rejects struct casts that `cast` accepts) # Documentation - DataFusion `try_cast` planning: https://github.com/apache/datafusion/blob/main/datafusion/physical-expr/src/expressions/try_cast.rs --------- Signed-off-by: ahlag <[email protected]>
1 parent 10a7c47 commit 0a14212

2 files changed

Lines changed: 109 additions & 9 deletions

File tree

‎crates/core/src/operations/merge/mod.rs‎

Lines changed: 64 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1366,8 +1366,11 @@ async fn execute(
13661366
.get(&column)
13671367
.map(|expr| expr.to_owned())
13681368
.unwrap_or_else(|| col(column.clone()));
1369+
// `try_cast` can't plan struct casts that add fields (DataFusion checks them with
1370+
// arrow's `can_cast_types`), and the merged type is a superset of the target and
1371+
// source types, so a strict cast loses nothing.
13691372
let op = if field_changed {
1370-
merge_cast(op, cast_type.clone(), safe_cast)
1373+
cast(op, cast_type.clone())
13711374
} else {
13721375
op
13731376
};
@@ -1398,7 +1401,7 @@ async fn execute(
13981401
merge_cast(
13991402
Expr::Column(Column::new(qualifier.clone(), delta_field.name())),
14001403
cast_type.clone(),
1401-
safe_cast,
1404+
safe_cast && !field_changed,
14021405
)
14031406
});
14041407
if should_cdc {
@@ -2045,6 +2048,7 @@ mod tests {
20452048
use itertools::Itertools;
20462049
use pretty_assertions::assert_eq;
20472050
use regex::Regex;
2051+
use rstest::rstest;
20482052
use serde_json::json;
20492053
use std::collections::HashMap;
20502054
use std::ops::Neg;
@@ -4239,8 +4243,11 @@ mod tests {
42394243
assert_batches_sorted_eq!(&expected, &actual);
42404244
}
42414245

4246+
#[rstest]
42424247
#[tokio::test]
4243-
async fn test_merge_with_schema_merge_and_pre_existing_struct_added_column() {
4248+
async fn test_merge_with_schema_merge_and_pre_existing_struct_added_column(
4249+
#[values(false, true)] safe_cast: bool,
4250+
) {
42444251
let table = setup_table(None).await;
42454252

42464253
let nested_schema = Arc::new(ArrowSchema::new(vec![Field::new(
@@ -4306,6 +4313,7 @@ mod tests {
43064313
.with_source_alias("source")
43074314
.with_target_alias("target")
43084315
.with_merge_schema(true)
4316+
.with_safe_cast(safe_cast)
43094317
.when_not_matched_insert(|insert| {
43104318
insert
43114319
.set("id", col("source.id"))
@@ -4336,7 +4344,11 @@ mod tests {
43364344

43374345
/// Merges a source row for id "B" whose `nested` struct has an extra field into the
43384346
/// struct table, which rewrites the file holding every target row.
4339-
async fn merge_widened_struct(update_nested: bool) -> DeltaTable {
4347+
async fn merge_widened_struct(
4348+
table: DeltaTable,
4349+
update_nested: bool,
4350+
safe_cast: bool,
4351+
) -> DeltaTable {
43404352
let nested_schema = Arc::new(ArrowSchema::new(vec![Field::new(
43414353
"count",
43424354
ArrowDataType::Int64,
@@ -4352,7 +4364,7 @@ mod tests {
43524364
true,
43534365
),
43544366
]));
4355-
let table = write_data_struct(setup_table(None).await, &schema).await;
4367+
let table = write_data_struct(table, &schema).await;
43564368

43574369
let nested_schema_source = Arc::new(ArrowSchema::new(vec![
43584370
Field::new("count", ArrowDataType::Int64, true),
@@ -4392,6 +4404,7 @@ mod tests {
43924404
.with_source_alias("source")
43934405
.with_target_alias("target")
43944406
.with_merge_schema(true)
4407+
.with_safe_cast(safe_cast)
43954408
.when_matched_update(|update| {
43964409
let update = update.update("value", col("source.value"));
43974410
if update_nested {
@@ -4414,9 +4427,12 @@ mod tests {
44144427
table
44154428
}
44164429

4430+
#[rstest]
44174431
#[tokio::test]
4418-
async fn test_merge_with_schema_merge_struct_added_field_keeps_copied_rows() {
4419-
let table = merge_widened_struct(true).await;
4432+
async fn test_merge_with_schema_merge_struct_added_field_keeps_copied_rows(
4433+
#[values(false, true)] safe_cast: bool,
4434+
) {
4435+
let table = merge_widened_struct(setup_table(None).await, true, safe_cast).await;
44204436
let expected = vec![
44214437
"+----+-------+------------+------------------------+",
44224438
"| id | value | modified | nested |",
@@ -4430,9 +4446,12 @@ mod tests {
44304446
assert_batches_sorted_eq!(&expected, &get_data(&table).await);
44314447
}
44324448

4449+
#[rstest]
44334450
#[tokio::test]
4434-
async fn test_merge_with_schema_merge_struct_added_field_keeps_unset_column() {
4435-
let table = merge_widened_struct(false).await;
4451+
async fn test_merge_with_schema_merge_struct_added_field_keeps_unset_column(
4452+
#[values(false, true)] safe_cast: bool,
4453+
) {
4454+
let table = merge_widened_struct(setup_table(None).await, false, safe_cast).await;
44364455
let expected = vec![
44374456
"+----+-------+------------+--------------------+",
44384457
"| id | value | modified | nested |",
@@ -4446,6 +4465,42 @@ mod tests {
44464465
assert_batches_sorted_eq!(&expected, &get_data(&table).await);
44474466
}
44484467

4468+
#[rstest]
4469+
#[tokio::test]
4470+
async fn test_merge_with_schema_merge_struct_added_field_cdf(
4471+
#[values(false, true)] safe_cast: bool,
4472+
) {
4473+
use crate::kernel::ProtocolInner;
4474+
4475+
let table = DeltaTable::new_in_memory()
4476+
.create()
4477+
.with_columns(get_delta_schema().fields().cloned())
4478+
.with_actions(vec![Action::Protocol(ProtocolInner::new(1, 4).as_kernel())])
4479+
.with_configuration_property(TableProperty::EnableChangeDataFeed, Some("true"))
4480+
.await
4481+
.unwrap();
4482+
let table = merge_widened_struct(table, true, safe_cast).await;
4483+
4484+
let ctx = SessionContext::new();
4485+
let cdf = table
4486+
.scan_cdf()
4487+
.with_starting_version(2)
4488+
.build(&ctx.state(), None)
4489+
.await
4490+
.unwrap();
4491+
let mut batches = collect(cdf, ctx.task_ctx()).await.unwrap();
4492+
// drop _commit_timestamp
4493+
let _: Vec<_> = batches.iter_mut().map(|b| b.remove_column(6)).collect();
4494+
assert_batches_sorted_eq! {[
4495+
"+----+-------+------------+------------------------+------------------+-----------------+",
4496+
"| id | value | modified | nested | _change_type | _commit_version |",
4497+
"+----+-------+------------+------------------------+------------------+-----------------+",
4498+
"| B | 10 | 2021-02-01 | {count: 2, name: } | update_preimage | 2 |",
4499+
"| B | 20 | 2021-02-01 | {count: 9, name: John} | update_postimage | 2 |",
4500+
"+----+-------+------------+------------------------+------------------+-----------------+",
4501+
], &batches }
4502+
}
4503+
44494504
#[tokio::test]
44504505
async fn test_merge_schema_evolution_simple_update() {
44514506
let (table, _) = setup().await;

‎python/tests/test_merge.py‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3320,6 +3320,51 @@ def test_merge_schema_safe_cast_existing_column_failed_cast_becomes_null(
33203320
}
33213321

33223322

3323+
@pytest.mark.pyarrow
3324+
@pytest.mark.parametrize("error_on_type_mismatch", (True, False))
3325+
def test_merge_schema_struct_added_field(
3326+
tmp_path: pathlib.Path, error_on_type_mismatch: bool
3327+
):
3328+
import pyarrow as pa
3329+
3330+
old = pa.struct([("x", pa.int64())])
3331+
new = pa.struct([("x", pa.int64()), ("y", pa.string())])
3332+
target = pa.table({"id": [1, 2], "s": pa.array([{"x": 1}, {"x": 2}], type=old)})
3333+
source = pa.table({"id": [1], "s": pa.array([{"x": 9, "y": "a"}], type=new)})
3334+
write_deltalake(
3335+
tmp_path, target, configuration={"delta.enableChangeDataFeed": "true"}
3336+
)
3337+
3338+
dt = DeltaTable(tmp_path)
3339+
(
3340+
dt.merge(
3341+
source=source,
3342+
predicate="t.id = s.id",
3343+
source_alias="s",
3344+
target_alias="t",
3345+
merge_schema=True,
3346+
error_on_type_mismatch=error_on_type_mismatch,
3347+
)
3348+
.when_matched_update_all()
3349+
.execute()
3350+
)
3351+
3352+
result = dt.to_pyarrow_table().sort_by([("id", "ascending")]).to_pydict()
3353+
assert result == {
3354+
"id": [1, 2],
3355+
"s": [{"x": 9, "y": "a"}, {"x": 2, "y": None}],
3356+
}
3357+
3358+
changes = dt.load_cdf(starting_version=1).read_all()
3359+
assert sorted(
3360+
zip(changes["_change_type"].to_pylist(), changes["s"].to_pylist()),
3361+
key=lambda c: c[0],
3362+
) == [
3363+
("update_postimage", {"x": 9, "y": "a"}),
3364+
("update_preimage", {"x": 1, "y": None}),
3365+
]
3366+
3367+
33233368
@pytest.mark.pyarrow
33243369
def test_merge_on_decimal_3033(tmp_path):
33253370
import pyarrow as pa

0 commit comments

Comments
 (0)