Skip to content

Commit b84a105

Browse files
ahlagmistral-vibertyler
committed
fix(core): allow safe_cast when merge widens a struct (delta-io#4917)
Follow-up to delta-io#4889, from @ion-elgreco's [comment](delta-io#4889 (comment)). With `safe_cast` enabled (`error_on_type_mismatch=False` in Python), a merge with `merge_schema=True` that widens a struct fails during planning: ``` Unsupported TRY_CAST from Struct("x": Int64) to Struct("x": Int64, "y": Utf8) ``` DataFusion's `cast` checks struct casts by field name, but `try_cast` still plans with arrow's `can_cast_types`, which needs the same fields (`datafusion-physical-expr` 55.2 `try_cast.rs`, also on DataFusion `main`). The merged type is a superset of the target and source types, so this PR uses a strict `cast` for changed fields, in the CASE branches and in the CDF target preimage. This is the change Ion proposed. Tests: the struct-widening merge tests now run with and without `with_safe_cast(true)` (`rstest` `#[values(false, true)]`): - `test_merge_with_schema_merge_struct_added_field_keeps_copied_rows` - `test_merge_with_schema_merge_struct_added_field_keeps_unset_column` - `test_merge_with_schema_merge_and_pre_existing_struct_added_column` (the source struct only has the new field) - new `test_merge_with_schema_merge_struct_added_field_cdf`: CDF enabled, checks the preimage and postimage of the updated row. The four `safe_cast = true` cases fail on `main` with the error above. Testing (macOS arm64): - `cargo test -p deltalake-core --features datafusion --lib`: 1362 passed - `cargo fmt --check`: clean. `cargo clippy -p deltalake-core --features datafusion --tests --no-deps` reports nothing in `merge/mod.rs`. - Python bindings built from this branch: `tests/test_merge.py` 104 passed. With `error_on_type_mismatch=False`, a struct, a `list<struct>`, a source struct with only the new field, and a struct with CDF enabled all merge, and the unmatched row keeps `{x: 2, y: None}`. On `deltalake` 1.6.6, the first three drop the unmatched row's value to null and the CDF case fails with the same error. AI use: I used Claude Code to apply the change, write the tests, and run the test suites. I reviewed the diff and checked the DataFusion planning check and the test results myself. - follow-up to delta-io#4889 / delta-io#4887 - upstream: apache/datafusion#26528 (`try_cast` rejects struct casts that `cast` accepts) - DataFusion `try_cast` planning: https://github.com/apache/datafusion/blob/main/datafusion/physical-expr/src/expressions/try_cast.rs --------- Signed-off-by: ahlag <[email protected]> (cherry picked from commit 0a14212) Co-Authored-By: Devstral 2 <[email protected]> Signed-off-by: R. Tyler Croy <[email protected]> Co-Authored-By: R. Tyler Croy <[email protected]> Signed-off-by: R. Tyler Croy <[email protected]>
1 parent 9e899c5 commit b84a105

2 files changed

Lines changed: 109 additions & 9 deletions

File tree

‎crates/core/src/operations/merge/mod.rs‎

Lines changed: 64 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1366,8 +1366,11 @@ async fn execute(
13661366
.get(&column)
13671367
.map(|expr| expr.to_owned())
13681368
.unwrap_or_else(|| col(column.clone()));
1369+
// `try_cast` can't plan struct casts that add fields (DataFusion checks them with
1370+
// arrow's `can_cast_types`), and the merged type is a superset of the target and
1371+
// source types, so a strict cast loses nothing.
13691372
let op = if field_changed {
1370-
merge_cast(op, cast_type.clone(), safe_cast)
1373+
cast(op, cast_type.clone())
13711374
} else {
13721375
op
13731376
};
@@ -1398,7 +1401,7 @@ async fn execute(
13981401
merge_cast(
13991402
Expr::Column(Column::new(qualifier.clone(), delta_field.name())),
14001403
cast_type.clone(),
1401-
safe_cast,
1404+
safe_cast && !field_changed,
14021405
)
14031406
});
14041407
if should_cdc {
@@ -2049,6 +2052,7 @@ mod tests {
20492052
use itertools::Itertools;
20502053
use pretty_assertions::assert_eq;
20512054
use regex::Regex;
2055+
use rstest::rstest;
20522056
use serde_json::json;
20532057
use std::collections::HashMap;
20542058
use std::ops::Neg;
@@ -4243,8 +4247,11 @@ mod tests {
42434247
assert_batches_sorted_eq!(&expected, &actual);
42444248
}
42454249

4250+
#[rstest]
42464251
#[tokio::test]
4247-
async fn test_merge_with_schema_merge_and_pre_existing_struct_added_column() {
4252+
async fn test_merge_with_schema_merge_and_pre_existing_struct_added_column(
4253+
#[values(false, true)] safe_cast: bool,
4254+
) {
42484255
let table = setup_table(None).await;
42494256

42504257
let nested_schema = Arc::new(ArrowSchema::new(vec![Field::new(
@@ -4310,6 +4317,7 @@ mod tests {
43104317
.with_source_alias("source")
43114318
.with_target_alias("target")
43124319
.with_merge_schema(true)
4320+
.with_safe_cast(safe_cast)
43134321
.when_not_matched_insert(|insert| {
43144322
insert
43154323
.set("id", col("source.id"))
@@ -4340,7 +4348,11 @@ mod tests {
43404348

43414349
/// Merges a source row for id "B" whose `nested` struct has an extra field into the
43424350
/// struct table, which rewrites the file holding every target row.
4343-
async fn merge_widened_struct(update_nested: bool) -> DeltaTable {
4351+
async fn merge_widened_struct(
4352+
table: DeltaTable,
4353+
update_nested: bool,
4354+
safe_cast: bool,
4355+
) -> DeltaTable {
43444356
let nested_schema = Arc::new(ArrowSchema::new(vec![Field::new(
43454357
"count",
43464358
ArrowDataType::Int64,
@@ -4356,7 +4368,7 @@ mod tests {
43564368
true,
43574369
),
43584370
]));
4359-
let table = write_data_struct(setup_table(None).await, &schema).await;
4371+
let table = write_data_struct(table, &schema).await;
43604372

43614373
let nested_schema_source = Arc::new(ArrowSchema::new(vec![
43624374
Field::new("count", ArrowDataType::Int64, true),
@@ -4396,6 +4408,7 @@ mod tests {
43964408
.with_source_alias("source")
43974409
.with_target_alias("target")
43984410
.with_merge_schema(true)
4411+
.with_safe_cast(safe_cast)
43994412
.when_matched_update(|update| {
44004413
let update = update.update("value", col("source.value"));
44014414
if update_nested {
@@ -4418,9 +4431,12 @@ mod tests {
44184431
table
44194432
}
44204433

4434+
#[rstest]
44214435
#[tokio::test]
4422-
async fn test_merge_with_schema_merge_struct_added_field_keeps_copied_rows() {
4423-
let table = merge_widened_struct(true).await;
4436+
async fn test_merge_with_schema_merge_struct_added_field_keeps_copied_rows(
4437+
#[values(false, true)] safe_cast: bool,
4438+
) {
4439+
let table = merge_widened_struct(setup_table(None).await, true, safe_cast).await;
44244440
let expected = vec![
44254441
"+----+-------+------------+------------------------+",
44264442
"| id | value | modified | nested |",
@@ -4434,9 +4450,12 @@ mod tests {
44344450
assert_batches_sorted_eq!(&expected, &get_data(&table).await);
44354451
}
44364452

4453+
#[rstest]
44374454
#[tokio::test]
4438-
async fn test_merge_with_schema_merge_struct_added_field_keeps_unset_column() {
4439-
let table = merge_widened_struct(false).await;
4455+
async fn test_merge_with_schema_merge_struct_added_field_keeps_unset_column(
4456+
#[values(false, true)] safe_cast: bool,
4457+
) {
4458+
let table = merge_widened_struct(setup_table(None).await, false, safe_cast).await;
44404459
let expected = vec![
44414460
"+----+-------+------------+--------------------+",
44424461
"| id | value | modified | nested |",
@@ -4450,6 +4469,42 @@ mod tests {
44504469
assert_batches_sorted_eq!(&expected, &get_data(&table).await);
44514470
}
44524471

4472+
#[rstest]
4473+
#[tokio::test]
4474+
async fn test_merge_with_schema_merge_struct_added_field_cdf(
4475+
#[values(false, true)] safe_cast: bool,
4476+
) {
4477+
use crate::kernel::ProtocolInner;
4478+
4479+
let table = DeltaTable::new_in_memory()
4480+
.create()
4481+
.with_columns(get_delta_schema().fields().cloned())
4482+
.with_actions(vec![Action::Protocol(ProtocolInner::new(1, 4).as_kernel())])
4483+
.with_configuration_property(TableProperty::EnableChangeDataFeed, Some("true"))
4484+
.await
4485+
.unwrap();
4486+
let table = merge_widened_struct(table, true, safe_cast).await;
4487+
4488+
let ctx = SessionContext::new();
4489+
let cdf = table
4490+
.scan_cdf()
4491+
.with_starting_version(2)
4492+
.build(&ctx.state(), None)
4493+
.await
4494+
.unwrap();
4495+
let mut batches = collect(cdf, ctx.task_ctx()).await.unwrap();
4496+
// drop _commit_timestamp
4497+
let _: Vec<_> = batches.iter_mut().map(|b| b.remove_column(6)).collect();
4498+
assert_batches_sorted_eq! {[
4499+
"+----+-------+------------+------------------------+------------------+-----------------+",
4500+
"| id | value | modified | nested | _change_type | _commit_version |",
4501+
"+----+-------+------------+------------------------+------------------+-----------------+",
4502+
"| B | 10 | 2021-02-01 | {count: 2, name: } | update_preimage | 2 |",
4503+
"| B | 20 | 2021-02-01 | {count: 9, name: John} | update_postimage | 2 |",
4504+
"+----+-------+------------+------------------------+------------------+-----------------+",
4505+
], &batches }
4506+
}
4507+
44534508
#[tokio::test]
44544509
async fn test_merge_schema_evolution_simple_update() {
44554510
let (table, _) = setup().await;

‎python/tests/test_merge.py‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3320,6 +3320,51 @@ def test_merge_schema_safe_cast_existing_column_failed_cast_becomes_null(
33203320
}
33213321

33223322

3323+
@pytest.mark.pyarrow
3324+
@pytest.mark.parametrize("error_on_type_mismatch", (True, False))
3325+
def test_merge_schema_struct_added_field(
3326+
tmp_path: pathlib.Path, error_on_type_mismatch: bool
3327+
):
3328+
import pyarrow as pa
3329+
3330+
old = pa.struct([("x", pa.int64())])
3331+
new = pa.struct([("x", pa.int64()), ("y", pa.string())])
3332+
target = pa.table({"id": [1, 2], "s": pa.array([{"x": 1}, {"x": 2}], type=old)})
3333+
source = pa.table({"id": [1], "s": pa.array([{"x": 9, "y": "a"}], type=new)})
3334+
write_deltalake(
3335+
tmp_path, target, configuration={"delta.enableChangeDataFeed": "true"}
3336+
)
3337+
3338+
dt = DeltaTable(tmp_path)
3339+
(
3340+
dt.merge(
3341+
source=source,
3342+
predicate="t.id = s.id",
3343+
source_alias="s",
3344+
target_alias="t",
3345+
merge_schema=True,
3346+
error_on_type_mismatch=error_on_type_mismatch,
3347+
)
3348+
.when_matched_update_all()
3349+
.execute()
3350+
)
3351+
3352+
result = dt.to_pyarrow_table().sort_by([("id", "ascending")]).to_pydict()
3353+
assert result == {
3354+
"id": [1, 2],
3355+
"s": [{"x": 9, "y": "a"}, {"x": 2, "y": None}],
3356+
}
3357+
3358+
changes = dt.load_cdf(starting_version=1).read_all()
3359+
assert sorted(
3360+
zip(changes["_change_type"].to_pylist(), changes["s"].to_pylist()),
3361+
key=lambda c: c[0],
3362+
) == [
3363+
("update_postimage", {"x": 9, "y": "a"}),
3364+
("update_preimage", {"x": 1, "y": None}),
3365+
]
3366+
3367+
33233368
@pytest.mark.pyarrow
33243369
def test_merge_on_decimal_3033(tmp_path):
33253370
import pyarrow as pa

0 commit comments

Comments
 (0)