Skip to content

Commit 8d898c2

Browse files
committed
test: pin the Iceberg kernels to iceberg-rust's partition transforms
A partitioned write sorts on Comet's kernels and then groups by the partition values iceberg-rust computes, and the clustered writer fails at runtime when the two disagree. Assert they agree over the boundary inputs of every shared type, so an iceberg-rust bump that changes a transform breaks here first. The excluded cases are the interesting ones: iceberg-rust's `truncate` does not wrap like Java's, and its `years` / `months` go through Arrow's `date_part`, which honours the array's timezone tag. The last test pins that tag dependency, since it is the reason `years` and `months` keep local kernels while `bucket`, `days`, and `hours` could in principle be delegated.
1 parent 1622503 commit 8d898c2

1 file changed

Lines changed: 358 additions & 0 deletions

File tree

‎native/core/src/execution/operators/iceberg_write.rs‎

Lines changed: 358 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1172,3 +1172,361 @@ mod tests {
11721172
}
11731173
}
11741174
}
1175+
1176+
/// Pins Comet's Iceberg system-function kernels to iceberg-rust's partition transforms.
1177+
///
1178+
/// A partitioned write runs both: the sort in front of [`IcebergWriteExec`] is keyed on the
1179+
/// `datafusion-comet-spark-expr` kernels (Iceberg plans the sort as `bucket(...)`, `days(...)`,
1180+
/// ... system-function calls), while [`ClusteredWriter`] groups the sorted rows by the partition
1181+
/// values that [`PartitionValueCalculator`] computes with iceberg-rust's transforms. The writer
1182+
/// requires the two to agree: when they do not it fails at runtime with "The input is not sorted!
1183+
/// Cannot write to partition that was previously closed". These tests make an iceberg-rust bump
1184+
/// that changes a transform break here first.
1185+
#[cfg(test)]
1186+
mod iceberg_rust_transform_parity {
1187+
use arrow::array::{
1188+
ArrayRef, BinaryArray, Date32Array, Decimal128Array, Int32Array, Int64Array, StringArray,
1189+
TimestampMicrosecondArray,
1190+
};
1191+
use arrow::datatypes::{DataType, Field};
1192+
use datafusion::common::ScalarValue;
1193+
use datafusion::config::ConfigOptions;
1194+
use datafusion::logical_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl};
1195+
use datafusion_comet_spark_expr::{
1196+
SparkIcebergBucket, SparkIcebergTemporalTransform, SparkIcebergTruncate,
1197+
};
1198+
use iceberg::spec::Transform;
1199+
use iceberg::transform::create_transform_function;
1200+
use std::sync::Arc;
1201+
1202+
const MICROS_PER_DAY: i64 = 86_400_000_000;
1203+
1204+
/// Runs a Comet kernel over `value`, prepending `parameter` for the two-argument transforms.
1205+
fn comet(udf: &dyn ScalarUDFImpl, parameter: Option<i32>, value: &ArrayRef) -> ArrayRef {
1206+
let mut args: Vec<ColumnarValue> = parameter
1207+
.map(|p| ColumnarValue::Scalar(ScalarValue::Int32(Some(p))))
1208+
.into_iter()
1209+
.collect();
1210+
args.push(ColumnarValue::Array(Arc::clone(value)));
1211+
let arg_fields: Vec<_> = args
1212+
.iter()
1213+
.enumerate()
1214+
.map(|(i, a)| Arc::new(Field::new(format!("arg{i}"), a.data_type(), true)))
1215+
.collect();
1216+
let arg_types: Vec<DataType> = arg_fields.iter().map(|f| f.data_type().clone()).collect();
1217+
let return_type = udf.return_type(&arg_types).unwrap();
1218+
udf.invoke_with_args(ScalarFunctionArgs {
1219+
args,
1220+
arg_fields,
1221+
number_rows: value.len(),
1222+
return_field: Arc::new(Field::new(udf.name(), return_type, true)),
1223+
config_options: Arc::new(ConfigOptions::default()),
1224+
})
1225+
.unwrap()
1226+
.to_array(value.len())
1227+
.unwrap()
1228+
}
1229+
1230+
fn iceberg_rust(transform: Transform, value: &ArrayRef) -> ArrayRef {
1231+
create_transform_function(&transform)
1232+
.unwrap()
1233+
.transform(Arc::clone(value))
1234+
.unwrap()
1235+
}
1236+
1237+
fn assert_agree(label: &str, transform: Transform, udf: &dyn ScalarUDFImpl, value: &ArrayRef) {
1238+
let parameter = match transform {
1239+
Transform::Bucket(n) => Some(n as i32),
1240+
Transform::Truncate(w) => Some(w as i32),
1241+
_ => None,
1242+
};
1243+
assert_eq!(
1244+
comet(udf, parameter, value).as_ref(),
1245+
iceberg_rust(transform, value).as_ref(),
1246+
"{label} disagrees with iceberg-rust's {transform}"
1247+
);
1248+
}
1249+
1250+
fn timestamps(micros: Vec<Option<i64>>) -> Vec<(&'static str, ArrayRef)> {
1251+
// The two tags Comet can produce: `TimestampNTZType` is untagged and `TimestampType` is
1252+
// always tagged UTC.
1253+
vec![
1254+
(
1255+
"timestamp_ntz",
1256+
Arc::new(TimestampMicrosecondArray::from(micros.clone())) as ArrayRef,
1257+
),
1258+
(
1259+
"timestamp_utc",
1260+
Arc::new(TimestampMicrosecondArray::from(micros).with_timezone("UTC")) as ArrayRef,
1261+
),
1262+
]
1263+
}
1264+
1265+
/// Every type both sides accept. `Int8` and `Int16` are missing on purpose: Iceberg binds
1266+
/// tinyint and smallint to `BucketInt`, iceberg-rust has no arm for them, and Comet's kernel
1267+
/// widens them to the same 8 little-endian bytes that the `Int32` case pins here.
1268+
#[test]
1269+
fn bucket_agrees_with_iceberg_rust() {
1270+
let mut inputs: Vec<(&str, ArrayRef)> = vec![
1271+
(
1272+
"int",
1273+
Arc::new(Int32Array::from(vec![
1274+
Some(i32::MIN),
1275+
Some(-1),
1276+
Some(0),
1277+
Some(34),
1278+
Some(i32::MAX),
1279+
None,
1280+
])),
1281+
),
1282+
(
1283+
"long",
1284+
Arc::new(Int64Array::from(vec![
1285+
Some(i64::MIN),
1286+
Some(-1),
1287+
Some(0),
1288+
Some(34),
1289+
Some(i64::MAX),
1290+
None,
1291+
])),
1292+
),
1293+
(
1294+
"date",
1295+
Arc::new(Date32Array::from(vec![
1296+
Some(i32::MIN),
1297+
Some(-1),
1298+
Some(0),
1299+
Some(17_486),
1300+
Some(i32::MAX),
1301+
None,
1302+
])),
1303+
),
1304+
(
1305+
"decimal",
1306+
Arc::new(
1307+
Decimal128Array::from(vec![
1308+
Some(-(10i128.pow(38) - 1)),
1309+
Some(-129),
1310+
Some(0),
1311+
Some(1420),
1312+
Some(10i128.pow(38) - 1),
1313+
None,
1314+
])
1315+
.with_precision_and_scale(38, 10)
1316+
.unwrap(),
1317+
),
1318+
),
1319+
(
1320+
"string",
1321+
Arc::new(StringArray::from(vec![
1322+
Some(""),
1323+
Some("a"),
1324+
Some("iceberg"),
1325+
Some("日本語😀"),
1326+
None,
1327+
])),
1328+
),
1329+
(
1330+
"binary",
1331+
Arc::new(BinaryArray::from(vec![
1332+
Some([].as_slice()),
1333+
Some([0u8, 1, 2, 3].as_slice()),
1334+
Some([0xffu8; 9].as_slice()),
1335+
None,
1336+
])),
1337+
),
1338+
];
1339+
inputs.extend(timestamps(vec![
1340+
Some(i64::MIN),
1341+
Some(-1),
1342+
Some(0),
1343+
Some(1_510_871_468_000_000),
1344+
Some(i64::MAX),
1345+
None,
1346+
]));
1347+
1348+
let udf = SparkIcebergBucket::new();
1349+
for num_buckets in [1u32, 7, 16, i32::MAX as u32] {
1350+
for (label, input) in &inputs {
1351+
assert_agree(
1352+
&format!("bucket({num_buckets}, {label})"),
1353+
Transform::Bucket(num_buckets),
1354+
&udf,
1355+
input,
1356+
);
1357+
}
1358+
}
1359+
}
1360+
1361+
/// `i32::MIN`, `i64::MIN`, and widths above 2^30 are left out: Java's `TruncateUtil` wraps
1362+
/// there and iceberg-rust does not (`truncate_i32` uses `rem_euclid`, `truncate_i64` and the
1363+
/// decimal kernel subtract without wrapping and overflow in a debug build). That gap is an
1364+
/// iceberg-rust bug affecting the writer's own partition values, independent of these
1365+
/// kernels; the wrapping cases are pinned against the JVM in the kernel's own unit tests.
1366+
#[test]
1367+
fn truncate_agrees_with_iceberg_rust() {
1368+
let inputs: Vec<(&str, ArrayRef)> = vec![
1369+
(
1370+
"int",
1371+
Arc::new(Int32Array::from(vec![
1372+
Some(i32::MIN + 1_000_000),
1373+
Some(-1),
1374+
Some(0),
1375+
Some(1),
1376+
Some(i32::MAX - 1_000_000),
1377+
None,
1378+
])),
1379+
),
1380+
(
1381+
"long",
1382+
Arc::new(Int64Array::from(vec![
1383+
Some(i64::MIN + 1_000_000),
1384+
Some(-1),
1385+
Some(0),
1386+
Some(1),
1387+
Some(i64::MAX - 1_000_000),
1388+
None,
1389+
])),
1390+
),
1391+
(
1392+
"decimal",
1393+
Arc::new(
1394+
Decimal128Array::from(vec![Some(-1065), Some(0), Some(1065), None])
1395+
.with_precision_and_scale(18, 2)
1396+
.unwrap(),
1397+
),
1398+
),
1399+
(
1400+
"string",
1401+
Arc::new(StringArray::from(vec![
1402+
Some(""),
1403+
Some("ic"),
1404+
Some("iceberg"),
1405+
Some("日本語テキスト"),
1406+
Some("a😀b😀c"),
1407+
None,
1408+
])),
1409+
),
1410+
(
1411+
"binary",
1412+
Arc::new(BinaryArray::from(vec![
1413+
Some([].as_slice()),
1414+
Some([1u8].as_slice()),
1415+
Some([1u8, 2, 3, 4, 5].as_slice()),
1416+
None,
1417+
])),
1418+
),
1419+
];
1420+
1421+
let udf = SparkIcebergTruncate::new();
1422+
for width in [1u32, 3, 10, 1000, 1 << 30] {
1423+
for (label, input) in &inputs {
1424+
assert_agree(
1425+
&format!("truncate({width}, {label})"),
1426+
Transform::Truncate(width),
1427+
&udf,
1428+
input,
1429+
);
1430+
}
1431+
}
1432+
}
1433+
1434+
/// `days` and `hours` are plain floor division on both sides, so the whole domain agrees.
1435+
#[test]
1436+
fn days_and_hours_agree_with_iceberg_rust() {
1437+
let micros = vec![
1438+
Some(0),
1439+
Some(-1),
1440+
Some(-MICROS_PER_DAY),
1441+
Some(-MICROS_PER_DAY - 1),
1442+
Some(1_510_871_468_000_000),
1443+
Some(365 * MICROS_PER_DAY - 1),
1444+
None,
1445+
];
1446+
let days_udf = SparkIcebergTemporalTransform::days();
1447+
let hours_udf = SparkIcebergTemporalTransform::hours();
1448+
for (label, input) in timestamps(micros) {
1449+
assert_agree(&format!("days({label})"), Transform::Day, &days_udf, &input);
1450+
assert_agree(
1451+
&format!("hours({label})"),
1452+
Transform::Hour,
1453+
&hours_udf,
1454+
&input,
1455+
);
1456+
}
1457+
let dates: ArrayRef = Arc::new(Date32Array::from(vec![
1458+
Some(i32::MIN),
1459+
Some(-366),
1460+
Some(0),
1461+
Some(17_486),
1462+
Some(i32::MAX),
1463+
None,
1464+
]));
1465+
assert_agree("days(date)", Transform::Day, &days_udf, &dates);
1466+
}
1467+
1468+
/// `years` and `months` agree over the dates iceberg-rust can represent -- it splits the
1469+
/// calendar with `chrono`, so anything past year 262143 errors there while Comet and the JVM
1470+
/// keep going (see the kernel's own unit tests for those).
1471+
#[test]
1472+
fn years_and_months_agree_with_iceberg_rust_within_its_range() {
1473+
let years_udf = SparkIcebergTemporalTransform::years();
1474+
let months_udf = SparkIcebergTemporalTransform::months();
1475+
let dates: ArrayRef = Arc::new(Date32Array::from(vec![
1476+
Some(-100_000),
1477+
Some(-366),
1478+
Some(-365),
1479+
Some(-1),
1480+
Some(0),
1481+
Some(30),
1482+
Some(17_486),
1483+
Some(100_000),
1484+
None,
1485+
]));
1486+
assert_agree("years(date)", Transform::Year, &years_udf, &dates);
1487+
assert_agree("months(date)", Transform::Month, &months_udf, &dates);
1488+
for (label, input) in timestamps(vec![
1489+
Some(-100_000 * MICROS_PER_DAY),
1490+
Some(-1),
1491+
Some(0),
1492+
Some(1_510_871_468_000_000),
1493+
None,
1494+
]) {
1495+
assert_agree(
1496+
&format!("years({label})"),
1497+
Transform::Year,
1498+
&years_udf,
1499+
&input,
1500+
);
1501+
assert_agree(
1502+
&format!("months({label})"),
1503+
Transform::Month,
1504+
&months_udf,
1505+
&input,
1506+
);
1507+
}
1508+
}
1509+
1510+
/// Why `years` and `months` are not delegated to iceberg-rust even though `bucket`, `days`,
1511+
/// and `hours` could be: its kernels go through Arrow's `date_part`, which honours the
1512+
/// array's timezone tag, while Iceberg's Java `DateTimeUtil` is always UTC. Comet only ever
1513+
/// produces `UTC` and untagged timestamps today, so the parity above holds; this pins the
1514+
/// reason the local kernel exists. If this ever fails, iceberg-rust dropped the tag
1515+
/// dependency and delegating becomes safe.
1516+
#[test]
1517+
fn iceberg_rust_years_follow_the_timezone_tag() {
1518+
// 1969-12-31T23:59:59.999999Z, which is 1970-01-01T05:44:59.999999 in Kathmandu.
1519+
let tagged: ArrayRef =
1520+
Arc::new(TimestampMicrosecondArray::from(vec![-1i64]).with_timezone("Asia/Kathmandu"));
1521+
let comet_years = comet(&SparkIcebergTemporalTransform::years(), None, &tagged);
1522+
let iceberg_years = iceberg_rust(Transform::Year, &tagged);
1523+
assert_eq!(
1524+
comet_years.as_ref(),
1525+
&Int32Array::from(vec![-1]) as &dyn arrow::array::Array
1526+
);
1527+
assert_eq!(
1528+
iceberg_years.as_ref(),
1529+
&Int32Array::from(vec![0]) as &dyn arrow::array::Array
1530+
);
1531+
}
1532+
}

0 commit comments

Comments
 (0)