@@ -1172,3 +1172,361 @@ mod tests {
11721172 }
11731173 }
11741174}
1175+
1176+ /// Pins Comet's Iceberg system-function kernels to iceberg-rust's partition transforms.
1177+ ///
1178+ /// A partitioned write runs both: the sort in front of [`IcebergWriteExec`] is keyed on the
1179+ /// `datafusion-comet-spark-expr` kernels (Iceberg plans the sort as `bucket(...)`, `days(...)`,
1180+ /// ... system-function calls), while [`ClusteredWriter`] groups the sorted rows by the partition
1181+ /// values that [`PartitionValueCalculator`] computes with iceberg-rust's transforms. The writer
1182+ /// requires the two to agree: when they do not it fails at runtime with "The input is not sorted!
1183+ /// Cannot write to partition that was previously closed". These tests make an iceberg-rust bump
1184+ /// that changes a transform break here first.
1185+ #[ cfg( test) ]
1186+ mod iceberg_rust_transform_parity {
1187+ use arrow:: array:: {
1188+ ArrayRef , BinaryArray , Date32Array , Decimal128Array , Int32Array , Int64Array , StringArray ,
1189+ TimestampMicrosecondArray ,
1190+ } ;
1191+ use arrow:: datatypes:: { DataType , Field } ;
1192+ use datafusion:: common:: ScalarValue ;
1193+ use datafusion:: config:: ConfigOptions ;
1194+ use datafusion:: logical_expr:: { ColumnarValue , ScalarFunctionArgs , ScalarUDFImpl } ;
1195+ use datafusion_comet_spark_expr:: {
1196+ SparkIcebergBucket , SparkIcebergTemporalTransform , SparkIcebergTruncate ,
1197+ } ;
1198+ use iceberg:: spec:: Transform ;
1199+ use iceberg:: transform:: create_transform_function;
1200+ use std:: sync:: Arc ;
1201+
1202+ const MICROS_PER_DAY : i64 = 86_400_000_000 ;
1203+
1204+ /// Runs a Comet kernel over `value`, prepending `parameter` for the two-argument transforms.
1205+ fn comet ( udf : & dyn ScalarUDFImpl , parameter : Option < i32 > , value : & ArrayRef ) -> ArrayRef {
1206+ let mut args: Vec < ColumnarValue > = parameter
1207+ . map ( |p| ColumnarValue :: Scalar ( ScalarValue :: Int32 ( Some ( p) ) ) )
1208+ . into_iter ( )
1209+ . collect ( ) ;
1210+ args. push ( ColumnarValue :: Array ( Arc :: clone ( value) ) ) ;
1211+ let arg_fields: Vec < _ > = args
1212+ . iter ( )
1213+ . enumerate ( )
1214+ . map ( |( i, a) | Arc :: new ( Field :: new ( format ! ( "arg{i}" ) , a. data_type ( ) , true ) ) )
1215+ . collect ( ) ;
1216+ let arg_types: Vec < DataType > = arg_fields. iter ( ) . map ( |f| f. data_type ( ) . clone ( ) ) . collect ( ) ;
1217+ let return_type = udf. return_type ( & arg_types) . unwrap ( ) ;
1218+ udf. invoke_with_args ( ScalarFunctionArgs {
1219+ args,
1220+ arg_fields,
1221+ number_rows : value. len ( ) ,
1222+ return_field : Arc :: new ( Field :: new ( udf. name ( ) , return_type, true ) ) ,
1223+ config_options : Arc :: new ( ConfigOptions :: default ( ) ) ,
1224+ } )
1225+ . unwrap ( )
1226+ . to_array ( value. len ( ) )
1227+ . unwrap ( )
1228+ }
1229+
1230+ fn iceberg_rust ( transform : Transform , value : & ArrayRef ) -> ArrayRef {
1231+ create_transform_function ( & transform)
1232+ . unwrap ( )
1233+ . transform ( Arc :: clone ( value) )
1234+ . unwrap ( )
1235+ }
1236+
1237+ fn assert_agree ( label : & str , transform : Transform , udf : & dyn ScalarUDFImpl , value : & ArrayRef ) {
1238+ let parameter = match transform {
1239+ Transform :: Bucket ( n) => Some ( n as i32 ) ,
1240+ Transform :: Truncate ( w) => Some ( w as i32 ) ,
1241+ _ => None ,
1242+ } ;
1243+ assert_eq ! (
1244+ comet( udf, parameter, value) . as_ref( ) ,
1245+ iceberg_rust( transform, value) . as_ref( ) ,
1246+ "{label} disagrees with iceberg-rust's {transform}"
1247+ ) ;
1248+ }
1249+
1250+ fn timestamps ( micros : Vec < Option < i64 > > ) -> Vec < ( & ' static str , ArrayRef ) > {
1251+ // The two tags Comet can produce: `TimestampNTZType` is untagged and `TimestampType` is
1252+ // always tagged UTC.
1253+ vec ! [
1254+ (
1255+ "timestamp_ntz" ,
1256+ Arc :: new( TimestampMicrosecondArray :: from( micros. clone( ) ) ) as ArrayRef ,
1257+ ) ,
1258+ (
1259+ "timestamp_utc" ,
1260+ Arc :: new( TimestampMicrosecondArray :: from( micros) . with_timezone( "UTC" ) ) as ArrayRef ,
1261+ ) ,
1262+ ]
1263+ }
1264+
1265+ /// Every type both sides accept. `Int8` and `Int16` are missing on purpose: Iceberg binds
1266+ /// tinyint and smallint to `BucketInt`, iceberg-rust has no arm for them, and Comet's kernel
1267+ /// widens them to the same 8 little-endian bytes that the `Int32` case pins here.
1268+ #[ test]
1269+ fn bucket_agrees_with_iceberg_rust ( ) {
1270+ let mut inputs: Vec < ( & str , ArrayRef ) > = vec ! [
1271+ (
1272+ "int" ,
1273+ Arc :: new( Int32Array :: from( vec![
1274+ Some ( i32 :: MIN ) ,
1275+ Some ( -1 ) ,
1276+ Some ( 0 ) ,
1277+ Some ( 34 ) ,
1278+ Some ( i32 :: MAX ) ,
1279+ None ,
1280+ ] ) ) ,
1281+ ) ,
1282+ (
1283+ "long" ,
1284+ Arc :: new( Int64Array :: from( vec![
1285+ Some ( i64 :: MIN ) ,
1286+ Some ( -1 ) ,
1287+ Some ( 0 ) ,
1288+ Some ( 34 ) ,
1289+ Some ( i64 :: MAX ) ,
1290+ None ,
1291+ ] ) ) ,
1292+ ) ,
1293+ (
1294+ "date" ,
1295+ Arc :: new( Date32Array :: from( vec![
1296+ Some ( i32 :: MIN ) ,
1297+ Some ( -1 ) ,
1298+ Some ( 0 ) ,
1299+ Some ( 17_486 ) ,
1300+ Some ( i32 :: MAX ) ,
1301+ None ,
1302+ ] ) ) ,
1303+ ) ,
1304+ (
1305+ "decimal" ,
1306+ Arc :: new(
1307+ Decimal128Array :: from( vec![
1308+ Some ( -( 10i128 . pow( 38 ) - 1 ) ) ,
1309+ Some ( -129 ) ,
1310+ Some ( 0 ) ,
1311+ Some ( 1420 ) ,
1312+ Some ( 10i128 . pow( 38 ) - 1 ) ,
1313+ None ,
1314+ ] )
1315+ . with_precision_and_scale( 38 , 10 )
1316+ . unwrap( ) ,
1317+ ) ,
1318+ ) ,
1319+ (
1320+ "string" ,
1321+ Arc :: new( StringArray :: from( vec![
1322+ Some ( "" ) ,
1323+ Some ( "a" ) ,
1324+ Some ( "iceberg" ) ,
1325+ Some ( "日本語😀" ) ,
1326+ None ,
1327+ ] ) ) ,
1328+ ) ,
1329+ (
1330+ "binary" ,
1331+ Arc :: new( BinaryArray :: from( vec![
1332+ Some ( [ ] . as_slice( ) ) ,
1333+ Some ( [ 0u8 , 1 , 2 , 3 ] . as_slice( ) ) ,
1334+ Some ( [ 0xffu8 ; 9 ] . as_slice( ) ) ,
1335+ None ,
1336+ ] ) ) ,
1337+ ) ,
1338+ ] ;
1339+ inputs. extend ( timestamps ( vec ! [
1340+ Some ( i64 :: MIN ) ,
1341+ Some ( -1 ) ,
1342+ Some ( 0 ) ,
1343+ Some ( 1_510_871_468_000_000 ) ,
1344+ Some ( i64 :: MAX ) ,
1345+ None ,
1346+ ] ) ) ;
1347+
1348+ let udf = SparkIcebergBucket :: new ( ) ;
1349+ for num_buckets in [ 1u32 , 7 , 16 , i32:: MAX as u32 ] {
1350+ for ( label, input) in & inputs {
1351+ assert_agree (
1352+ & format ! ( "bucket({num_buckets}, {label})" ) ,
1353+ Transform :: Bucket ( num_buckets) ,
1354+ & udf,
1355+ input,
1356+ ) ;
1357+ }
1358+ }
1359+ }
1360+
1361+ /// `i32::MIN`, `i64::MIN`, and widths above 2^30 are left out: Java's `TruncateUtil` wraps
1362+ /// there and iceberg-rust does not (`truncate_i32` uses `rem_euclid`, `truncate_i64` and the
1363+ /// decimal kernel subtract without wrapping and overflow in a debug build). That gap is an
1364+ /// iceberg-rust bug affecting the writer's own partition values, independent of these
1365+ /// kernels; the wrapping cases are pinned against the JVM in the kernel's own unit tests.
1366+ #[ test]
1367+ fn truncate_agrees_with_iceberg_rust ( ) {
1368+ let inputs: Vec < ( & str , ArrayRef ) > = vec ! [
1369+ (
1370+ "int" ,
1371+ Arc :: new( Int32Array :: from( vec![
1372+ Some ( i32 :: MIN + 1_000_000 ) ,
1373+ Some ( -1 ) ,
1374+ Some ( 0 ) ,
1375+ Some ( 1 ) ,
1376+ Some ( i32 :: MAX - 1_000_000 ) ,
1377+ None ,
1378+ ] ) ) ,
1379+ ) ,
1380+ (
1381+ "long" ,
1382+ Arc :: new( Int64Array :: from( vec![
1383+ Some ( i64 :: MIN + 1_000_000 ) ,
1384+ Some ( -1 ) ,
1385+ Some ( 0 ) ,
1386+ Some ( 1 ) ,
1387+ Some ( i64 :: MAX - 1_000_000 ) ,
1388+ None ,
1389+ ] ) ) ,
1390+ ) ,
1391+ (
1392+ "decimal" ,
1393+ Arc :: new(
1394+ Decimal128Array :: from( vec![ Some ( -1065 ) , Some ( 0 ) , Some ( 1065 ) , None ] )
1395+ . with_precision_and_scale( 18 , 2 )
1396+ . unwrap( ) ,
1397+ ) ,
1398+ ) ,
1399+ (
1400+ "string" ,
1401+ Arc :: new( StringArray :: from( vec![
1402+ Some ( "" ) ,
1403+ Some ( "ic" ) ,
1404+ Some ( "iceberg" ) ,
1405+ Some ( "日本語テキスト" ) ,
1406+ Some ( "a😀b😀c" ) ,
1407+ None ,
1408+ ] ) ) ,
1409+ ) ,
1410+ (
1411+ "binary" ,
1412+ Arc :: new( BinaryArray :: from( vec![
1413+ Some ( [ ] . as_slice( ) ) ,
1414+ Some ( [ 1u8 ] . as_slice( ) ) ,
1415+ Some ( [ 1u8 , 2 , 3 , 4 , 5 ] . as_slice( ) ) ,
1416+ None ,
1417+ ] ) ) ,
1418+ ) ,
1419+ ] ;
1420+
1421+ let udf = SparkIcebergTruncate :: new ( ) ;
1422+ for width in [ 1u32 , 3 , 10 , 1000 , 1 << 30 ] {
1423+ for ( label, input) in & inputs {
1424+ assert_agree (
1425+ & format ! ( "truncate({width}, {label})" ) ,
1426+ Transform :: Truncate ( width) ,
1427+ & udf,
1428+ input,
1429+ ) ;
1430+ }
1431+ }
1432+ }
1433+
1434+ /// `days` and `hours` are plain floor division on both sides, so the whole domain agrees.
1435+ #[ test]
1436+ fn days_and_hours_agree_with_iceberg_rust ( ) {
1437+ let micros = vec ! [
1438+ Some ( 0 ) ,
1439+ Some ( -1 ) ,
1440+ Some ( -MICROS_PER_DAY ) ,
1441+ Some ( -MICROS_PER_DAY - 1 ) ,
1442+ Some ( 1_510_871_468_000_000 ) ,
1443+ Some ( 365 * MICROS_PER_DAY - 1 ) ,
1444+ None ,
1445+ ] ;
1446+ let days_udf = SparkIcebergTemporalTransform :: days ( ) ;
1447+ let hours_udf = SparkIcebergTemporalTransform :: hours ( ) ;
1448+ for ( label, input) in timestamps ( micros) {
1449+ assert_agree ( & format ! ( "days({label})" ) , Transform :: Day , & days_udf, & input) ;
1450+ assert_agree (
1451+ & format ! ( "hours({label})" ) ,
1452+ Transform :: Hour ,
1453+ & hours_udf,
1454+ & input,
1455+ ) ;
1456+ }
1457+ let dates: ArrayRef = Arc :: new ( Date32Array :: from ( vec ! [
1458+ Some ( i32 :: MIN ) ,
1459+ Some ( -366 ) ,
1460+ Some ( 0 ) ,
1461+ Some ( 17_486 ) ,
1462+ Some ( i32 :: MAX ) ,
1463+ None ,
1464+ ] ) ) ;
1465+ assert_agree ( "days(date)" , Transform :: Day , & days_udf, & dates) ;
1466+ }
1467+
1468+ /// `years` and `months` agree over the dates iceberg-rust can represent -- it splits the
1469+ /// calendar with `chrono`, so anything past year 262143 errors there while Comet and the JVM
1470+ /// keep going (see the kernel's own unit tests for those).
1471+ #[ test]
1472+ fn years_and_months_agree_with_iceberg_rust_within_its_range ( ) {
1473+ let years_udf = SparkIcebergTemporalTransform :: years ( ) ;
1474+ let months_udf = SparkIcebergTemporalTransform :: months ( ) ;
1475+ let dates: ArrayRef = Arc :: new ( Date32Array :: from ( vec ! [
1476+ Some ( -100_000 ) ,
1477+ Some ( -366 ) ,
1478+ Some ( -365 ) ,
1479+ Some ( -1 ) ,
1480+ Some ( 0 ) ,
1481+ Some ( 30 ) ,
1482+ Some ( 17_486 ) ,
1483+ Some ( 100_000 ) ,
1484+ None ,
1485+ ] ) ) ;
1486+ assert_agree ( "years(date)" , Transform :: Year , & years_udf, & dates) ;
1487+ assert_agree ( "months(date)" , Transform :: Month , & months_udf, & dates) ;
1488+ for ( label, input) in timestamps ( vec ! [
1489+ Some ( -100_000 * MICROS_PER_DAY ) ,
1490+ Some ( -1 ) ,
1491+ Some ( 0 ) ,
1492+ Some ( 1_510_871_468_000_000 ) ,
1493+ None ,
1494+ ] ) {
1495+ assert_agree (
1496+ & format ! ( "years({label})" ) ,
1497+ Transform :: Year ,
1498+ & years_udf,
1499+ & input,
1500+ ) ;
1501+ assert_agree (
1502+ & format ! ( "months({label})" ) ,
1503+ Transform :: Month ,
1504+ & months_udf,
1505+ & input,
1506+ ) ;
1507+ }
1508+ }
1509+
1510+ /// Why `years` and `months` are not delegated to iceberg-rust even though `bucket`, `days`,
1511+ /// and `hours` could be: its kernels go through Arrow's `date_part`, which honours the
1512+ /// array's timezone tag, while Iceberg's Java `DateTimeUtil` is always UTC. Comet only ever
1513+ /// produces `UTC` and untagged timestamps today, so the parity above holds; this pins the
1514+ /// reason the local kernel exists. If this ever fails, iceberg-rust dropped the tag
1515+ /// dependency and delegating becomes safe.
1516+ #[ test]
1517+ fn iceberg_rust_years_follow_the_timezone_tag ( ) {
1518+ // 1969-12-31T23:59:59.999999Z, which is 1970-01-01T05:44:59.999999 in Kathmandu.
1519+ let tagged: ArrayRef =
1520+ Arc :: new ( TimestampMicrosecondArray :: from ( vec ! [ -1i64 ] ) . with_timezone ( "Asia/Kathmandu" ) ) ;
1521+ let comet_years = comet ( & SparkIcebergTemporalTransform :: years ( ) , None , & tagged) ;
1522+ let iceberg_years = iceberg_rust ( Transform :: Year , & tagged) ;
1523+ assert_eq ! (
1524+ comet_years. as_ref( ) ,
1525+ & Int32Array :: from( vec![ -1 ] ) as & dyn arrow:: array:: Array
1526+ ) ;
1527+ assert_eq ! (
1528+ iceberg_years. as_ref( ) ,
1529+ & Int32Array :: from( vec![ 0 ] ) as & dyn arrow:: array:: Array
1530+ ) ;
1531+ }
1532+ }
0 commit comments