|
15 | 15 | //! Preflight validation of persisted partition expressions against an ALTER TABLE schema change. |
16 | 16 | //! |
17 | 17 | //! Persisted partition bounds keep the types they were written with. When a column |
18 | | -//! referenced by a partition expression changes its type, the region rewrites the bound |
19 | | -//! literal to the new column type while building the partition predicate on scan. If a |
20 | | -//! bound cannot be represented with the new type (a classic example is a millisecond |
21 | | -//! timestamp bound that overflows when the time index column is widened to nanoseconds), |
22 | | -//! every scan of that region fails at that point. Such an ALTER must be rejected before |
23 | | -//! it is submitted to any region, otherwise already-altered regions keep failing while |
24 | | -//! the rest of the table is untouched. |
| 18 | +//! referenced by a partition expression changes its type, the region coerces the bound |
| 19 | +//! literal and the new column type to a common comparison type while building the |
| 20 | +//! partition predicate on scan. If no common type exists, or the bound cannot be cast |
| 21 | +//! to it (a classic example is a millisecond timestamp bound that overflows when the |
| 22 | +//! time index column is widened to nanoseconds), every scan of that region fails at |
| 23 | +//! that point. Such an ALTER must be rejected before it is submitted to any region, |
| 24 | +//! otherwise already-altered regions keep failing while the rest of the table is |
| 25 | +//! untouched. |
25 | 26 | //! |
26 | 27 | //! The `partition` crate depends on `common-meta`, so this crate cannot depend on |
27 | 28 | //! `partition` back and cannot reuse [`partition::expr::PartitionExpr`]. The `Persisted*` |
28 | 29 | //! types below mirror the JSON representation of `partition::partition::PartitionBound` and |
29 | 30 | //! `partition::expr::PartitionExpr` and must be kept in sync with `partition/src/partition.rs` |
30 | | -//! and `partition/src/expr.rs`. The compatibility check itself deliberately repeats the |
31 | | -//! same conversion (value -> [`ScalarValue`]) and the same checked cast |
32 | | -//! ([`ScalarValue::cast_to`]) that the region performs when it coerces the persisted bound |
33 | | -//! to the new column type, so a bound that would fail on scan is rejected here as well. |
| 31 | +//! and `partition/src/expr.rs`; round-trip tests in `partition` run real serialized |
| 32 | +//! expressions through this validator so drift fails in CI. The compatibility check itself |
| 33 | +//! deliberately repeats the same conversion (value -> [`ScalarValue`]) and the same |
| 34 | +//! comparison coercion the region performs when it builds the partition predicate |
| 35 | +//! (`ExprSimplifier::coerce` / [`comparison_coercion`]): both sides are cast to the common |
| 36 | +//! comparison type, and the persisted bound must be castable to that type with |
| 37 | +//! [`ScalarValue::cast_to`]. A bound that would fail on scan is rejected here as well. |
34 | 38 |
|
35 | 39 | use std::collections::HashMap; |
36 | 40 |
|
37 | 41 | use common_telemetry::warn; |
38 | 42 | use datafusion_common::ScalarValue; |
| 43 | +use datafusion_expr::type_coercion::binary::comparison_coercion; |
39 | 44 | use datatypes::data_type::{ConcreteDataType, DataType}; |
40 | 45 | use datatypes::schema::Schema; |
41 | 46 | use datatypes::value::{ |
@@ -111,7 +116,10 @@ pub(crate) fn changed_column_types( |
111 | 116 | /// Validates that the persisted partition bounds of every region are still representable |
112 | 117 | /// with the types of `changed_columns`. Returns an error naming the first offending |
113 | 118 | /// region without modifying anything. |
114 | | -pub(crate) fn validate_region_partition_bounds( |
| 119 | +/// |
| 120 | +/// Public so that round-trip tests in the `partition` crate can run real serialized |
| 121 | +/// partition expressions through the same mirror used here. |
| 122 | +pub fn validate_region_partition_bounds( |
115 | 123 | changed_columns: &HashMap<String, ConcreteDataType>, |
116 | 124 | region_routes: &[RegionRoute], |
117 | 125 | ) -> Result<()> { |
@@ -205,11 +213,29 @@ fn validate_comparison( |
205 | 213 | return Ok(()); |
206 | 214 | }; |
207 | 215 |
|
| 216 | + // Mirror `ExprSimplifier::coerce` in `PredicateGroup::new`: the region casts |
| 217 | + // both the column and the persisted bound to the common comparison type. The |
| 218 | + // bound is not cast to the column type itself, so narrowing a column (e.g. |
| 219 | + // BIGINT to INT with an out-of-range bound) is fine as long as the bound |
| 220 | + // survives the common type, while a bound that cannot be cast to it — or a |
| 221 | + // pair of types without a common comparison type — fails every scan. |
208 | 222 | let target_arrow_type = target_type.as_arrow_type(); |
209 | | - if scalar.data_type() == target_arrow_type { |
210 | | - return Ok(()); |
211 | | - } |
212 | | - if let Err(error) = scalar.cast_to(&target_arrow_type) { |
| 223 | + let Some(common_type) = comparison_coercion(&target_arrow_type, &scalar.data_type()) else { |
| 224 | + warn!( |
| 225 | + "Persisted partition bound has no common comparison type with the altered \ |
| 226 | + column type, region_id: {}, column: '{}', target_type: {}", |
| 227 | + region_id, column, target_type |
| 228 | + ); |
| 229 | + return PartitionExprIncompatibleSnafu { |
| 230 | + region_id, |
| 231 | + column: column.clone(), |
| 232 | + target_type: target_type.to_string(), |
| 233 | + } |
| 234 | + .fail(); |
| 235 | + }; |
| 236 | + if scalar.data_type() != common_type |
| 237 | + && let Err(error) = scalar.cast_to(&common_type) |
| 238 | + { |
213 | 239 | warn!( |
214 | 240 | error; |
215 | 241 | "Persisted partition bound is not representable with the altered column type, \ |
@@ -408,4 +434,31 @@ mod tests { |
408 | 434 | ) |
409 | 435 | .unwrap(); |
410 | 436 | } |
| 437 | + |
| 438 | + #[test] |
| 439 | + fn test_field_narrowing_keeps_bounds_the_scan_coerces() { |
| 440 | + // BIGINT bound outside the INT range with the column narrowed to INT: the |
| 441 | + // region casts the column up to the common Int64 comparison type, so the |
| 442 | + // bound still scans and the ALTER must not be rejected. |
| 443 | + let bound = |
| 444 | + r#"{"Expr":{"lhs":{"Column":"v"},"op":"Lt","rhs":{"Value":{"Int64":3000000000}}}}"#; |
| 445 | + validate_region_partition_bounds( |
| 446 | + &changed("v", ConcreteDataType::int32_datatype()), |
| 447 | + &routes(&[bound]), |
| 448 | + ) |
| 449 | + .unwrap(); |
| 450 | + } |
| 451 | + |
| 452 | + #[test] |
| 453 | + fn test_bound_without_common_comparison_type_is_rejected() { |
| 454 | + // No comparison coercion exists between a boolean bound and an int column, |
| 455 | + // so the region's predicate build fails and the ALTER must be rejected. |
| 456 | + let bound = r#"{"Expr":{"lhs":{"Column":"v"},"op":"Lt","rhs":{"Value":{"Boolean":true}}}}"#; |
| 457 | + let err = validate_region_partition_bounds( |
| 458 | + &changed("v", ConcreteDataType::int32_datatype()), |
| 459 | + &routes(&[bound]), |
| 460 | + ) |
| 461 | + .unwrap_err(); |
| 462 | + assert!(err.to_string().contains("'v'"), "unexpected error: {err}"); |
| 463 | + } |
411 | 464 | } |
0 commit comments