Skip to content

Commit 8e18e95

Browse files
committed
fix: compare literals with negative-scale decimal columns without panicking (#145)
Arrow allows a negative column scale. Against a scale-38 literal, a decimal128(38, -1) column is 39 digits coarser, and fit_decimal raised 10 to that gap and panicked. A gap past i128 now places the literal within one step of zero: 0 or -1 as the floor, exact only for zero. Below 38 widened digits, a negative-scale column had a second panic. DataFusion's unwrap_cast simplifier raises 10 to the column's scale as an unsigned power (apache/datafusion#24896). Typed Decimal literals started reaching it (`main` sent them as strings, which failed with a ValueError), and integer literals already panicked on `main`. A comparison or is_in against a negative-scale column now gives Decimal and integer literals the column's own type, so DataFusion never casts the column. Float literals still reach the simplifier and panic as on `main`; that is #226, and the grid marks those cells strict xfail.
1 parent 8910622 commit 8e18e95

4 files changed

Lines changed: 119 additions & 18 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ Literal values in lambdas are typed (#145). They used to be sent to the engine a
1818
### Fixed
1919

2020
- `Decimal` literals: compared with a float column as that float, instead of casting the column to decimal (which failed on NaN and infinity); compared exactly with decimal and integer columns, also when the result would need more than 38 digits; usable in arithmetic.
21+
- An integer or `Decimal` literal compared with a decimal column of negative scale (`decimal128(5, -2)`) is compared exactly; an integer literal used to panic in DataFusion's simplifier. A float literal still panics there (#226).
2122
- A timestamp literal finer than the column's unit (a `pandas.Timestamp` with nanoseconds against microseconds, a `datetime` with microseconds against seconds) was truncated; it compares exactly in filters, windows, group filters and `is_in` (#200).
2223
- A `date` compared with a timezone-aware column is that zone's local midnight, and a naive `datetime` its wall-clock time; a naive time in a DST gap or fold raises `ValueError` naming the zone instead of failing inside the optimizer.
2324
- A `datetime` before 1677 or after 2262 can be compared with date and timestamp columns.

‎py-ltseq/tests/test_literal_column_rules.py‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,41 @@ def test_high_scale_decimal_against_integer_column(t):
156156
assert _v(t, lambda r: r.i == Decimal("2.000000000000000000000000000000")) == [F, T, F, F, F]
157157

158158

159+
NEG_SCALE = [Decimal("10"), Decimal("-10"), None, Decimal("0")]
160+
161+
162+
@pytest.fixture(scope="module")
163+
def neg_scale():
164+
"""A decimal128(38, -1) column: Arrow allows a negative column scale, the literal wire does not."""
165+
return LTSeq.from_arrow(
166+
pa.table({"k": range(4), "x": pa.array(NEG_SCALE, pa.decimal128(38, -1))})
167+
).sort("k")
168+
169+
170+
@pytest.mark.parametrize(
171+
"lit", [Decimal("1E-38"), Decimal("-1E-38"), Decimal("0E-38"), Decimal("15"), Decimal("-15")]
172+
)
173+
def test_negative_scale_column_against_a_finer_literal(neg_scale, lit):
174+
"""A scale-38 literal is 39 digits finer than the column, past any power of
175+
ten an i128 holds: it used to panic while planning (15 and -15 are within
176+
reach). Compared exactly, in either operand order."""
177+
for op in (operator.gt, operator.ge, operator.lt, operator.le, operator.eq, operator.ne):
178+
assert _v(neg_scale, lambda r: op(r.x, lit)) == [None if v is None else op(v, lit) for v in NEG_SCALE]
179+
assert _v(neg_scale, lambda r: op(lit, r.x)) == [None if v is None else op(lit, v) for v in NEG_SCALE]
180+
assert _v(neg_scale, lambda r: r.x.is_in([lit, Decimal("-10")])) == [lit == 10, T, N, lit == 0]
181+
182+
183+
def test_negative_scale_column_value_positions(neg_scale):
184+
with pytest.raises(ValueError, match="does not fit column 'x'"):
185+
neg_scale.derive(v=lambda r: r.x.fill_null(Decimal("1E-38")))
186+
with pytest.raises(ValueError, match="does not fit column 'x'"):
187+
neg_scale.derive(v=lambda r: r.x.shift(1, default=Decimal("1E-38")))
188+
kind, values = _typed(neg_scale, lambda r: r.x.fill_null(Decimal("0E-38")))
189+
assert (kind, values) == ("decimal128(38, -1)", [10, -10, 0, 0])
190+
kind, values = _typed(neg_scale, lambda r: r.x.shift(1, default=Decimal("0E-38")))
191+
assert (kind, values) == ("decimal128(38, -1)", [0, 10, -10, None])
192+
193+
159194
# ---- Decimal / date / datetime literal against a string operand: an error ----
160195

161196

‎py-ltseq/tests/test_literal_crosstype.py‎

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,14 @@ def _ts_ticks(unit):
6868
"decw": pa.array(
6969
[Decimal(1), Decimal(10) ** 27, None, Decimal("0.1234567891"), Decimal(2)], pa.decimal128(38, 10)
7070
),
71+
# Negative scales: 39+ digits coarser than a scale-38 literal, and one
72+
# narrow enough that DataFusion's own coercion stays within 38 digits.
73+
"decneg": pa.array(
74+
[Decimal(10), Decimal(-10), None, Decimal(0), Decimal("9E+37")], pa.decimal128(38, -1)
75+
),
76+
"decneg_narrow": pa.array(
77+
[Decimal(100), Decimal(-200), None, Decimal(0), Decimal(9_999_900)], pa.decimal128(5, -2)
78+
),
7179
"utf8": pa.array(["a", "b", None, "c", "B"]),
7280
"large_utf8": pa.array(["a", "b", None, "c", "B"], pa.large_string()),
7381
"bool": pa.array([True, False, None, True, False]),
@@ -87,6 +95,9 @@ def _ts_ticks(unit):
8795
"decimal": Decimal("1.5"),
8896
"decimal-0.1": Decimal("0.1"),
8997
"decimal-hiscale": Decimal("0.1234567890123456789"),
98+
"decimal-tiny": Decimal("1E-38"),
99+
"decimal-tiny-neg": Decimal("-1E-38"),
100+
"decimal-zero-scale38": Decimal("0E-38"),
90101
"np-int": np.int64(2),
91102
"bool": True,
92103
"np-bool": np.bool_(True),
@@ -214,9 +225,21 @@ def table():
214225
return LTSeq.from_arrow(pa.table({"k": pa.array(range(5), pa.int64()), **COLUMNS}))
215226

216227

217-
@pytest.mark.parametrize(
218-
"column, lname, op, expected", CELLS, ids=[f"{c}-{op}-{lname}" for c, lname, op, _ in CELLS]
219-
)
228+
# DataFusion's unwrap_cast simplifier panics planning a float literal against a
229+
# negative-scale decimal column (#226, apache/datafusion#24896).
230+
KNOWN_PANICS = {("decneg", "float"), ("decneg_narrow", "float")}
231+
232+
233+
def _param(column, lname, op, expected):
234+
marks = (
235+
[pytest.mark.xfail(strict=True, reason="#226: DataFusion's unwrap_cast panics")]
236+
if (column, lname) in KNOWN_PANICS
237+
else []
238+
)
239+
return pytest.param(column, lname, op, expected, marks=marks, id=f"{column}-{op}-{lname}")
240+
241+
242+
@pytest.mark.parametrize("column, lname, op, expected", [_param(*cell) for cell in CELLS])
220243
def test_comparison_matches_oracle(table, column, lname, op, expected):
221244
lit = LITERALS[lname]
222245
swapped = {"gt": operator.lt, "eq": operator.eq}[op]

‎src/transpiler/literal_rules.rs‎

Lines changed: 57 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@
44
//! DataFusion's coercion rules are written for two columns, and against a
55
//! literal they lose information (#145): a Decimal against a float column
66
//! casts the *column* to decimal, which fails on NaN and infinity; a decimal
7-
//! whose widened type needs more than 38 digits overflows; a timestamp
7+
//! whose widened type needs more than 38 digits overflows, and the simplifier
8+
//! panics on a cast of a negative-scale decimal column; a timestamp
89
//! literal finer than the column's unit is truncated; a date against a zoned
910
//! column means UTC midnight; a timestamp against a date column is compared
1011
//! in nanoseconds and overflows outside 1677–2262. The planner has the
@@ -475,14 +476,23 @@ fn fit_literal(
475476
"{name} is a string; compare it with a string, or cast it to the literal's type first"
476477
))
477478
}
478-
T::Decimal128(precision, scale) => match literal {
479-
S::Decimal128(Some(value), lit_precision, lit_scale) => Ok(fit_decimal(
480-
(*precision, *scale),
481-
*value,
482-
(*lit_precision, *lit_scale),
483-
)),
484-
_ => Ok(Fit::Unchanged),
485-
},
479+
T::Decimal128(precision, scale) => {
480+
let operand = (*precision, *scale);
481+
Ok(match literal {
482+
// DataFusion's simplifier panics unwrapping a cast of a
483+
// negative-scale operand next to a literal (it raises 10 to
484+
// the scale as an unsigned power), so such an operand gets a
485+
// literal of its own type.
486+
S::Decimal128(Some(value), _, lit_scale) if *scale < 0 => {
487+
place_decimal(operand, *value, *lit_scale)
488+
}
489+
S::Int64(Some(value)) if *scale < 0 => place_decimal(operand, i128::from(*value), 0),
490+
S::Decimal128(Some(value), lit_precision, lit_scale) => {
491+
fit_decimal(operand, *value, (*lit_precision, *lit_scale))
492+
}
493+
_ => Fit::Unchanged,
494+
})
495+
}
486496
T::Int8 | T::Int16 | T::Int32 | T::Int64 | T::UInt8 | T::UInt16 | T::UInt32 | T::UInt64 => {
487497
match literal {
488498
// DataFusion compares an integer with a decimal as the decimal
@@ -546,16 +556,23 @@ fn integer_decimal_precision(integer: &DataType) -> u8 {
546556
///
547557
/// DataFusion compares two decimals at the widened type that holds both,
548558
/// which is exact while it fits 38 digits; past that, casting one side to it
549-
/// overflows. Then the literal is moved to the operand's own scale: exactly
550-
/// when it has no more fractional digits than the operand, as a floor
551-
/// otherwise, and as beyond every value when it does not fit the operand.
559+
/// overflows, and the literal is placed by [`place_decimal`].
552560
fn fit_decimal(operand: (u8, i8), value: i128, literal: (u8, i8)) -> Fit {
553561
let (precision, scale) = (i32::from(operand.0), i32::from(operand.1));
554562
let (lit_precision, lit_scale) = (i32::from(literal.0), i32::from(literal.1));
555563
let widened = scale.max(lit_scale) + (precision - scale).max(lit_precision - lit_scale);
556564
if widened <= 38 {
557565
return Fit::Unchanged;
558566
}
567+
place_decimal(operand, value, literal.1)
568+
}
569+
570+
/// The decimal `value / 10^lit_scale` moved to a decimal operand's own
571+
/// `(precision, scale)`: exactly when it has no more fractional digits than
572+
/// the operand, as a floor otherwise, and as beyond every value when it does
573+
/// not fit the operand.
574+
fn place_decimal(operand: (u8, i8), value: i128, lit_scale: i8) -> Fit {
575+
let (scale, lit_scale) = (i32::from(operand.1), i32::from(lit_scale));
559576
let bound = |v: i128| ScalarValue::Decimal128(Some(v), operand.0, operand.1);
560577
// An operand value is below 10^precision in unscaled units.
561578
let limit = 10i128.pow(u32::from(operand.0));
@@ -568,13 +585,17 @@ fn fit_decimal(operand: (u8, i8), value: i128, literal: (u8, i8)) -> Fit {
568585
_ => Fit::Beyond(value.cmp(&0)),
569586
};
570587
}
571-
let divisor = 10i128.pow((lit_scale - scale).unsigned_abs());
572-
let floor = value.div_euclid(divisor);
588+
// A negative-scale operand can be more than 38 digits coarser than the
589+
// literal. 10^gap then exceeds every i128, so the floor is 0 or -1.
590+
let (floor, exact) = match 10i128.checked_pow((lit_scale - scale).unsigned_abs()) {
591+
Some(divisor) => (value.div_euclid(divisor), value.rem_euclid(divisor) == 0),
592+
None => (if value < 0 { -1 } else { 0 }, value == 0),
593+
};
573594
if floor >= limit {
574595
Fit::Beyond(Ordering::Greater)
575596
} else if floor <= -limit {
576597
Fit::Beyond(Ordering::Less)
577-
} else if value.rem_euclid(divisor) == 0 {
598+
} else if exact {
578599
Fit::Replace(bound(floor))
579600
} else {
580601
Fit::Between(bound(floor))
@@ -738,6 +759,14 @@ mod tests {
738759
((20, 0), 15 * 10i128.pow(30), (32, 31), Fit::Between(dec(1, 20, 0))),
739760
((20, 0), -15 * 10i128.pow(30), (32, 31), Fit::Between(dec(-2, 20, 0))),
740761
((20, 0), 2 * 10i128.pow(30), (31, 30), Fit::Replace(dec(2, 20, 0))),
762+
// A negative-scale operand 39 or more digits coarser than the
763+
// literal: 10^gap exceeds i128, and the literal is within one step of zero.
764+
((38, -1), 1, (38, 38), Fit::Between(dec(0, 38, -1))),
765+
((38, -1), -1, (38, 38), Fit::Between(dec(-1, 38, -1))),
766+
((38, -1), 0, (38, 38), Fit::Replace(dec(0, 38, -1))),
767+
((38, -38), 10i128.pow(38) - 1, (38, 38), Fit::Between(dec(0, 38, -38))),
768+
// A gap that fits: 15 at scale -1 is between 10 and 20.
769+
((38, -1), 15, (2, 0), Fit::Between(dec(1, 38, -1))),
741770
];
742771
for (operand, value, literal, expected) in table {
743772
assert_eq!(fit_decimal(operand, value, literal), expected, "{operand:?} {value} {literal:?}");
@@ -845,6 +874,19 @@ mod tests {
845874
assert_eq!(fit_literal(&DataType::Decimal128(38, 10), &wide, false, "d"), Ok(Fit::Unchanged));
846875
}
847876

877+
#[test]
878+
fn negative_scale_operands_get_a_literal_of_their_own_type() {
879+
let operand = DataType::Decimal128(5, -2);
880+
let int = |v| ScalarValue::Int64(Some(v));
881+
// Within 38 digits, but DataFusion cannot plan the cast: placed here.
882+
assert_eq!(fit_literal(&operand, &dec(15, 2, 1), true, "x"), Ok(Fit::Between(dec(0, 5, -2))));
883+
assert_eq!(fit_literal(&operand, &int(300), true, "x"), Ok(Fit::Replace(dec(3, 5, -2))));
884+
assert_eq!(fit_literal(&operand, &int(-150), true, "x"), Ok(Fit::Between(dec(-2, 5, -2))));
885+
// A non-negative scale is left to DataFusion.
886+
assert_eq!(fit_literal(&DataType::Decimal128(5, 0), &int(3), true, "x"), Ok(Fit::Unchanged));
887+
assert_eq!(fit_literal(&DataType::Decimal128(5, 0), &dec(15, 2, 1), true, "x"), Ok(Fit::Unchanged));
888+
}
889+
848890
fn schema() -> ArrowSchema {
849891
ArrowSchema::new(vec![
850892
Field::new("ts", DataType::Timestamp(TimeUnit::Second, None), true),

0 commit comments

Comments
 (0)