Skip to content

Commit 2a11559

Browse files
Fix DECIMAL bugs (#654)
Decimal('1E+2') now converts to DECIMAL(3,0) 100 instead of 1.00. Inferred type and value always agree. Too-wide values fall back to DOUBLE, including exponents of 256 and up and exponents beyond 32 bits. Fixes #566
2 parents b2d23c0 + 2c4cb43 commit 2a11559

5 files changed

Lines changed: 418 additions & 54 deletions

File tree

‎src/include/duckdb_python/python_objects.hpp‎

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -73,22 +73,18 @@ struct PyDecimal {
7373

7474
struct PyDecimalPowerConverter {
7575
template <typename T, typename = std::enable_if<std::numeric_limits<T>::is_integer, T>>
76-
static Value Operation(bool signed_value, vector<uint8_t> &digits, uint8_t width, uint8_t scale) {
76+
static Value Operation(bool signed_value, vector<uint8_t> &digits, uint8_t width, uint8_t exponent) {
7777
T value = 0;
7878
for (auto &digit : digits) {
7979
value = value * 10 + digit;
8080
}
81-
D_ASSERT(scale >= 0);
82-
int64_t multiplier =
83-
NumericHelper::POWERS_OF_TEN[MinValue<uint8_t>(scale, NumericHelper::CACHED_POWERS_OF_TEN - 1)];
84-
for (auto power = scale; power > NumericHelper::CACHED_POWERS_OF_TEN; power--) {
85-
multiplier *= 10;
81+
for (idx_t i = 0; i < exponent; i++) {
82+
value *= 10;
8683
}
87-
value *= multiplier;
8884
if (signed_value) {
8985
value = -value;
9086
}
91-
return Value::DECIMAL(value, width, scale);
87+
return Value::DECIMAL(value, width, 0);
9288
}
9389
};
9490

@@ -98,7 +94,7 @@ struct PyDecimal {
9894
bool signed_value = false;
9995

10096
PyDecimalExponentType exponent_type;
101-
int32_t exponent_value;
97+
int64_t exponent_value;
10298

10399
public:
104100
bool TryGetType(LogicalType &type);

‎src/native/python_objects.cpp‎

Lines changed: 53 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -74,26 +74,53 @@ PyDecimal::PyDecimal(nb::handle &obj) : obj(obj) {
7474
}
7575
}
7676

77-
bool PyDecimal::TryGetType(LogicalType &type) {
78-
int32_t width = digits.size();
77+
struct DecimalWidthScale {
78+
int32_t width = 0;
79+
int64_t scale = 0;
80+
bool fits = false;
81+
};
82+
83+
static DecimalWidthScale ComputeDecimalWidthScale(idx_t digit_count, PyDecimalExponentType exponent_type,
84+
int64_t exponent_value) {
85+
DecimalWidthScale result;
86+
87+
if (digit_count > Decimal::MAX_WIDTH_DECIMAL) {
88+
return result;
89+
}
90+
91+
if (exponent_type == PyDecimalExponentType::EXPONENT_POWER) {
92+
if (exponent_value > Decimal::MAX_WIDTH_DECIMAL - digit_count) {
93+
return result;
94+
}
95+
result.width = static_cast<int32_t>(digit_count) + static_cast<int32_t>(exponent_value);
96+
result.scale = 0;
97+
result.fits = true;
98+
return result;
99+
}
100+
101+
result.scale = exponent_value;
102+
if (result.scale > Decimal::MAX_WIDTH_INT128) {
103+
return result;
104+
}
79105

106+
result.width = static_cast<int32_t>(digit_count);
107+
if (result.scale > result.width) {
108+
result.width = static_cast<int32_t>(result.scale) + 1; // add 1 for the non-decimal values
109+
}
110+
result.fits = result.width <= Decimal::MAX_WIDTH_INT128;
111+
return result;
112+
}
113+
114+
bool PyDecimal::TryGetType(LogicalType &type) {
80115
switch (exponent_type) {
81-
case PyDecimalExponentType::EXPONENT_SCALE: {
116+
case PyDecimalExponentType::EXPONENT_SCALE:
82117
case PyDecimalExponentType::EXPONENT_POWER: {
83-
auto scale = exponent_value;
84-
if (exponent_type == PyDecimalExponentType::EXPONENT_POWER) {
85-
width += scale;
86-
}
87-
if (scale > width) {
88-
// The value starts with 1 or more zeros, which are optimized out of the 'digits' array
89-
// 0.001; width=1, exponent=-3
90-
width = scale + 1; // DECIMAL(4,3) - add 1 for the non-decimal values
91-
}
92-
if (width > Decimal::MAX_WIDTH_INT128) {
118+
auto ws = ComputeDecimalWidthScale(digits.size(), exponent_type, exponent_value);
119+
if (!ws.fits) {
93120
type = LogicalType::DOUBLE;
94121
return true;
95122
}
96-
type = LogicalType::DECIMAL(width, scale);
123+
type = LogicalType::DECIMAL(ws.width, static_cast<uint8_t>(ws.scale));
97124
return true;
98125
}
99126
case PyDecimalExponentType::EXPONENT_INFINITY: {
@@ -107,7 +134,6 @@ bool PyDecimal::TryGetType(LogicalType &type) {
107134
default: // LCOV_EXCL_START
108135
throw NotImplementedException("case not implemented for type PyDecimalExponentType");
109136
} // LCOV_EXCL_STOP
110-
}
111137
}
112138
// LCOV_EXCL_START
113139
static void ExponentNotRecognized() {
@@ -117,12 +143,12 @@ static void ExponentNotRecognized() {
117143

118144
void PyDecimal::SetExponent(nb::handle &exponent) {
119145
if (nb::isinstance<nb::int_>(exponent)) {
120-
this->exponent_value = nb::cast<int32_t>(exponent);
146+
this->exponent_value = nb::cast<int64_t>(exponent);
121147
if (this->exponent_value >= 0) {
122148
exponent_type = PyDecimalExponentType::EXPONENT_POWER;
123149
return;
124150
}
125-
exponent_value *= -1;
151+
exponent_value = exponent_value <= -INT64_MAX ? INT64_MAX : -exponent_value;
126152
exponent_type = PyDecimalExponentType::EXPONENT_SCALE;
127153
return;
128154
}
@@ -142,10 +168,6 @@ void PyDecimal::SetExponent(nb::handle &exponent) {
142168
// LCOV_EXCL_STOP
143169
}
144170

145-
static bool WidthFitsInDecimal(int32_t width) {
146-
return width >= 0 && width <= Decimal::MAX_WIDTH_DECIMAL;
147-
}
148-
149171
template <class OP>
150172
Value PyDecimalCastSwitch(PyDecimal &decimal, uint8_t width, uint8_t scale) {
151173
if (width > DecimalWidth<int64_t>::max) {
@@ -165,43 +187,33 @@ static Value CastToDouble(nb::handle &obj) {
165187
string converted = nb::cast<std::string>(nb::str(obj));
166188
string_t decimal_string(converted);
167189
double double_val;
168-
bool try_cast = TryCast::Operation<string_t, double>(decimal_string, double_val, true);
169-
(void)try_cast;
170-
D_ASSERT(try_cast);
190+
if (!TryCast::Operation<string_t, double>(decimal_string, double_val, true)) {
191+
throw InvalidInputException("Failed to convert Decimal value to DOUBLE: %s", converted);
192+
}
171193
return Value::DOUBLE(double_val);
172194
}
173195

174196
Value PyDecimal::ToDuckValue() {
175-
int32_t width = digits.size();
176-
if (!WidthFitsInDecimal(width)) {
177-
return CastToDouble(obj);
178-
}
179197
switch (exponent_type) {
180198
case PyDecimalExponentType::EXPONENT_SCALE: {
181-
uint8_t scale = exponent_value;
182-
D_ASSERT(WidthFitsInDecimal(width));
183-
if (scale > width) {
184-
// Values like '0.001'
185-
width = scale + 1; // leave 1 room for the non-decimal value
186-
}
187-
if (!WidthFitsInDecimal(width)) {
199+
auto ws = ComputeDecimalWidthScale(digits.size(), exponent_type, exponent_value);
200+
if (!ws.fits) {
188201
return CastToDouble(obj);
189202
}
190-
return PyDecimalCastSwitch<PyDecimalScaleConverter>(*this, width, scale);
203+
return PyDecimalCastSwitch<PyDecimalScaleConverter>(*this, ws.width, static_cast<uint8_t>(ws.scale));
191204
}
192205
case PyDecimalExponentType::EXPONENT_POWER: {
193-
uint8_t scale = exponent_value;
194-
width += scale;
195-
if (!WidthFitsInDecimal(width)) {
206+
auto ws = ComputeDecimalWidthScale(digits.size(), exponent_type, exponent_value);
207+
if (!ws.fits) {
196208
return CastToDouble(obj);
197209
}
198-
return PyDecimalCastSwitch<PyDecimalPowerConverter>(*this, width, scale);
210+
return PyDecimalCastSwitch<PyDecimalPowerConverter>(*this, ws.width, static_cast<uint8_t>(exponent_value));
199211
}
200212
case PyDecimalExponentType::EXPONENT_NAN: {
201213
return Value::FLOAT(NAN);
202214
}
203215
case PyDecimalExponentType::EXPONENT_INFINITY: {
204-
return Value::FLOAT(INFINITY);
216+
return Value::FLOAT(signed_value ? -INFINITY : INFINITY);
205217
}
206218
// LCOV_EXCL_START
207219
default: {

‎tests/fast/pandas/test_df_object_resolution.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -577,7 +577,7 @@ def test_numeric_decimal(self, duckdb_cursor):
577577
(12.0, 13, 324234234.00000005),
578578
(-123.0, -12.0000000005, -128),
579579
(-234234.0, 7453324234.0, 345345),
580-
(NULL, NULL, 1.00000),
580+
(NULL, NULL, 100000),
581581
(1.234, -324234234, 1324234359)
582582
) tbl(a, b, c);
583583
"""
@@ -611,8 +611,8 @@ def test_numeric_decimal_coverage(self, duckdb_cursor):
611611
assert math.isnan(conversion[2][0])
612612
assert math.isinf(conversion[3][0])
613613
assert math.isinf(conversion[4][0])
614-
assert math.isinf(conversion[5][0])
615-
assert str(conversion) == "[(nan,), (nan,), (nan,), (inf,), (inf,), (inf,)]"
614+
assert conversion[5][0] == float("-inf")
615+
assert str(conversion) == "[(nan,), (nan,), (nan,), (inf,), (inf,), (-inf,)]"
616616

617617
# Test that the column 'offset' is actually used when converting,
618618
# and that the same 2048 (STANDARD_VECTOR_SIZE) values are not being scanned over and over again

‎tests/fast/sqlite/test_types.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ def test_CheckDecimalWithExponent(self):
8888
self.cur.execute("insert into test(f) values (?)", (val,))
8989
self.cur.execute("select f from test")
9090
row = self.cur.fetchone()
91-
assert row[0] == self.cur.execute("select 1.00000::DOUBLE").fetchone()[0]
91+
assert row[0] == self.cur.execute("select 100000::DOUBLE").fetchone()[0]
9292

9393
def test_CheckNaN(self):
9494
import math

0 commit comments

Comments
 (0)