Skip to content

Commit e44b77f

Browse files
viiryaIsaac
andcommitted
Normalize zero and share the fast-fail between fromString and fromStringANSI
A zero with a large exponent such as '0e39' now reaches Decimal.set, which derives the precision from the negative scale (40 for '0e39', Int.MinValue for '0e2147483647'). Drop the negative scale of a zero first so the result has precision 1 and scale 0. Move the fast-fail into one helper used by both fromString and fromStringANSI, check the precision and scale of zero results in DecimalSuite, move the zero cast checks to CastSuiteBase, and add the JIRA prefix to the new test name. Co-authored-by: Isaac <[email protected]>
1 parent 9f53a39 commit e44b77f

5 files changed

Lines changed: 37 additions & 29 deletions

File tree

‎sql/api/src/main/scala/org/apache/spark/sql/types/Decimal.scala‎

Lines changed: 23 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -645,19 +645,28 @@ object Decimal {
645645
new JavaBigDecimal(str.toString.trim)
646646
}
647647

648+
/**
649+
* Converts a `JavaBigDecimal` parsed from a string to `Decimal`, or returns null if no decimal
650+
* type can hold it. We fast fail because constructing a very large JavaBigDecimal to Decimal is
651+
* very slow. For example: Decimal("6.0790316E+25569151")
652+
*/
653+
private def stringBigDecimalToDecimal(bigDecimal: JavaBigDecimal): Decimal = {
654+
if (SqlApiConf.get.allowNegativeScaleOfDecimalEnabled) {
655+
Decimal(bigDecimal)
656+
} else if (bigDecimal.signum == 0) {
657+
// Zero fits any decimal type whatever its exponent, e.g. "0e39". Drop a negative scale so
658+
// that the precision does not grow with the exponent.
659+
Decimal(bigDecimal.setScale(math.max(bigDecimal.scale, 0)))
660+
} else if (numDigitsInIntegralPart(bigDecimal) > DecimalType.MAX_PRECISION) {
661+
null
662+
} else {
663+
Decimal(bigDecimal)
664+
}
665+
}
666+
648667
def fromString(str: UTF8String): Decimal = {
649668
try {
650-
val bigDecimal = stringToJavaBigDecimal(str)
651-
// We fast fail because constructing a very large JavaBigDecimal to Decimal is very slow.
652-
// For example: Decimal("6.0790316E+25569151"). Zero fits any decimal type whatever its
653-
// exponent, e.g. "0e39", so it is not checked.
654-
if (bigDecimal.signum != 0 &&
655-
numDigitsInIntegralPart(bigDecimal) > DecimalType.MAX_PRECISION &&
656-
!SqlApiConf.get.allowNegativeScaleOfDecimalEnabled) {
657-
null
658-
} else {
659-
Decimal(bigDecimal)
660-
}
669+
stringBigDecimalToDecimal(stringToJavaBigDecimal(str))
661670
} catch {
662671
case _: NumberFormatException =>
663672
null
@@ -669,17 +678,11 @@ object Decimal {
669678
to: DecimalType = DecimalType.USER_DEFAULT,
670679
context: QueryContext = null): Decimal = {
671680
try {
672-
val bigDecimal = stringToJavaBigDecimal(str)
673-
// We fast fail because constructing a very large JavaBigDecimal to Decimal is very slow.
674-
// For example: Decimal("6.0790316E+25569151"). Zero fits any decimal type whatever its
675-
// exponent, e.g. "0e39", so it is not checked.
676-
if (bigDecimal.signum != 0 &&
677-
numDigitsInIntegralPart(bigDecimal) > DecimalType.MAX_PRECISION &&
678-
!SqlApiConf.get.allowNegativeScaleOfDecimalEnabled) {
681+
val decimal = stringBigDecimalToDecimal(stringToJavaBigDecimal(str))
682+
if (decimal == null) {
679683
throw DataTypeErrors.outOfDecimalTypeRangeError(str)
680-
} else {
681-
Decimal(bigDecimal)
682684
}
685+
decimal
683686
} catch {
684687
case _: NumberFormatException =>
685688
throw DataTypeErrors.invalidInputInCastToNumberError(to, str, context)

‎sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuiteBase.scala‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -330,6 +330,12 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
330330
assert(cast("abcdef", FloatType).nullable)
331331
}
332332

333+
test("SPARK-60118: cast string of zero with a large exponent to decimal") {
334+
Seq("0e39", "-0E+100", "0e2147483647").foreach { s =>
335+
checkEvaluation(cast(s, DecimalType(10, 2)), Decimal("0.00"))
336+
}
337+
}
338+
333339
test("cast from timestamp") {
334340
val millis = 15 * 1000 + 3
335341
val seconds = millis * 1000 + 3

‎sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastWithAnsiOffSuite.scala‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -562,9 +562,6 @@ class CastWithAnsiOffSuite extends CastSuiteBase {
562562
Seq("1e2147483647", "1.5e2147483647", "12e2147483647").foreach { s =>
563563
checkEvaluation(cast(s, DecimalType(10, 2)), null)
564564
}
565-
Seq("0e39", "0e2147483647").foreach { s =>
566-
checkEvaluation(cast(s, DecimalType(10, 2)), Decimal("0.00"))
567-
}
568565

569566
checkEvaluation(cast("abcd", DecimalType(38, 1)), null)
570567
}

‎sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastWithAnsiOnSuite.scala‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -480,9 +480,6 @@ class CastWithAnsiOnSuite extends CastSuiteBase with QueryErrorsBase {
480480
cast(s, DecimalType(10, 2)),
481481
"NUMERIC_OUT_OF_SUPPORTED_RANGE")
482482
}
483-
Seq("0e39", "0e2147483647").foreach { s =>
484-
checkEvaluation(cast(s, DecimalType(10, 2)), Decimal("0.00"))
485-
}
486483

487484
checkExceptionInExpression[NumberFormatException](
488485
cast("abcd", DecimalType(38, 1)),

‎sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -446,7 +446,7 @@ class DecimalSuite extends SparkFunSuite with PrivateMethodTester with SQLHelper
446446
}
447447
}
448448

449-
test("UTF8String to Decimal with exponent near Int.MaxValue") {
449+
test("SPARK-60118: UTF8String to Decimal with exponent near Int.MaxValue") {
450450
// The number of integral digits (precision - scale) of these values overflows Int.
451451
Seq("1e2147483647", "-1E+2147483647", "12e2147483647",
452452
"1.5e2147483647").foreach { string =>
@@ -459,9 +459,14 @@ class DecimalSuite extends SparkFunSuite with PrivateMethodTester with SQLHelper
459459
}
460460
// Zero fits any decimal type whatever its exponent.
461461
Seq("0e39", "-0E+100", "0e2147483647").foreach { string =>
462-
assert(Decimal.fromString(UTF8String.fromString(string)) === Decimal(0))
463-
assert(Decimal.fromStringANSI(UTF8String.fromString(string)) === Decimal(0))
462+
Seq(Decimal.fromString(UTF8String.fromString(string)),
463+
Decimal.fromStringANSI(UTF8String.fromString(string))).foreach { d =>
464+
assert(d === Decimal(0))
465+
assert(d.precision === 1)
466+
assert(d.scale === 0)
467+
}
464468
}
469+
assert(Decimal.fromString(UTF8String.fromString("0.00")).scale === 2)
465470
}
466471

467472
// 18 is a max number of digits in Decimal's compact long

0 commit comments

Comments
 (0)