Skip to content

Commit 1b126e0

Browse files
kz930claude
andcommitted
fix(workflow-operator): read an overflowing hex double as an infinity
Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 5acfad6 commit 1b126e0

3 files changed

Lines changed: 13 additions & 6 deletions

File tree

‎common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneHelpers.scala‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -147,12 +147,17 @@ object StandaloneHelpers {
147147
|def _texera_java_double(text):
148148
| # Double.parseDouble on the trimmed text, which a read field reaches
149149
| # through `toDouble`. It takes a trailing d or f as a type suffix and
150-
| # 0x...p... as a hex literal, both of which float() refuses.
150+
| # 0x...p... as a hex literal, both of which float() refuses. A hex
151+
| # literal too large for a double is an infinity in Java, where
152+
| # float.fromhex raises.
151153
| text = text.strip()
152154
| if text[-1:] in ("d", "D", "f", "F"):
153155
| text = text[:-1]
154156
| if text.lstrip("+-")[:2].lower() == "0x":
155-
| return float.fromhex(text)
157+
| try:
158+
| return float.fromhex(text)
159+
| except OverflowError:
160+
| return float("-inf") if text.startswith("-") else float("inf")
156161
| return float(text)
157162
|
158163
|

‎common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -681,7 +681,8 @@ class CSVScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter {
681681
// numbers and the column is DOUBLE. pandas' float parser takes none of them
682682
// and kept the column as strings, and a blank beside a "NaN" has to stay null.
683683
// `u` holds a "NaN" and no blank, which a float64 column would still report
684-
// as missing to every operator that reads its mask.
684+
// as missing to every operator that reads its mask. "0x1p1024" is past the
685+
// largest double, which Java reads as an infinity.
685686
it should "read a DOUBLE column the way the engine's toDouble does" taggedAs NeedsPythonPackages in {
686687
val python = resolvePython().getOrElse(cancel("No runnable python executable"))
687688
if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas")
@@ -690,7 +691,7 @@ class CSVScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter {
690691
tmpFile.toFile.deleteOnExit()
691692
Files.write(
692693
tmpFile,
693-
"v,w,u\n1.5,1.5,1.5\nNaN,2d,NaN\n,0x1p3,2.5\n-Infinity,3f,3.5\n"
694+
"v,w,u\n1.5,1.5,1.5\nNaN,2d,NaN\n,0x1p3,2.5\n-Infinity,3f,3.5\n4.5,0x1p1024,-0x1p1024\n"
694695
.getBytes(StandardCharsets.UTF_8)
695696
)
696697
val path = tmpFile.toString

‎common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDescSpec.scala‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -168,12 +168,13 @@ class CSVOldScanSourceOpDescSpec extends AnyFlatSpec with Matchers {
168168
// The engine reads a field with `toDouble`, so "NaN", "2d" and "0x1p3" are
169169
// numbers and the column is DOUBLE, where pandas kept it as strings. A cell
170170
// is read as null only where the frame's mask says so, as every operator
171-
// downstream reads it, so the "NaN" has to stay a value.
171+
// downstream reads it, so the "NaN" has to stay a value. "0x1p1024" is past
172+
// the largest double, which Java reads as an infinity.
172173
it should "read a DOUBLE column the way the engine's toDouble does" taggedAs NeedsPythonPackages in {
173174
val python = resolvePython().getOrElse(cancel("No runnable python executable"))
174175
if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas")
175176

176-
val d = describing(writeCsv("w\n1.5\nNaN\n2d\n0x1p3\n-Infinity\n"))
177+
val d = describing(writeCsv("w\n1.5\nNaN\n2d\n0x1p3\n-Infinity\n0x1p1024\n-0x1p1024\n"))
177178
d.sourceSchema().getAttribute("w").getType shouldBe AttributeType.DOUBLE
178179

179180
val exec = new CSVOldScanSourceOpExec(objectMapper.writeValueAsString(d))

0 commit comments

Comments
 (0)