Skip to content

Commit 7d0ebe5

Browse files
kz930claude
andcommitted
fix(operator): leave a binning column of stored NaN uncut
A nullable Float64 column read from Arrow can hold NaN as a value, which notna() reports present, so an all-NaN column reached pd.cut and raised on duplicate edges while the engine path left it uncut. The guard now tests a float64 view. Adds a stored-NaN case to the runtime test. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent b040c1b commit 7d0ebe5

2 files changed

Lines changed: 15 additions & 4 deletions

File tree

‎common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/binning/BinningOpDesc.scala‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,9 @@ class BinningOpDesc extends PythonOperatorDescriptor with StandaloneCodeGenerato
118118
// The guard is there because edges cannot be found in a column that holds
119119
// nothing: an equal-width cut raises both when every cell is empty and when
120120
// no row arrived at all. Leaving those values uncut keeps the holes, and the
121-
// suffix turns them into the empty bins they already are.
121+
// suffix turns them into the empty bins they already are. The test runs on a
122+
// float64 view because a nullable Float64 column can store NaN, which
123+
// `notna()` counts as a value there but the cut still cannot place.
122124
pyb"""from pytexera import *
123125
|import pandas as pd
124126
|
@@ -129,7 +131,7 @@ class BinningOpDesc extends PythonOperatorDescriptor with StandaloneCodeGenerato
129131
| out = table.copy()
130132
| _column = $attribute
131133
| _binned = out[_column]
132-
| if _binned.notna().any():
134+
| if _binned.astype("float64").notna().any():
133135
| _binned = ${cut}(_binned, ${args})
134136
| out[_column + "_bin"] = _binned${suffix}
135137
| yield out""".encode
@@ -140,7 +142,7 @@ class BinningOpDesc extends PythonOperatorDescriptor with StandaloneCodeGenerato
140142
val result = pyStringLiteral(resultColumn)
141143
s"""out1df = in1df.copy()
142144
|_binned = out1df[$column]
143-
|if _binned.notna().any():
145+
|if _binned.astype("float64").notna().any():
144146
| _binned = $cutName(_binned, $cutArgs)
145147
|out1df[$result] = _binned$labelSuffix""".stripMargin
146148
}

‎common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/binning/BinningOpDescSpec.scala‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ class BinningOpDescSpec extends AnyFlatSpec with Matchers {
106106
// raises rather than handing back empty bins. Both renderings skip the cut, which
107107
// leaves the holes for the suffix to turn into the empty labels they already are.
108108
it should "leave a column with nothing in it uncut" in {
109-
val guard = "if _binned.notna().any():"
109+
val guard = """if _binned.astype("float64").notna().any():"""
110110
desc().generateStandaloneCode() should include(guard)
111111
desc().generatePythonCode() should include(guard)
112112
}
@@ -168,10 +168,18 @@ class BinningOpDescSpec extends AnyFlatSpec with Matchers {
168168
private val runtimeDriverScript: String =
169169
"""import sys
170170
|import pandas as pd
171+
|import pyarrow as pa
172+
|
173+
|# NaN stored as a value in a nullable Float64 column, as the Arrow reader
174+
|# leaves it, where `notna()` reports it present.
175+
|storednan = pa.table(
176+
| {"age": pa.array([float("nan"), float("nan")], type=pa.float64(), from_pandas=False)}
177+
|).to_pandas(types_mapper={pa.float64(): pd.Float64Dtype()}.get)
171178
|
172179
|code = open(sys.argv[1]).read()
173180
|cases = {
174181
| "allnull": pd.DataFrame({"age": pd.Series([None, None], dtype="float64")}),
182+
| "storednan": storednan,
175183
| "empty": pd.DataFrame({"age": pd.Series([], dtype="float64")}),
176184
| "mixed": pd.DataFrame({"age": pd.Series([1.0, 2.0, None])}),
177185
|}
@@ -220,6 +228,7 @@ class BinningOpDescSpec extends AnyFlatSpec with Matchers {
220228
.toMap
221229
verdicts shouldBe Map(
222230
"allnull" -> "OK:2:0", // two rows, neither of them in a bin
231+
"storednan" -> "OK:2:0", // NaN is no more placeable than a hole
223232
"empty" -> "OK:0:0", // no rows to put in one
224233
"mixed" -> "OK:3:2" // the hole is the only row left unlabelled
225234
)

0 commit comments

Comments
 (0)