Skip to content

Commit ed55981

Browse files
kz930claude
andcommitted
fix(workflow-operator): keep a stored NaN in an Aggregate CONCAT
Read nulls from the column's mask instead of pd.isna on each value, which took a stored NaN for a null too. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 79be3d4 commit ed55981

2 files changed

Lines changed: 11 additions & 5 deletions

File tree

‎common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDesc.scala‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -174,10 +174,12 @@ class AggregateOpDesc extends LogicalOp with StandaloneCodeGenerator {
174174
| # comma: "", "a", "" concatenates to "a," and not ",a,". A null is
175175
| # read as the empty string, which is what makes the two the same
176176
| # here. This is concatAgg's fold, written out.
177-
| # Each value is written as Java's toString writes it.
177+
| # Each value is written as Java's toString writes it. The column's
178+
| # own mask says which are null: pd.isna on the value would take a
179+
| # stored NaN for one too.
178180
| partial = ""
179-
| for v in series:
180-
| text = "" if pd.isna(v) else _texera_java_text(v)
181+
| for v, missing in zip(series, series.isna()):
182+
| text = "" if missing else _texera_java_text(v)
181183
| partial = text if partial == "" else partial + "," + text
182184
| return partial
183185
|

‎common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregateOpDescSpec.scala‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -272,6 +272,7 @@ class AggregateOpDescSpec extends AnyFlatSpec with Matchers {
272272
// A DOUBLE column read from Arrow keeps a stored NaN apart from null. The
273273
// engine skips the null and folds the NaN in, so SUM, AVERAGE and MAX answer
274274
// NaN while MIN, which Java's Double.compare sends NaN past, answers 1.0.
275+
// CONCAT writes the NaN out and only a null as the empty string.
275276
it should "fold a stored NaN in a DOUBLE column as the engine does" taggedAs NeedsPythonPackages in {
276277
val python = resolvePython().getOrElse(cancel("No runnable python executable"))
277278
if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas")
@@ -285,7 +286,8 @@ class AggregateOpDescSpec extends AnyFlatSpec with Matchers {
285286
AggregationFunction.SUM,
286287
AggregationFunction.AVERAGE,
287288
AggregationFunction.MIN,
288-
AggregationFunction.MAX
289+
AggregationFunction.MAX,
290+
AggregationFunction.CONCAT
289291
)
290292

291293
def render(value: Any): String =
@@ -318,6 +320,7 @@ class AggregateOpDescSpec extends AnyFlatSpec with Matchers {
318320
| np.array([False, False, False, True, True]),
319321
| ),
320322
|})
323+
|${desc().standaloneHelpers().mkString("\n")}
321324
|${desc().generateStandaloneCode(Map(PortIdentity() -> input))}
322325
|def render(v):
323326
| if v is pd.NA or v is None:
@@ -327,7 +330,8 @@ class AggregateOpDescSpec extends AnyFlatSpec with Matchers {
327330
| print(row)
328331
|""".stripMargin
329332
val out = runPython(python, driver, "aggregate-double-nan-", "UTC")
330-
withClue(s"script:\n$driver")(out.trim.linesIterator.toSeq shouldBe engine)
333+
// No trim: an all-null group's CONCAT is "", which ends its line in a space.
334+
withClue(s"script:\n$driver")(out.linesIterator.toSeq shouldBe engine)
331335
}
332336

333337
check(List("k"))

0 commit comments

Comments
 (0)