Skip to content

Commit d9e6173

Browse files
kz930claude
andcommitted
test(verify): keep a null in a widened column as the dtype's own hole
float64 cannot hold pd.NA, so two identical outputs with a widened LONG or BOOLEAN column and a null failed with a TypeError instead of comparing equal. Pass None through and let each dtype pick its marker. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent bdfef7e commit d9e6173

2 files changed

Lines changed: 27 additions & 10 deletions

File tree

‎workflow-compiling-service/src/test/resources/python/compare.py‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -337,12 +337,11 @@ def arrays(values):
337337
for column, cells in values.items():
338338
# A column either side widened is compared as the float it now is,
339339
# exactly rather than by tolerance; one both sides kept integral is
340-
# carried in the nullable integer, where a long stays exact.
340+
# carried in the nullable integer, where a long stays exact. A null
341+
# stays None so each dtype takes its own hole: float64 cannot hold
342+
# pd.NA.
341343
widened = any(isinstance(c, float) for c in cells)
342-
dtype = "float64" if widened else "Int64"
343-
out[column] = pd.array(
344-
[pd.NA if c is None else c for c in cells], dtype=dtype
345-
)
344+
out[column] = pd.array(cells, dtype="float64" if widened else "Int64")
346345
return out
347346

348347
return arrays(actual), arrays(expected)
@@ -371,7 +370,6 @@ def arrays(values):
371370
out = {}
372371
for column, cells in values.items():
373372
kept = all(c is None or isinstance(c, bool) for c in cells)
374-
cells = [pd.NA if c is None else c for c in cells]
375373
out[column] = pd.array(cells, dtype="boolean" if kept else "float64")
376374
return out
377375

‎workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/ComparatorSpec.scala‎

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -174,10 +174,11 @@ class ComparatorSpec extends AnyFlatSpec with Matchers {
174174
// kept the type.
175175
val dir = Files.createTempDirectory("comparator-spec-int-both-widened-")
176176
// writeLongs for the schema sidecar it leaves; the rows are then rewritten
177-
// as the widened form both paths would have produced.
178-
val a = writeLongs(dir, "a.jsonl", Seq(6L))
179-
Files.writeString(a, "{\"n\":6.0}\n")
180-
val b = writeLines(dir, "b.jsonl", Seq("""{"n":6.0}"""))
177+
// as the widened form both paths would have produced. The null is the hole
178+
// that widened it, and float64 has to hold it too.
179+
val a = writeLongs(dir, "a.jsonl", Seq(6L, null))
180+
Files.writeString(a, "{\"n\":6.0}\n{\"n\":null}\n")
181+
val b = writeLines(dir, "b.jsonl", Seq("""{"n":6.0}""", """{"n":null}"""))
181182
noException should be thrownBy Comparator.assertEqual(a, b)
182183
}
183184

@@ -250,6 +251,24 @@ class ComparatorSpec extends AnyFlatSpec with Matchers {
250251
}.getMessage should include("the two sides wrote it differently")
251252
}
252253

254+
it should "accept a boolean column both sides widened alike" in {
255+
// A hole costs a pandas boolean column its type on both paths alike, so
256+
// both write the true as 1.0 next to the null.
257+
val dir = Files.createTempDirectory("comparator-spec-bool-both-widened-")
258+
val boolSchema = Schema().add(new Attribute("flag", AttributeType.BOOLEAN))
259+
val a = dir.resolve("a.jsonl")
260+
TupleIO.writeTuples(
261+
a,
262+
Iterator(
263+
Tuple.builder(boolSchema).add(boolSchema.getAttribute("flag"), Boolean.box(true)).build()
264+
),
265+
boolSchema
266+
)
267+
Files.writeString(a, "{\"flag\":1.0}\n{\"flag\":null}\n")
268+
val b = writeLines(dir, "b.jsonl", Seq("""{"flag":1.0}""", """{"flag":null}"""))
269+
noException should be thrownBy Comparator.assertEqual(a, b)
270+
}
271+
253272
it should "report a model column only one side produced" in {
254273
// Model columns are compared by behavior and then dropped from both frames,
255274
// so a side that never wrote one has to fail here: once the column is gone,

0 commit comments

Comments
 (0)