Skip to content

Commit ca8c6ee

Browse files
kz930claude
andcommitted
fix(operator): read Parquet INT96 timestamps in microseconds
pyarrow reads an INT96 in nanoseconds by default, which only reach 1677 to 2262, so a valid date outside that range wrapped silently: 2500 read back as 1915 and 1500 as 2084. The footer and the rows are both read in microseconds now. The digits under the microsecond are dropped, where the executor keeps them. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent d2836ed commit ca8c6ee

2 files changed

Lines changed: 42 additions & 9 deletions

File tree

‎common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/parquet/ParquetScanSourceOpDesc.scala‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,11 +73,19 @@ class ParquetScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenera
7373
// the footer states and knows nothing of an index, so it hands `customer_id`
7474
// on like any other. Stripped, pandas reads every column the file holds,
7575
// under the name the footer gives it and in the order it states them.
76+
//
77+
// An INT96 timestamp is read in microseconds, in the footer and in the rows
78+
// alike. pyarrow's default is nanoseconds, which only reach 1677 to 2262, and
79+
// a date outside them wraps silently: 2500 read back as 1915. The executor
80+
// keeps the digits under the microsecond, which this drops.
7681
val read =
7782
s"""|out1df = pd.read_parquet(
7883
| $SourceFilePlaceholder,
7984
| dtype_backend="numpy_nullable",
80-
| schema=pq.read_schema($SourceFilePlaceholder).remove_metadata(),
85+
| schema=pq.ParquetFile(
86+
| $SourceFilePlaceholder, coerce_int96_timestamp_unit="us"
87+
| ).schema_arrow.remove_metadata(),
88+
| coerce_int96_timestamp_unit="us",
8189
|)""".stripMargin
8290
// The columns pandas does not land on the same value as the executor. A
8391
// DECIMAL arrives as decimal.Decimal objects and a FLOAT keeps the single

‎common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/parquet/ParquetScanSourceOpDescSpec.scala‎

Lines changed: 33 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,7 @@ import java.io.File
4545
import java.nio.charset.StandardCharsets
4646
import java.nio.file.Files
4747
import java.sql.Timestamp
48-
import java.time.temporal.ChronoUnit
49-
import java.time.{Instant, LocalDateTime, ZoneOffset}
48+
import java.time.{LocalDateTime, ZoneOffset}
5049
import java.util.concurrent.TimeUnit
5150
import scala.jdk.CollectionConverters._
5251
import scala.util.{Try, Using}
@@ -496,9 +495,19 @@ class ParquetScanSourceOpDescSpec extends AnyFlatSpec with Matchers {
496495
val d = new ParquetScanSourceOpDesc
497496
d.fileName = Some("file:///tmp/data.parquet")
498497
d.standaloneImports() should contain("import pyarrow.parquet as pq")
499-
d.generateStandaloneCode() should include(
500-
"schema=pq.read_schema(sourceFile).remove_metadata()"
501-
)
498+
d.generateStandaloneCode() should include(".schema_arrow.remove_metadata()")
499+
}
500+
501+
// pyarrow reads an INT96 in nanoseconds by default, which only reach 1677 to
502+
// 2262, and a date outside them wraps silently. The footer is asked for
503+
// microseconds as well as the rows, since a schema stating nanoseconds wins.
504+
it should "read a twelve-byte timestamp in microseconds" in {
505+
val d = new ParquetScanSourceOpDesc
506+
d.fileName = Some("file:///tmp/data.parquet")
507+
val code = d.generateStandaloneCode()
508+
code should include("""sourceFile, coerce_int96_timestamp_unit="us"""")
509+
code should include(""" coerce_int96_timestamp_unit="us",
510+
|)""".stripMargin)
502511
}
503512

504513
// The rest of what pandas reads differently from the executor: a FLOAT keeps
@@ -626,7 +635,8 @@ class ParquetScanSourceOpDescSpec extends AnyFlatSpec with Matchers {
626635
row.append("size", 16777216.0f)
627636
row.append("big", 9007199254740993L) // past what a double counts exactly
628637
row.append("seen_at", 1709388309123456L)
629-
row.append("legacy_at", new NanoTime(2460312, 11045123456789L).toBinary)
638+
// In whole microseconds, which is what the script reads an INT96 in.
639+
row.append("legacy_at", new NanoTime(2460312, 11045123456000L).toBinary)
630640
row.append("born_on", 19723) // 2024-01-01
631641
row.append("tiny", 120)
632642
row.append("active", true)
@@ -651,6 +661,16 @@ class ParquetScanSourceOpDescSpec extends AnyFlatSpec with Matchers {
651661
// leaves it null.
652662
row => {
653663
row.append("label", "carol")
664+
},
665+
// Dates nanoseconds cannot reach, which read back as 1915 and 2084.
666+
// 2500-01-02 03:04:05.123456 and 1500-01-02.
667+
row => {
668+
row.append("label", "dave")
669+
row.append("legacy_at", new NanoTime(2634168, 11045123456000L).toBinary)
670+
},
671+
row => {
672+
row.append("label", "erin")
673+
row.append("legacy_at", new NanoTime(2268925, 0L).toBinary)
654674
}
655675
)
656676

@@ -740,8 +760,10 @@ class ParquetScanSourceOpDescSpec extends AnyFlatSpec with Matchers {
740760
/** A cell as the two sides can be held to the same answer for it. */
741761
private def comparable(cell: Any): Any =
742762
cell match {
763+
// Counted past a Long, which holds nanoseconds only from 1677 to 2262.
743764
case moment: Timestamp =>
744-
ChronoUnit.NANOS.between(Instant.EPOCH, moment.toLocalDateTime.toInstant(ZoneOffset.UTC))
765+
val instant = moment.toLocalDateTime.toInstant(ZoneOffset.UTC)
766+
(BigInt(instant.getEpochSecond) * 1000000000L + instant.getNano).bigInteger
745767
case bytes: Array[Byte] => bytes.map(_ & 0xff).toList
746768
case other => other
747769
}
@@ -761,8 +783,11 @@ class ParquetScanSourceOpDescSpec extends AnyFlatSpec with Matchers {
761783
| # executor hands a null over, so that is what it is compared as.
762784
| elif _value is None or pd.isna(_value):
763785
| _values.append(None)
786+
| # Nanoseconds counted from the microseconds, since `.value` raises
787+
| # on a date nanoseconds cannot reach.
764788
| elif hasattr(_value, "isoformat"):
765-
| _values.append(int(_value.value))
789+
| _micros = int(_value.asm8.astype("datetime64[us]").astype("int64"))
790+
| _values.append(_micros * 1000 + _value.nanosecond)
766791
| elif hasattr(_value, "item"):
767792
| _values.append(_value.item())
768793
| else:

0 commit comments

Comments
 (0)