Skip to content

Commit c4bb488

Browse files
kz930claude
andcommitted
fix(operator): parse the timestamp column at microsecond resolution
pandas parses into nanoseconds by default, which reach only 1677 to 2262, so the exported script ended the run on a moment the executor reads like any other: it holds a java.sql.Timestamp, where the year 2500 is ordinary. Microseconds cover the whole range the executor answers for, and the fields are compared against it at and past the nanosecond edge. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
1 parent 95aadac commit c4bb488

2 files changed

Lines changed: 98 additions & 3 deletions

File tree

‎common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/extractdatetime/ExtractDateTimeOpDesc.scala‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -109,9 +109,12 @@ class ExtractDateTimeOpDesc extends MapOpDesc with StandaloneCodeGenerator {
109109
val lines = scala.collection.mutable.ArrayBuffer[String](
110110
"out1df = in1df.copy()",
111111
// A no-op where the source already parsed its input, which is the usual case;
112-
// made anyway for one that handed the column over as text. NOT coerced: the
113-
// engine reads a real moment here, so a cell Python cannot is a disagreement.
114-
s"""_texera_ts = pd.to_datetime(out1df[$source])"""
112+
// made anyway for one that handed the column over as text. Microseconds and
113+
// not the nanoseconds pandas parses into by default: nanoseconds reach 1677
114+
// to 2262, where the engine reads a java.sql.Timestamp and the year 2500 is
115+
// an ordinary moment. NOT coerced: the engine reads a real moment here, so a
116+
// cell Python cannot is a disagreement.
117+
s"""_texera_ts = out1df[$source].astype("datetime64[us]")"""
115118
)
116119
asked.foreach { field =>
117120
val target = pyStringLiteral(columnFor(field))

‎common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/extractdatetime/ExtractDateTimeOpDescSpec.scala‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
package org.apache.texera.amber.operator.extractdatetime
2121

22+
import com.typesafe.config.ConfigFactory
2223
import org.apache.texera.amber.core.executor.OpExecWithClassName
2324
import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema, Tuple}
2425
import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity}
@@ -28,7 +29,12 @@ import org.apache.texera.amber.util.JSONUtils.objectMapper
2829
import org.scalatest.flatspec.AnyFlatSpec
2930
import org.scalatest.matchers.should.Matchers
3031

32+
import java.nio.charset.StandardCharsets
33+
import java.nio.file.Files
3134
import java.sql.Timestamp
35+
import java.util.concurrent.TimeUnit
36+
import scala.io.Source
37+
import scala.util.Try
3238

3339
class ExtractDateTimeOpDescSpec extends AnyFlatSpec with Matchers {
3440

@@ -179,4 +185,90 @@ class ExtractDateTimeOpDescSpec extends AnyFlatSpec with Matchers {
179185
it should "copy the frame through when no field is asked for" in {
180186
desc().generateStandaloneCode() shouldBe "out1df = in1df.copy()"
181187
}
188+
189+
// pandas parses into nanoseconds by default, and nanoseconds reach only 1677 to
190+
// 2262. The engine reads a java.sql.Timestamp, which holds the year 2500 like any
191+
// other, so parsing the column the default way would end the exported run on a
192+
// moment the operator itself has no trouble with.
193+
it should "parse the column at a resolution that reaches past the year 2262" in {
194+
desc(DateTimeField.YEAR).generateStandaloneCode() should
195+
include("""astype("datetime64[us]")""")
196+
}
197+
198+
private val allFields = Seq(
199+
DateTimeField.YEAR,
200+
DateTimeField.QUARTER,
201+
DateTimeField.MONTH,
202+
DateTimeField.DAY,
203+
DateTimeField.DAY_OF_WEEK,
204+
DateTimeField.DAY_OF_YEAR,
205+
DateTimeField.WEEK_OF_YEAR,
206+
DateTimeField.HOUR,
207+
DateTimeField.MINUTE,
208+
DateTimeField.SECOND
209+
)
210+
211+
// The moments either side of the nanosecond edge, and far past it in both
212+
// directions, read against the executor's own answer for the same moment.
213+
it should "read the same fields as the executor at and past the nanosecond edge" in {
214+
val python = resolvePython().getOrElse(cancel("No runnable python executable"))
215+
if (!canImport(python, "pandas")) cancel(s"'$python' cannot import pandas")
216+
217+
val moments = Seq(
218+
"1677-09-22 00:12:44",
219+
"2262-04-11 23:47:16",
220+
"2500-01-01 14:09:07",
221+
"1500-06-15 08:30:00",
222+
"9999-12-31 23:59:59"
223+
)
224+
val d = desc(allFields: _*)
225+
val expected = rowsOf(d, moments.map(Some(_)): _*).map(_.drop(2).mkString(","))
226+
227+
val driver =
228+
s"""import pandas as pd
229+
|
230+
|in1df = pd.DataFrame({"ts": [${moments.map(m => s""""$m"""").mkString(", ")}]})
231+
|${d.generateStandaloneCode()}
232+
|
233+
|for _row in out1df.drop(columns=["ts"]).itertuples(index=False):
234+
| print(",".join(str(_v) for _v in _row))
235+
|""".stripMargin
236+
237+
val script = Files.createTempFile("extract-datetime-bounds-", ".py")
238+
script.toFile.deleteOnExit()
239+
Files.write(script, driver.getBytes(StandardCharsets.UTF_8))
240+
val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start()
241+
val out = Source.fromInputStream(process.getInputStream).mkString
242+
process.waitFor(180, TimeUnit.SECONDS)
243+
withClue(s"python said:\n$out\nscript:\n$driver") {
244+
process.exitValue() shouldBe 0
245+
out.trim.linesIterator.toSeq shouldBe expected
246+
}
247+
}
248+
249+
private def resolvePython(): Option[String] = {
250+
def fromConfig: Option[String] =
251+
Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption
252+
.orElse(Try(ConfigFactory.load()).toOption)
253+
.flatMap(c => Try(c.getConfig("python").getString("path")).toOption)
254+
.map(_.trim)
255+
.filter(_.nonEmpty)
256+
257+
def runnable(exe: String): Boolean =
258+
Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption
259+
.exists { p =>
260+
if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false }
261+
else p.exitValue() == 0
262+
}
263+
264+
(fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable)
265+
}
266+
267+
private def canImport(python: String, modules: String): Boolean =
268+
Try(
269+
new ProcessBuilder(python, "-c", s"import $modules").redirectErrorStream(true).start()
270+
).toOption.exists { p =>
271+
if (!p.waitFor(120, TimeUnit.SECONDS)) { p.destroyForcibly(); false }
272+
else p.exitValue() == 0
273+
}
182274
}

0 commit comments

Comments
 (0)