Skip to content

Commit f0eedd9

Browse files
kz930claude
andcommitted
feat(translator): hand a Python operator its table the way pyamber does
pyamber hands a Python operator a DOUBLE column as float64, where a NaN and a null are one value, and writes a NaN out as null. The double cast here builds Float64, which keeps them apart, so a Python operator after it answered differently: Sort ordered a NaN apart from the nulls, a chart drew a point the engine dropped, and a passed-through NaN stayed NaN. StandaloneCodeGenerator gains standaloneInputAdapter(), and for a Python operator the translator feeds the fragment a converted copy of each input, leaving the upstream frame as it was. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 7a3203d commit f0eedd9

3 files changed

Lines changed: 108 additions & 3 deletions

File tree

‎common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneCodeGenerator.scala‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,4 +109,39 @@ trait StandaloneCodeGenerator {
109109
* to start.
110110
*/
111111
def standaloneImports(): Seq[String] = Seq.empty
112+
113+
/**
114+
* A helper each input frame goes through before this operator's code runs,
115+
* for an operator the engine hands its table in a different form than the
116+
* script's upstream does. The translator defines it once and feeds the
117+
* operator what it returns, leaving the upstream frame as it was.
118+
*
119+
* A Python operator runs in pyamber, which hands it a DOUBLE column as
120+
* float64, where a NaN and a null are one value. A source can hand the script
121+
* a Float64 column that keeps them apart, and an operator written against
122+
* float64 then answers differently.
123+
*/
124+
def standaloneInputAdapter(): Option[StandaloneCodeGenerator.InputAdapter] =
125+
this match {
126+
case _: PythonOperatorDescriptor => Some(StandaloneCodeGenerator.PythonOperatorInput)
127+
case _ => None
128+
}
129+
}
130+
131+
object StandaloneCodeGenerator {
132+
133+
/** A helper's name and the definition that binds it. */
134+
final case class InputAdapter(name: String, definition: String)
135+
136+
val PythonOperatorInput: InputAdapter = InputAdapter(
137+
"_texera_python_operator_input",
138+
"""def _texera_python_operator_input(df):
139+
| # pyamber hands a Python operator float64, where a NaN and a null are one
140+
| # value. A copy, so the upstream frame another operator reads stays as it was.
141+
| df = df.copy()
142+
| for _c in df.columns:
143+
| if isinstance(df[_c].dtype, (pd.Float64Dtype, pd.Float32Dtype)):
144+
| df[_c] = df[_c].astype("float64")
145+
| return df""".stripMargin
146+
)
112147
}

‎workflow-compiling-service/src/main/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslator.scala‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,10 @@ class WorkflowToPythonTranslator extends LazyLogging {
8686
// order, which keeps the script stable for a given plan.
8787
val helpers = topoOrder
8888
.map(logicalPlan.getOperator)
89-
.collect { case gen: StandaloneCodeGenerator => gen.standaloneHelpers() }
89+
.collect {
90+
case gen: StandaloneCodeGenerator =>
91+
gen.standaloneHelpers() ++ gen.standaloneInputAdapter().map(_.definition)
92+
}
9093
.flatten
9194
.distinct
9295
if (helpers.nonEmpty) {
@@ -140,11 +143,24 @@ class WorkflowToPythonTranslator extends LazyLogging {
140143
}
141144
.toMap
142145

146+
// An operator the engine hands its table in another form reads it in
147+
// that form here too, under a name of its own, so the upstream frame
148+
// another operator reads is not the one converted.
149+
val fedVars = gen.standaloneInputAdapter() match {
150+
case Some(adapter) =>
151+
inVars.map { v =>
152+
val fed = s"${v}_${adapter.name.stripPrefix("_texera_")}"
153+
script += s"$fed = ${adapter.name}($v)"
154+
fed
155+
}
156+
case None => inVars
157+
}
158+
143159
// generateStandaloneCode() returns a code block using in{N}df / out{N}df
144160
// placeholders; substituteVars() replaces them with the assigned vars.
145161
script += substituteVars(
146162
gen.generateStandaloneCode(inputSchemas),
147-
inVars,
163+
fedVars,
148164
outVars,
149165
fileBase(displayName, fileBaseCounts),
150166
displayName

‎workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslatorSpec.scala‎

Lines changed: 55 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,12 @@ package org.apache.texera.amber.translator
2222
import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity}
2323
import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PhysicalOp, PortIdentity}
2424
import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo}
25-
import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator}
25+
import org.apache.texera.amber.core.tuple.Schema
26+
import org.apache.texera.amber.operator.{
27+
LogicalOp,
28+
PythonOperatorDescriptor,
29+
StandaloneCodeGenerator
30+
}
2631
import org.apache.texera.common.compiler.model.{LogicalLink, LogicalPlan}
2732
import org.scalatest.flatspec.AnyFlatSpec
2833
import org.scalatest.matchers.should.Matchers
@@ -63,6 +68,21 @@ class WorkflowToPythonTranslatorSpec extends AnyFlatSpec with Matchers {
6368
op
6469
}
6570

71+
/** A stub the engine would run in pyamber, which is all the input adapter asks. */
72+
private class PythonStubOp(block: String) extends StubOp(block) with PythonOperatorDescriptor {
73+
override def generatePythonCode(): String = ""
74+
override def getOutputSchemas(
75+
inputSchemas: Map[PortIdentity, Schema]
76+
): Map[PortIdentity, Schema] =
77+
Map.empty
78+
}
79+
80+
private def pythonStub(id: String, block: String): LogicalOp = {
81+
val op = new PythonStubOp(block)
82+
op.setOperatorId(id)
83+
op
84+
}
85+
6686
private def upstream(id: String): LogicalOp = stub(id, "out1df = in1df.copy()")
6787

6888
private def link(from: LogicalOp, to: LogicalOp): LogicalLink =
@@ -134,6 +154,40 @@ class WorkflowToPythonTranslatorSpec extends AnyFlatSpec with Matchers {
134154
script should include("df2 = df1.copy()")
135155
}
136156

157+
/** pyamber hands a Python operator float64, where a NaN and a null are one value,
158+
* and a source can hand the script Float64, which keeps them apart. The
159+
* operator reads a converted copy, so the frame another operator reads from the
160+
* same upstream is the one the source wrote.
161+
*/
162+
it should "feed a Python operator a converted copy of each input" in {
163+
val source = upstream("source")
164+
val python = pythonStub("python", "out1df = in1df.copy()")
165+
val script = new WorkflowToPythonTranslator().translate(
166+
LogicalPlan(List(source, python), List(link(source, python)))
167+
)
168+
script should include("df1_python_operator_input = _texera_python_operator_input(df1)")
169+
script should include("df2 = df1_python_operator_input.copy()")
170+
}
171+
172+
it should "leave an operator the engine runs on the JVM reading its upstream as it is" in {
173+
val first = upstream("first")
174+
val second = upstream("second")
175+
val script = new WorkflowToPythonTranslator().translate(
176+
LogicalPlan(List(first, second), List(link(first, second)))
177+
)
178+
script should not include "_texera_python_operator_input"
179+
}
180+
181+
it should "define the Python operator input helper once and leave the upstream frame alone" in {
182+
val source = upstream("source")
183+
val pythons = List("a", "b").map(id => pythonStub(id, "out1df = in1df.copy()"))
184+
val script = new WorkflowToPythonTranslator().translate(
185+
LogicalPlan(source :: pythons, pythons.map(link(source, _)))
186+
)
187+
script.linesIterator.count(_ == "def _texera_python_operator_input(df):") shouldBe 1
188+
script should not include "df1 = _texera_python_operator_input("
189+
}
190+
137191
/** Nothing stops a column from being named after a placeholder. The
138192
* substitution rewrites the variable a block reads with, never the column
139193
* name it asks that variable for.

0 commit comments

Comments
 (0)