Skip to content

Commit cb773d8

Browse files
committed
test(amber): read a loop variable in a Python UDF on every iteration
Two LoopIntegrationSpec cases run a Python UDF that reads `self.loop_variables["i"]` inside a real loop and writes it into an `i` column, and assert the (line, i) rows the Loop End materialized: - TextInput -> LoopStart -> UDF (process_tuple) -> LoopEnd: each of the three iterations' rows carries that iteration's i. - TextInput -> LoopStart -> UDF returning its own state from produce_state_on_finish -> UDF reading i in process_table -> LoopEnd: the second UDF still reads the loop's i after the first one's state arrives. Before the previous commit it failed with KeyError 'i'.
1 parent ad9b1e2 commit cb773d8

1 file changed

Lines changed: 100 additions & 0 deletions

File tree

‎amber/src/test/integration/org/apache/texera/amber/engine/e2e/LoopIntegrationSpec.scala‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import org.apache.pekko.actor.{ActorSystem, Props}
2424
import org.apache.pekko.testkit.{ImplicitSender, TestKit}
2525
import org.apache.pekko.util.Timeout
2626
import org.apache.texera.amber.clustering.SingleNodeListener
27+
import org.apache.texera.amber.core.tuple.{Attribute, AttributeType}
2728
import org.apache.texera.amber.core.virtualidentity.OperatorIdentity
2829
import org.apache.texera.amber.core.workflow.{
2930
ExecutionMode,
@@ -237,9 +238,65 @@ class LoopIntegrationSpec
237238
op
238239
}
239240

241+
/** A Python UDF that adds an `i` column to every row it passes: loop variable `i`, as it reads
242+
* it off `self.loop_variables` in `process_tuple`.
243+
*/
244+
private def pythonUDFReadingLoopVariableInProcessTuple(): PythonUDFOpDescV2 = {
245+
val op = new PythonUDFOpDescV2()
246+
op.code = """from pytexera import *
247+
|
248+
|class ProcessTupleOperator(UDFOperatorV2):
249+
|
250+
| @overrides
251+
| def process_tuple(self, tuple_: Tuple, port: int) -> Iterator[Optional[TupleLike]]:
252+
| tuple_["i"] = self.loop_variables["i"]
253+
| yield tuple_
254+
|""".stripMargin
255+
op.workers = 1
256+
op.retainInputColumns = true
257+
op.outputColumns = List(new Attribute("i", AttributeType.INTEGER))
258+
op
259+
}
260+
261+
/** As `pythonUDFReadingLoopVariableInProcessTuple`, but reading `i` in `process_table`, which
262+
* runs once its input has ended, after every state message on it arrived.
263+
*/
264+
private def pythonUDFReadingLoopVariableInProcessTable(): PythonUDFOpDescV2 = {
265+
val op = new PythonUDFOpDescV2()
266+
op.code = """from pytexera import *
267+
|
268+
|class ProcessTableOperator(UDFTableOperator):
269+
|
270+
| @overrides
271+
| def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]:
272+
| for line in table["line"]:
273+
| yield {"line": line, "i": self.loop_variables["i"]}
274+
|""".stripMargin
275+
op.workers = 1
276+
op.retainInputColumns = true
277+
op.outputColumns = List(new Attribute("i", AttributeType.INTEGER))
278+
op
279+
}
280+
240281
private def link(from: LogicalOp, to: LogicalOp): LogicalLink =
241282
LogicalLink(from.operatorIdentifier, PortIdentity(), to.operatorIdentifier, PortIdentity())
242283

284+
/** Run the loop workflow and return the (line, i) rows the Loop End `end` materialized, sorted. */
285+
private def runAndGetLinesWithLoopVariable(
286+
operators: List[LogicalOp],
287+
links: List[LogicalLink],
288+
end: LogicalOp
289+
): List[(String, Int)] =
290+
runWorkflowAndReadResults(
291+
system,
292+
buildWorkflow(operators, links, materializedContext()),
293+
List(end.operatorIdentifier),
294+
_.get().toList,
295+
Duration.fromSeconds(90)
296+
).getOrElse(end.operatorIdentifier, Nil)
297+
.map(row => (row.getField[String]("line"), row.getField[Int]("i")))
298+
.sorted
299+
243300
"Engine" should "run a single TextInput -> LoopStart -> LoopEnd loop for exactly 3 iterations" in {
244301
val src = textInput("1\n2\n3")
245302
val start = loopStart("i = 0", "table.iloc[i]")
@@ -496,4 +553,47 @@ class LoopIntegrationSpec
496553
)
497554
}
498555

556+
it should "read a loop variable in a Python UDF on every iteration" in {
557+
// TextInput("a","b","c") -> LoopStart(i = 0 / table.iloc[i]) -> Python UDF -> LoopEnd.
558+
// Iteration i hands the UDF the row of index i, and the UDF adds the i it reads off
559+
// self.loop_variables in process_tuple, so the LoopEnd accumulates (a, 0), (b, 1), (c, 2).
560+
// A UDF that kept reading the first iteration's i would write 0 on every row.
561+
val src = textInput("a\nb\nc")
562+
val start = loopStart("i = 0", "table.iloc[i]")
563+
val mid = pythonUDFReadingLoopVariableInProcessTuple()
564+
val end = loopEnd("i += 1", "i < len(table)")
565+
val rows = runAndGetLinesWithLoopVariable(
566+
List(src, start, mid, end),
567+
List(link(src, start), link(start, mid), link(mid, end)),
568+
end
569+
)
570+
assert(
571+
rows == List(("a", 0), ("b", 1), ("c", 2)),
572+
s"each iteration's row must carry that iteration's i: got $rows"
573+
)
574+
}
575+
576+
it should "read a loop variable in a Python UDF after another body operator's own state arrives" in {
577+
// TextInput("a","b","c") -> LoopStart(i = 0 / table.iloc[i]) -> stateful Python UDF
578+
// -> Python UDF reading i in process_table -> LoopEnd.
579+
// The first UDF returns its own state, {"note": ...}, from produce_state_on_finish, so the
580+
// second receives that state after the loop's. process_table runs once the input ends, after
581+
// both arrived, and must still read the loop's i: that state names no loop variable, so it
582+
// must not hide one. Reading only the latest state message would fail with KeyError 'i'.
583+
val src = textInput("a\nb\nc")
584+
val start = loopStart("i = 0", "table.iloc[i]")
585+
val first = statefulPythonUDF()
586+
val second = pythonUDFReadingLoopVariableInProcessTable()
587+
val end = loopEnd("i += 1", "i < len(table)")
588+
val rows = runAndGetLinesWithLoopVariable(
589+
List(src, start, first, second, end),
590+
List(link(src, start), link(start, first), link(first, second), link(second, end)),
591+
end
592+
)
593+
assert(
594+
rows == List(("a", 0), ("b", 1), ("c", 2)),
595+
s"each iteration's row must carry that iteration's i: got $rows"
596+
)
597+
}
598+
499599
}

0 commit comments

Comments
 (0)