@@ -24,6 +24,7 @@ import org.apache.pekko.actor.{ActorSystem, Props}
2424import org .apache .pekko .testkit .{ImplicitSender , TestKit }
2525import org .apache .pekko .util .Timeout
2626import org .apache .texera .amber .clustering .SingleNodeListener
27+ import org .apache .texera .amber .core .tuple .{Attribute , AttributeType }
2728import org .apache .texera .amber .core .virtualidentity .OperatorIdentity
2829import org .apache .texera .amber .core .workflow .{
2930 ExecutionMode ,
@@ -237,9 +238,65 @@ class LoopIntegrationSpec
237238 op
238239 }
239240
241+ /** A Python UDF that adds an `i` column to every row it passes: loop variable `i`, as it reads
242+ * it off `self.loop_variables` in `process_tuple`.
243+ */
244+ private def pythonUDFReadingLoopVariableInProcessTuple (): PythonUDFOpDescV2 = {
245+ val op = new PythonUDFOpDescV2 ()
246+ op.code = """ from pytexera import *
247+ |
248+ |class ProcessTupleOperator(UDFOperatorV2):
249+ |
250+ | @overrides
251+ | def process_tuple(self, tuple_: Tuple, port: int) -> Iterator[Optional[TupleLike]]:
252+ | tuple_["i"] = self.loop_variables["i"]
253+ | yield tuple_
254+ |""" .stripMargin
255+ op.workers = 1
256+ op.retainInputColumns = true
257+ op.outputColumns = List (new Attribute (" i" , AttributeType .INTEGER ))
258+ op
259+ }
260+
261+ /** As `pythonUDFReadingLoopVariableInProcessTuple`, but reading `i` in `process_table`, which
262+ * runs once its input has ended, after every state message on it arrived.
263+ */
264+ private def pythonUDFReadingLoopVariableInProcessTable (): PythonUDFOpDescV2 = {
265+ val op = new PythonUDFOpDescV2 ()
266+ op.code = """ from pytexera import *
267+ |
268+ |class ProcessTableOperator(UDFTableOperator):
269+ |
270+ | @overrides
271+ | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]:
272+ | for line in table["line"]:
273+ | yield {"line": line, "i": self.loop_variables["i"]}
274+ |""" .stripMargin
275+ op.workers = 1
276+ op.retainInputColumns = true
277+ op.outputColumns = List (new Attribute (" i" , AttributeType .INTEGER ))
278+ op
279+ }
280+
240281 private def link (from : LogicalOp , to : LogicalOp ): LogicalLink =
241282 LogicalLink (from.operatorIdentifier, PortIdentity (), to.operatorIdentifier, PortIdentity ())
242283
284+ /** Run the loop workflow and return the (line, i) rows the Loop End `end` materialized, sorted. */
285+ private def runAndGetLinesWithLoopVariable (
286+ operators : List [LogicalOp ],
287+ links : List [LogicalLink ],
288+ end : LogicalOp
289+ ): List [(String , Int )] =
290+ runWorkflowAndReadResults(
291+ system,
292+ buildWorkflow(operators, links, materializedContext()),
293+ List (end.operatorIdentifier),
294+ _.get().toList,
295+ Duration .fromSeconds(90 )
296+ ).getOrElse(end.operatorIdentifier, Nil )
297+ .map(row => (row.getField[String ](" line" ), row.getField[Int ](" i" )))
298+ .sorted
299+
243300 " Engine" should " run a single TextInput -> LoopStart -> LoopEnd loop for exactly 3 iterations" in {
244301 val src = textInput(" 1\n 2\n 3" )
245302 val start = loopStart(" i = 0" , " table.iloc[i]" )
@@ -496,4 +553,47 @@ class LoopIntegrationSpec
496553 )
497554 }
498555
556+ it should " read a loop variable in a Python UDF on every iteration" in {
557+ // TextInput("a","b","c") -> LoopStart(i = 0 / table.iloc[i]) -> Python UDF -> LoopEnd.
558+ // Iteration i hands the UDF the row of index i, and the UDF adds the i it reads off
559+ // self.loop_variables in process_tuple, so the LoopEnd accumulates (a, 0), (b, 1), (c, 2).
560+ // A UDF that kept reading the first iteration's i would write 0 on every row.
561+ val src = textInput(" a\n b\n c" )
562+ val start = loopStart(" i = 0" , " table.iloc[i]" )
563+ val mid = pythonUDFReadingLoopVariableInProcessTuple()
564+ val end = loopEnd(" i += 1" , " i < len(table)" )
565+ val rows = runAndGetLinesWithLoopVariable(
566+ List (src, start, mid, end),
567+ List (link(src, start), link(start, mid), link(mid, end)),
568+ end
569+ )
570+ assert(
571+ rows == List ((" a" , 0 ), (" b" , 1 ), (" c" , 2 )),
572+ s " each iteration's row must carry that iteration's i: got $rows"
573+ )
574+ }
575+
576+ it should " read a loop variable in a Python UDF after another body operator's own state arrives" in {
577+ // TextInput("a","b","c") -> LoopStart(i = 0 / table.iloc[i]) -> stateful Python UDF
578+ // -> Python UDF reading i in process_table -> LoopEnd.
579+ // The first UDF returns its own state, {"note": ...}, from produce_state_on_finish, so the
580+ // second receives that state after the loop's. process_table runs once the input ends, after
581+ // both arrived, and must still read the loop's i: that state names no loop variable, so it
582+ // must not hide one. Reading only the latest state message would fail with KeyError 'i'.
583+ val src = textInput(" a\n b\n c" )
584+ val start = loopStart(" i = 0" , " table.iloc[i]" )
585+ val first = statefulPythonUDF()
586+ val second = pythonUDFReadingLoopVariableInProcessTable()
587+ val end = loopEnd(" i += 1" , " i < len(table)" )
588+ val rows = runAndGetLinesWithLoopVariable(
589+ List (src, start, first, second, end),
590+ List (link(src, start), link(start, first), link(first, second), link(second, end)),
591+ end
592+ )
593+ assert(
594+ rows == List ((" a" , 0 ), (" b" , 1 ), (" c" , 2 )),
595+ s " each iteration's row must carry that iteration's i: got $rows"
596+ )
597+ }
598+
499599}
0 commit comments