1919
2020package org .apache .texera .amber .engine .architecture .worker
2121
22- import org .apache .texera .amber .core .executor .OperatorExecutor
23- import org .apache .texera .amber .core .state .State
22+ import com .fasterxml .jackson .annotation .JsonProperty
23+ import org .apache .texera .amber .core .executor .{ExecFactory , OpExecWithClassName , OperatorExecutor }
24+ import org .apache .texera .amber .core .state .{State , StateReferencing }
2425import org .apache .texera .amber .core .tuple .{AttributeType , Schema , Tuple , TupleLike }
2526import org .apache .texera .amber .core .virtualidentity ._
2627import org .apache .texera .amber .core .workflow .{PhysicalLink , PortIdentity }
@@ -30,14 +31,18 @@ import org.apache.texera.amber.engine.architecture.logreplay.{ReplayLogManager,
3031import org .apache .texera .amber .engine .architecture .messaginglayer .WorkerTimerService
3132import org .apache .texera .amber .engine .architecture .rpc .controlcommands .{
3233 AsyncRPCContext ,
34+ ConsoleMessageTriggeredRequest ,
3335 EmbeddedControlMessage ,
3436 EmbeddedControlMessageType ,
35- EmptyRequest
37+ EmptyRequest ,
38+ InitializeExecutorRequest
3639}
3740import org .apache .texera .amber .engine .architecture .rpc .workerservice .WorkerServiceGrpc .{
3841 METHOD_END_CHANNEL ,
3942 METHOD_FLUSH_NETWORK_BUFFER ,
40- METHOD_OPEN_EXECUTOR
43+ METHOD_INITIALIZE_EXECUTOR ,
44+ METHOD_OPEN_EXECUTOR ,
45+ METHOD_START_CHANNEL
4146}
4247import org .apache .texera .amber .engine .architecture .worker .WorkflowWorker .{
4348 DPInputQueueElement ,
@@ -52,15 +57,20 @@ import org.apache.texera.amber.engine.common.ambermessage.{
5257import org .apache .texera .amber .engine .common .rpc .AsyncRPCClient .ControlInvocation
5358import org .apache .texera .amber .engine .common .storage .SequentialRecordStorage
5459import org .apache .texera .amber .engine .common .virtualidentity .util .COORDINATOR
60+ import org .apache .texera .amber .util .JSONUtils .objectMapper
5561import org .apache .texera .amber .util .VirtualIdentityUtils
5662import org .scalamock .scalatest .MockFactory
5763import org .scalatest .BeforeAndAfterEach
5864import org .scalatest .flatspec .AnyFlatSpec
5965import org .scalatest .matchers .should .Matchers
6066
6167import java .util .concurrent .LinkedBlockingQueue
68+ import scala .collection .mutable .ArrayBuffer
6269
6370class DataProcessorSpec extends AnyFlatSpec with MockFactory with Matchers with BeforeAndAfterEach {
71+
72+ import DataProcessorSpec ._
73+
6474 private val testOpId = PhysicalOpIdentity (OperatorIdentity (" testop" ), " main" )
6575 private val upstreamOpId = PhysicalOpIdentity (OperatorIdentity (" sender" ), " main" )
6676 private val testWorkerId : ActorVirtualIdentity = VirtualIdentityUtils .createWorkerIdentity(
@@ -429,6 +439,199 @@ class DataProcessorSpec extends AnyFlatSpec with MockFactory with Matchers with
429439 dp.pauseManager.isPaused shouldBe true
430440 }
431441
442+ // ---------------------------------------------------------------------------
443+ // An operator whose setting refers to a loop variable
444+ // ---------------------------------------------------------------------------
445+
446+ private val senderChannel = ChannelIdentity (senderWorkerId, testWorkerId, isControl = false )
447+ private val coordinatorChannel = ChannelIdentity (COORDINATOR , testWorkerId, isControl = true )
448+
449+ private val startChannelPayload = EmbeddedControlMessage (
450+ EmbeddedControlMessageIdentity (" StartChannel" ),
451+ EmbeddedControlMessageType .NO_ALIGNMENT ,
452+ Seq (),
453+ Map (
454+ testWorkerId.name ->
455+ ControlInvocation (
456+ METHOD_START_CHANNEL .getBareMethodName,
457+ EmptyRequest (),
458+ AsyncRPCContext (ActorVirtualIdentity (" " ), ActorVirtualIdentity (" " )),
459+ - 1
460+ )
461+ )
462+ )
463+
464+ /**
465+ * A data processor whose executor is initialized the way the coordinator initializes a worker,
466+ * from `descString` for `LoopLimitExec`, with everything the worker sends collected.
467+ */
468+ private def initializedWith (
469+ descString : String
470+ ): (DataProcessor , LoopLimitExec , ArrayBuffer [WorkflowFIFOMessage ]) = {
471+ val dp = mkDataProcessor
472+ val sent = ArrayBuffer [WorkflowFIFOMessage ]()
473+ (outputHandler.apply _)
474+ .expects(* )
475+ .onCall { (m : Either [MainThreadDelegateMessage , WorkflowFIFOMessage ]) =>
476+ m.foreach(sent += _); ()
477+ }
478+ .anyNumberOfTimes()
479+ (adaptiveBatchingMonitor.startAdaptiveBatching _).expects().anyNumberOfTimes()
480+ dp.stateManager.transitTo(READY )
481+ dp.inputManager.addPort(inputPortId, schema, List .empty, List .empty)
482+ dp.inputGateway.getChannel(senderChannel).setPortId(inputPortId)
483+ dp.outputManager.addPort(outputPortId, schema, None )
484+ dp.outputManager.addPartitionerWithPartitioning(
485+ PhysicalLink (testOpId, outputPortId, upstreamOpId, inputPortId),
486+ OneToOnePartitioning (1 , Seq (ChannelIdentity (testWorkerId, senderWorkerId, isControl = false )))
487+ )
488+ dp.processDCM(
489+ coordinatorChannel,
490+ ControlInvocation (
491+ METHOD_INITIALIZE_EXECUTOR ,
492+ InitializeExecutorRequest (
493+ totalWorkerCount = 1 ,
494+ opExecInitInfo = OpExecWithClassName (classOf [LoopLimitExec ].getName, descString),
495+ isSource = false ,
496+ loopStartPortUris = Map .empty
497+ ),
498+ AsyncRPCContext (COORDINATOR , testWorkerId),
499+ 0
500+ )
501+ )
502+ (dp, dp.executor.asInstanceOf [LoopLimitExec ], sent)
503+ }
504+
505+ private def openExecutor (dp : DataProcessor ): Unit =
506+ dp.processDCM(
507+ coordinatorChannel,
508+ ControlInvocation (
509+ METHOD_OPEN_EXECUTOR ,
510+ EmptyRequest (),
511+ AsyncRPCContext (COORDINATOR , testWorkerId),
512+ 1
513+ )
514+ )
515+
516+ private def drain (dp : DataProcessor ): Unit =
517+ while (dp.inputManager.hasUnfinishedInput || dp.outputManager.hasUnfinishedOutput) {
518+ dp.continueDataProcessing()
519+ }
520+
521+ private def statesSent (sent : ArrayBuffer [WorkflowFIFOMessage ]): List [State ] =
522+ sent.map(_.payload).collect { case StateFrame (state, _, _) => state }.toList
523+
524+ private def tuplesSent (sent : ArrayBuffer [WorkflowFIFOMessage ]): List [Tuple ] =
525+ sent.map(_.payload).collect { case DataFrame (frame) => frame.toList }.flatten.toList
526+
527+ private def consoleTitles (sent : ArrayBuffer [WorkflowFIFOMessage ]): List [String ] =
528+ sent
529+ .map(_.payload)
530+ .collect {
531+ case invocation : org.apache.texera.amber.engine.architecture.rpc.controlcommands.ControlInvocation =>
532+ invocation.command
533+ }
534+ .collect { case request : ConsoleMessageTriggeredRequest => request.consoleMessage.title }
535+ .toList
536+
537+ private val limitReferringToN = """ {"limit":0,"stateReferences":{"/limit":"n"}}"""
538+
539+ " data processor" should " run an operator whose setting refers to a loop variable as any other, the variable written in as the state arrives" in {
540+ val (dp, exec, sent) = initializedWith(limitReferringToN)
541+ // The operator's own executor, built at worker start from the descString with its placeholder.
542+ assert(exec.setting.limit == 0 )
543+
544+ openExecutor(dp)
545+ dp.processECM(senderChannel, startChannelPayload, logManager)
546+ val first = State (Map (" n" -> 2L ))
547+ dp.processDataPayload(senderChannel, StateFrame (first))
548+ dp.processDataPayload(senderChannel, DataFrame (tuples.take(5 )))
549+ drain(dp)
550+ val later = State (Map (" n" -> 9L ))
551+ dp.processDataPayload(senderChannel, StateFrame (later))
552+
553+ // open() at worker start, before any state; produceStateOnStart on the operator itself, still
554+ // with the placeholder; then its own processState, with the message registered and the
555+ // variable written just before. A state after the first tuple is still registered and
556+ // processed, but the setting keeps what the first tuple was processed with.
557+ assert(
558+ exec.calls.toList == List (
559+ (" open" , 0 , None ),
560+ (" start" , 0 , None ),
561+ (" state" , 2 , Some (first)),
562+ (" state" , 2 , Some (later))
563+ )
564+ )
565+ // What the worker sends on is what the operator returned for each callback.
566+ assert(
567+ statesSent(sent) == List (
568+ State (Map (" start" -> 0 )),
569+ State (Map (" seen" -> 2L , " limit" -> 2 )),
570+ State (Map (" seen" -> 9L , " limit" -> 2 ))
571+ )
572+ )
573+ assert(tuplesSent(sent) == tuples.take(2 ).toList)
574+ assert(consoleTitles(sent).isEmpty)
575+ assert(! dp.pauseManager.isPaused)
576+ }
577+
578+ " data processor" should " fail the first tuple of an operator whose setting refers to a loop variable no state message carried" in {
579+ val (dp, exec, sent) = initializedWith(limitReferringToN)
580+ openExecutor(dp)
581+
582+ dp.processDataPayload(senderChannel, DataFrame (tuples.take(1 )))
583+
584+ // The operator never sees the tuple: it would be processed with the placeholder.
585+ assert(exec.tuplesSeen == 0 )
586+ assert(
587+ consoleTitles(sent) == List (
588+ new IllegalStateException (
589+ " property /limit refers to loop variable n, but no state message carried it"
590+ ).toString
591+ )
592+ )
593+ dp.pauseManager.isPaused shouldBe true
594+ }
595+
596+ " data processor" should " bind the references of an executor that replaces one already bound" in {
597+ // Reconfiguration and checkpoint restore install a new executor: its references are its own.
598+ val (dp, exec, sent) = initializedWith(limitReferringToN)
599+ openExecutor(dp)
600+ dp.processDataPayload(senderChannel, StateFrame (State (Map (" n" -> 2L ))))
601+ dp.processDataPayload(senderChannel, DataFrame (tuples.take(1 )))
602+ drain(dp)
603+ assert(exec.tuplesSeen == 1 )
604+
605+ val replacement = ExecFactory
606+ .newExecFromJavaClassName(classOf [LoopLimitExec ].getName, limitReferringToN)
607+ .asInstanceOf [LoopLimitExec ]
608+ dp.executor = replacement
609+ dp.processDataPayload(senderChannel, DataFrame (tuples.slice(1 , 2 )))
610+
611+ assert(replacement.tuplesSeen == 0 )
612+ assert(
613+ consoleTitles(sent) == List (
614+ new IllegalStateException (
615+ " property /limit refers to loop variable n, but no state message carried it"
616+ ).toString
617+ )
618+ )
619+ }
620+
621+ " data processor" should " leave the setting of an operator outside every loop block as it was parsed" in {
622+ // Outside every block the sidecar is empty: a state carrying `n` writes nothing.
623+ val (dp, exec, sent) = initializedWith(""" {"limit":1}""" )
624+ openExecutor(dp)
625+ val state = State (Map (" n" -> 2L ))
626+ dp.processDataPayload(senderChannel, StateFrame (state))
627+ dp.processDataPayload(senderChannel, DataFrame (tuples.take(3 )))
628+ drain(dp)
629+
630+ assert(exec.calls.toList == List ((" open" , 1 , None ), (" state" , 1 , Some (state))))
631+ assert(statesSent(sent) == List (State (Map (" seen" -> 2L , " limit" -> 1 ))))
632+ assert(tuplesSent(sent) == tuples.take(1 ).toList)
633+ }
634+
432635 " data processor" should " handle an exception thrown while advancing the output iterator" in {
433636 val dp = mkDataProcessor
434637 dp.executor = executor
@@ -459,3 +662,41 @@ class DataProcessorSpec extends AnyFlatSpec with MockFactory with Matchers with
459662 }
460663
461664}
665+
666+ object DataProcessorSpec {
667+
668+ /** A setting with one Int property, as Limit's. */
669+ class LimitSetting extends StateReferencing {
670+ @ JsonProperty var limit : Int = _
671+ }
672+
673+ /**
674+ * Passes the first `limit` tuples, as Limit does, and records each callback the worker makes with
675+ * the limit and the registered state at that moment. Public, for the factory's reflection.
676+ */
677+ class LoopLimitExec (descString : String ) extends OperatorExecutor {
678+ val setting : LimitSetting = objectMapper.readValue(descString, classOf [LimitSetting ])
679+ val calls : ArrayBuffer [(String , Int , Option [State ])] = ArrayBuffer ()
680+ var tuplesSeen = 0
681+
682+ private def record (call : String ): Unit = calls += ((call, setting.limit, state))
683+
684+ override def open (): Unit = record(" open" )
685+
686+ override def produceStateOnStart (port : Int ): Option [State ] = {
687+ record(" start" )
688+ Some (State (Map (" start" -> setting.limit)))
689+ }
690+
691+ // Transforms the state, so what the worker sends on shows whose return value it is.
692+ override def processState (incoming : State , port : Int ): Option [State ] = {
693+ record(" state" )
694+ Some (State (Map (" seen" -> incoming.values(" n" ), " limit" -> setting.limit)))
695+ }
696+
697+ override def processTuple (tuple : Tuple , port : Int ): Iterator [TupleLike ] = {
698+ tuplesSeen += 1
699+ if (tuplesSeen <= setting.limit) Iterator .single(tuple) else Iterator .empty
700+ }
701+ }
702+ }
0 commit comments