Skip to content

Commit 64c8160

Browse files
committed
feat(amber): bind $i in the operator's setting when the state arrives
1 parent 7cbdce4 commit 64c8160

25 files changed

Lines changed: 1432 additions & 530 deletions

File tree

‎amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessor.scala‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,9 @@ class DataProcessor(
7171

7272
@transient var executor: OperatorExecutor = _
7373

74+
/** The executor whose loop-variable references `bindStateReferences` has bound. */
75+
@transient private var executorWithBoundReferences: OperatorExecutor = _
76+
7477
def initTimerService(adaptiveBatchingMonitor: WorkerTimerService): Unit = {
7578
this.adaptiveBatchingMonitor = adaptiveBatchingMonitor
7679
}
@@ -97,6 +100,18 @@ class DataProcessor(
97100
def collectStatistics(): WorkerStatistics =
98101
statisticsManager.getStatistics(executor)
99102

103+
/**
104+
* Binds the executor's loop-variable references right before it first sees data or finishes
105+
* (`OperatorExecutor.bindStateReferences`). Once they are bound, and for an executor that has
106+
* none, this costs a reference comparison per tuple; an executor installed in its place binds
107+
* its own.
108+
*/
109+
def bindStateReferences(): Unit =
110+
if (executorWithBoundReferences ne executor) {
111+
executor.bindStateReferences()
112+
executorWithBoundReferences = executor
113+
}
114+
100115
/**
101116
* process currentInputTuple through executor logic.
102117
* this function is only called by the DP thread.
@@ -105,6 +120,8 @@ class DataProcessor(
105120
try {
106121
val portIdentity: PortIdentity =
107122
this.inputGateway.getChannel(inputManager.currentChannelId).getPortId
123+
// The executor sees data: a setting that refers to loop variables must have them by now.
124+
bindStateReferences()
108125
outputManager.outputIterator.setTupleOutput(
109126
executor.processTupleMultiPort(
110127
tuple,
@@ -128,6 +145,9 @@ class DataProcessor(
128145
loopStartId: String
129146
): Unit = {
130147
try {
148+
// Registered before the callback runs, so that processState (and every call after it) can
149+
// consult it, with the loop variables it carries already written into the setting.
150+
executor.registerState(state)
131151
val outputState = executor.processState(state, port)
132152
if (outputState.isDefined) {
133153
// Carry the incoming loop envelope through unchanged: loop operators

‎amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/EndChannelHandler.scala‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@ trait EndChannelHandler {
4141
dp.inputManager.getPort(portId).completed = true
4242
dp.inputManager.initBatch(channelId, Array.empty)
4343
try {
44+
// The executor finishes: a setting that refers to loop variables must have them by now.
45+
dp.bindStateReferences()
4446
val outputState = dp.executor.produceStateOnFinish(portId.id)
4547
if (outputState.isDefined) {
4648
// Operator-ORIGINATED boundary state, so no LoopStart stamp

‎amber/src/test/integration/org/apache/texera/amber/engine/e2e/LoopIntegrationSpec.scala‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -333,10 +333,10 @@ class LoopIntegrationSpec
333333
// TextInput("0","1","2") -> LoopStart(i = 0 / table.iloc[i]) -> Filter(line = $i) -> LoopEnd.
334334
//
335335
// The Filter's predicate value is the literal string "$i". The Filter sits inside the
336-
// block, so the compiler records "$i" in its sidecar, and ExecFactory hands the
337-
// worker a LateBoundExecutor, which binds "$i" to the iteration's `i` when the LoopStart's
338-
// state message arrives -- the region's input reader replays states before tuples -- and
339-
// only then instantiates the real Filter. Iteration i emits the single row
336+
// block, so the compiler records "$i" in its sidecar, and the worker writes the
337+
// iteration's `i` into the predicate the Filter's executor parsed when the LoopStart's
338+
// state message arrives -- the region's input reader replays states before tuples -- so
339+
// the Filter's first tuple already compares against it. Iteration i emits the single row
340340
// "i" and the Filter compares it against i, so every iteration's row passes and the
341341
// LoopEnd accumulates 3 rows. A constant "0" instead of "$i" would pass only the first
342342
// iteration's row and give 1; a reference that never rebinds to the new i would too.

‎amber/src/test/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessorSpec.scala‎

Lines changed: 245 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,9 @@
1919

2020
package org.apache.texera.amber.engine.architecture.worker
2121

22-
import org.apache.texera.amber.core.executor.OperatorExecutor
23-
import org.apache.texera.amber.core.state.State
22+
import com.fasterxml.jackson.annotation.JsonProperty
23+
import org.apache.texera.amber.core.executor.{ExecFactory, OpExecWithClassName, OperatorExecutor}
24+
import org.apache.texera.amber.core.state.{State, StateReferencing}
2425
import org.apache.texera.amber.core.tuple.{AttributeType, Schema, Tuple, TupleLike}
2526
import org.apache.texera.amber.core.virtualidentity._
2627
import org.apache.texera.amber.core.workflow.{PhysicalLink, PortIdentity}
@@ -30,14 +31,18 @@ import org.apache.texera.amber.engine.architecture.logreplay.{ReplayLogManager,
3031
import org.apache.texera.amber.engine.architecture.messaginglayer.WorkerTimerService
3132
import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{
3233
AsyncRPCContext,
34+
ConsoleMessageTriggeredRequest,
3335
EmbeddedControlMessage,
3436
EmbeddedControlMessageType,
35-
EmptyRequest
37+
EmptyRequest,
38+
InitializeExecutorRequest
3639
}
3740
import org.apache.texera.amber.engine.architecture.rpc.workerservice.WorkerServiceGrpc.{
3841
METHOD_END_CHANNEL,
3942
METHOD_FLUSH_NETWORK_BUFFER,
40-
METHOD_OPEN_EXECUTOR
43+
METHOD_INITIALIZE_EXECUTOR,
44+
METHOD_OPEN_EXECUTOR,
45+
METHOD_START_CHANNEL
4146
}
4247
import org.apache.texera.amber.engine.architecture.worker.WorkflowWorker.{
4348
DPInputQueueElement,
@@ -52,15 +57,20 @@ import org.apache.texera.amber.engine.common.ambermessage.{
5257
import org.apache.texera.amber.engine.common.rpc.AsyncRPCClient.ControlInvocation
5358
import org.apache.texera.amber.engine.common.storage.SequentialRecordStorage
5459
import org.apache.texera.amber.engine.common.virtualidentity.util.COORDINATOR
60+
import org.apache.texera.amber.util.JSONUtils.objectMapper
5561
import org.apache.texera.amber.util.VirtualIdentityUtils
5662
import org.scalamock.scalatest.MockFactory
5763
import org.scalatest.BeforeAndAfterEach
5864
import org.scalatest.flatspec.AnyFlatSpec
5965
import org.scalatest.matchers.should.Matchers
6066

6167
import java.util.concurrent.LinkedBlockingQueue
68+
import scala.collection.mutable.ArrayBuffer
6269

6370
class DataProcessorSpec extends AnyFlatSpec with MockFactory with Matchers with BeforeAndAfterEach {
71+
72+
import DataProcessorSpec._
73+
6474
private val testOpId = PhysicalOpIdentity(OperatorIdentity("testop"), "main")
6575
private val upstreamOpId = PhysicalOpIdentity(OperatorIdentity("sender"), "main")
6676
private val testWorkerId: ActorVirtualIdentity = VirtualIdentityUtils.createWorkerIdentity(
@@ -429,6 +439,199 @@ class DataProcessorSpec extends AnyFlatSpec with MockFactory with Matchers with
429439
dp.pauseManager.isPaused shouldBe true
430440
}
431441

442+
// ---------------------------------------------------------------------------
443+
// An operator whose setting refers to a loop variable
444+
// ---------------------------------------------------------------------------
445+
446+
private val senderChannel = ChannelIdentity(senderWorkerId, testWorkerId, isControl = false)
447+
private val coordinatorChannel = ChannelIdentity(COORDINATOR, testWorkerId, isControl = true)
448+
449+
private val startChannelPayload = EmbeddedControlMessage(
450+
EmbeddedControlMessageIdentity("StartChannel"),
451+
EmbeddedControlMessageType.NO_ALIGNMENT,
452+
Seq(),
453+
Map(
454+
testWorkerId.name ->
455+
ControlInvocation(
456+
METHOD_START_CHANNEL.getBareMethodName,
457+
EmptyRequest(),
458+
AsyncRPCContext(ActorVirtualIdentity(""), ActorVirtualIdentity("")),
459+
-1
460+
)
461+
)
462+
)
463+
464+
/**
465+
* A data processor whose executor is initialized the way the coordinator initializes a worker,
466+
* from `descString` for `LoopLimitExec`, with everything the worker sends collected.
467+
*/
468+
private def initializedWith(
469+
descString: String
470+
): (DataProcessor, LoopLimitExec, ArrayBuffer[WorkflowFIFOMessage]) = {
471+
val dp = mkDataProcessor
472+
val sent = ArrayBuffer[WorkflowFIFOMessage]()
473+
(outputHandler.apply _)
474+
.expects(*)
475+
.onCall { (m: Either[MainThreadDelegateMessage, WorkflowFIFOMessage]) =>
476+
m.foreach(sent += _); ()
477+
}
478+
.anyNumberOfTimes()
479+
(adaptiveBatchingMonitor.startAdaptiveBatching _).expects().anyNumberOfTimes()
480+
dp.stateManager.transitTo(READY)
481+
dp.inputManager.addPort(inputPortId, schema, List.empty, List.empty)
482+
dp.inputGateway.getChannel(senderChannel).setPortId(inputPortId)
483+
dp.outputManager.addPort(outputPortId, schema, None)
484+
dp.outputManager.addPartitionerWithPartitioning(
485+
PhysicalLink(testOpId, outputPortId, upstreamOpId, inputPortId),
486+
OneToOnePartitioning(1, Seq(ChannelIdentity(testWorkerId, senderWorkerId, isControl = false)))
487+
)
488+
dp.processDCM(
489+
coordinatorChannel,
490+
ControlInvocation(
491+
METHOD_INITIALIZE_EXECUTOR,
492+
InitializeExecutorRequest(
493+
totalWorkerCount = 1,
494+
opExecInitInfo = OpExecWithClassName(classOf[LoopLimitExec].getName, descString),
495+
isSource = false,
496+
loopStartPortUris = Map.empty
497+
),
498+
AsyncRPCContext(COORDINATOR, testWorkerId),
499+
0
500+
)
501+
)
502+
(dp, dp.executor.asInstanceOf[LoopLimitExec], sent)
503+
}
504+
505+
private def openExecutor(dp: DataProcessor): Unit =
506+
dp.processDCM(
507+
coordinatorChannel,
508+
ControlInvocation(
509+
METHOD_OPEN_EXECUTOR,
510+
EmptyRequest(),
511+
AsyncRPCContext(COORDINATOR, testWorkerId),
512+
1
513+
)
514+
)
515+
516+
private def drain(dp: DataProcessor): Unit =
517+
while (dp.inputManager.hasUnfinishedInput || dp.outputManager.hasUnfinishedOutput) {
518+
dp.continueDataProcessing()
519+
}
520+
521+
private def statesSent(sent: ArrayBuffer[WorkflowFIFOMessage]): List[State] =
522+
sent.map(_.payload).collect { case StateFrame(state, _, _) => state }.toList
523+
524+
private def tuplesSent(sent: ArrayBuffer[WorkflowFIFOMessage]): List[Tuple] =
525+
sent.map(_.payload).collect { case DataFrame(frame) => frame.toList }.flatten.toList
526+
527+
private def consoleTitles(sent: ArrayBuffer[WorkflowFIFOMessage]): List[String] =
528+
sent
529+
.map(_.payload)
530+
.collect {
531+
case invocation: org.apache.texera.amber.engine.architecture.rpc.controlcommands.ControlInvocation =>
532+
invocation.command
533+
}
534+
.collect { case request: ConsoleMessageTriggeredRequest => request.consoleMessage.title }
535+
.toList
536+
537+
private val limitReferringToN = """{"limit":0,"stateReferences":{"/limit":"n"}}"""
538+
539+
"data processor" should "run an operator whose setting refers to a loop variable as any other, the variable written in as the state arrives" in {
540+
val (dp, exec, sent) = initializedWith(limitReferringToN)
541+
// The operator's own executor, built at worker start from the descString with its placeholder.
542+
assert(exec.setting.limit == 0)
543+
544+
openExecutor(dp)
545+
dp.processECM(senderChannel, startChannelPayload, logManager)
546+
val first = State(Map("n" -> 2L))
547+
dp.processDataPayload(senderChannel, StateFrame(first))
548+
dp.processDataPayload(senderChannel, DataFrame(tuples.take(5)))
549+
drain(dp)
550+
val later = State(Map("n" -> 9L))
551+
dp.processDataPayload(senderChannel, StateFrame(later))
552+
553+
// open() at worker start, before any state; produceStateOnStart on the operator itself, still
554+
// with the placeholder; then its own processState, with the message registered and the
555+
// variable written just before. A state after the first tuple is still registered and
556+
// processed, but the setting keeps what the first tuple was processed with.
557+
assert(
558+
exec.calls.toList == List(
559+
("open", 0, None),
560+
("start", 0, None),
561+
("state", 2, Some(first)),
562+
("state", 2, Some(later))
563+
)
564+
)
565+
// What the worker sends on is what the operator returned for each callback.
566+
assert(
567+
statesSent(sent) == List(
568+
State(Map("start" -> 0)),
569+
State(Map("seen" -> 2L, "limit" -> 2)),
570+
State(Map("seen" -> 9L, "limit" -> 2))
571+
)
572+
)
573+
assert(tuplesSent(sent) == tuples.take(2).toList)
574+
assert(consoleTitles(sent).isEmpty)
575+
assert(!dp.pauseManager.isPaused)
576+
}
577+
578+
"data processor" should "fail the first tuple of an operator whose setting refers to a loop variable no state message carried" in {
579+
val (dp, exec, sent) = initializedWith(limitReferringToN)
580+
openExecutor(dp)
581+
582+
dp.processDataPayload(senderChannel, DataFrame(tuples.take(1)))
583+
584+
// The operator never sees the tuple: it would be processed with the placeholder.
585+
assert(exec.tuplesSeen == 0)
586+
assert(
587+
consoleTitles(sent) == List(
588+
new IllegalStateException(
589+
"property /limit refers to loop variable n, but no state message carried it"
590+
).toString
591+
)
592+
)
593+
dp.pauseManager.isPaused shouldBe true
594+
}
595+
596+
"data processor" should "bind the references of an executor that replaces one already bound" in {
597+
// Reconfiguration and checkpoint restore install a new executor: its references are its own.
598+
val (dp, exec, sent) = initializedWith(limitReferringToN)
599+
openExecutor(dp)
600+
dp.processDataPayload(senderChannel, StateFrame(State(Map("n" -> 2L))))
601+
dp.processDataPayload(senderChannel, DataFrame(tuples.take(1)))
602+
drain(dp)
603+
assert(exec.tuplesSeen == 1)
604+
605+
val replacement = ExecFactory
606+
.newExecFromJavaClassName(classOf[LoopLimitExec].getName, limitReferringToN)
607+
.asInstanceOf[LoopLimitExec]
608+
dp.executor = replacement
609+
dp.processDataPayload(senderChannel, DataFrame(tuples.slice(1, 2)))
610+
611+
assert(replacement.tuplesSeen == 0)
612+
assert(
613+
consoleTitles(sent) == List(
614+
new IllegalStateException(
615+
"property /limit refers to loop variable n, but no state message carried it"
616+
).toString
617+
)
618+
)
619+
}
620+
621+
"data processor" should "leave the setting of an operator outside every loop block as it was parsed" in {
622+
// Outside every block the sidecar is empty: a state carrying `n` writes nothing.
623+
val (dp, exec, sent) = initializedWith("""{"limit":1}""")
624+
openExecutor(dp)
625+
val state = State(Map("n" -> 2L))
626+
dp.processDataPayload(senderChannel, StateFrame(state))
627+
dp.processDataPayload(senderChannel, DataFrame(tuples.take(3)))
628+
drain(dp)
629+
630+
assert(exec.calls.toList == List(("open", 1, None), ("state", 1, Some(state))))
631+
assert(statesSent(sent) == List(State(Map("seen" -> 2L, "limit" -> 1))))
632+
assert(tuplesSent(sent) == tuples.take(1).toList)
633+
}
634+
432635
"data processor" should "handle an exception thrown while advancing the output iterator" in {
433636
val dp = mkDataProcessor
434637
dp.executor = executor
@@ -459,3 +662,41 @@ class DataProcessorSpec extends AnyFlatSpec with MockFactory with Matchers with
459662
}
460663

461664
}
665+
666+
object DataProcessorSpec {
667+
668+
/** A setting with one Int property, as Limit's. */
669+
class LimitSetting extends StateReferencing {
670+
@JsonProperty var limit: Int = _
671+
}
672+
673+
/**
674+
* Passes the first `limit` tuples, as Limit does, and records each callback the worker makes with
675+
* the limit and the registered state at that moment. Public, for the factory's reflection.
676+
*/
677+
class LoopLimitExec(descString: String) extends OperatorExecutor {
678+
val setting: LimitSetting = objectMapper.readValue(descString, classOf[LimitSetting])
679+
val calls: ArrayBuffer[(String, Int, Option[State])] = ArrayBuffer()
680+
var tuplesSeen = 0
681+
682+
private def record(call: String): Unit = calls += ((call, setting.limit, state))
683+
684+
override def open(): Unit = record("open")
685+
686+
override def produceStateOnStart(port: Int): Option[State] = {
687+
record("start")
688+
Some(State(Map("start" -> setting.limit)))
689+
}
690+
691+
// Transforms the state, so what the worker sends on shows whose return value it is.
692+
override def processState(incoming: State, port: Int): Option[State] = {
693+
record("state")
694+
Some(State(Map("seen" -> incoming.values("n"), "limit" -> setting.limit)))
695+
}
696+
697+
override def processTuple(tuple: Tuple, port: Int): Iterator[TupleLike] = {
698+
tuplesSeen += 1
699+
if (tuplesSeen <= setting.limit) Iterator.single(tuple) else Iterator.empty
700+
}
701+
}
702+
}

0 commit comments

Comments
 (0)