Skip to content

Commit ccf1079

Browse files
committed
feat(amber): bind $K loop-variable references in operators
A property of any operator inside a control block may name a loop variable as `$K`. The frontend keeps sending the literal string, whatever the property's type; the backend now handles it in two steps. At plan-parse time a Jackson module (StateReferenceModule, registered on JSONUtils.objectMapper and on the three Dropwizard mappers) wraps the deserializer of every StateReferencing bean -- LogicalOp now is one. An object that holds a `$K` (or a client-sent sidecar) is rewritten: each `$K` leaf becomes a placeholder of the property's JSON type (0 / 0.0 / false; a String keeps the literal) and the property's JSON pointer -> variable name is recorded in the `stateReferences` sidecar, which the property panel never shows. Every other object is re-fed untouched from a token buffer. At the worker, ExecFactory hands out a LateBoundExecutor whenever the descString carries a sidecar or a literal `$K`. It stays unbound until the first state message, then patches the placeholders with the loop variables (coerced to the placeholder's JSON type), instantiates and opens the real executor once, and delegates every later call to it. A source is never wrapped, since no state message reaches it. The worker registers each state message on the executor (`registerState` / `loopState`) before `processState` runs, and a reconfiguration or a checkpoint restore hands the replacement executor the state its predecessor consumed. An operator whose executor runs generated code (the UDFs and the Python-generating descriptors) cannot bind a reference yet: the compiler reports it instead of running with the placeholders. Closes apache#8635
1 parent 454089e commit ccf1079

36 files changed

Lines changed: 2349 additions & 18 deletions

File tree

‎amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessor.scala‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,11 @@ class DataProcessor(
128128
loopStartId: String
129129
): Unit = {
130130
try {
131+
// Register the state message before the callback runs, so a user executor's
132+
// processState (and every call after it) can consult `loopState`. A late-bound
133+
// executor does not rely on this order: it binds its `$K` placeholders inside
134+
// processState and registers the state there itself.
135+
executor.registerState(state)
131136
val outputState = executor.processState(state, port)
132137
if (outputState.isDefined) {
133138
// Carry the incoming loop envelope through unchanged: loop operators

‎amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/managers/SerializationManager.scala‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,17 @@
2020
package org.apache.texera.amber.engine.architecture.worker.managers
2121

2222
import org.apache.texera.amber.core.executor._
23+
import org.apache.texera.amber.core.state.State
2324
import org.apache.texera.amber.core.tuple.TupleLike
2425
import org.apache.texera.amber.core.virtualidentity.ActorVirtualIdentity
2526
import org.apache.texera.amber.core.workflow.PortIdentity
2627
import org.apache.texera.amber.engine.architecture.rpc.controlcommands.InitializeExecutorRequest
27-
import org.apache.texera.amber.engine.common.{AmberLogging, CheckpointState, CheckpointSupport}
28+
import org.apache.texera.amber.engine.common.{
29+
AmberLogging,
30+
CheckpointState,
31+
CheckpointSupport,
32+
SerializedState
33+
}
2834
import org.apache.texera.amber.util.VirtualIdentityUtils
2935

3036
class SerializationManager(val actorId: ActorVirtualIdentity) extends AmberLogging {
@@ -53,6 +59,11 @@ class SerializationManager(val actorId: ActorVirtualIdentity) extends AmberLoggi
5359
case OpExecWithCode(code, language) => ExecFactory.newExecFromJavaCode(code)
5460
case _ => throw new UnsupportedOperationException("Unsupported OpExec type")
5561
}
62+
// The state message the checkpointed executor had consumed is not replayed: hand it over,
63+
// so a late-bound executor binds its `$K` placeholders again.
64+
if (chkpt.has(SerializedState.LOOP_STATE_KEY)) {
65+
LateBoundExecutor.adoptLoopState(executor, chkpt.load[State](SerializedState.LOOP_STATE_KEY))
66+
}
5667

5768
val iter = executor match {
5869
case support: CheckpointSupport =>

‎amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/PrepareCheckpointHandler.scala‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,11 @@ trait PrepareCheckpointHandler {
5959
val chkpt = new CheckpointState()
6060
// 1. serialize DP state
6161
chkpt.save(SerializedState.DP_STATE_KEY, this.dp)
62+
// The executor is rebuilt on restore, and the state message it already consumed is not
63+
// replayed: keep its loop state, so the rebuilt executor can take it over.
64+
Option(dp.executor)
65+
.flatMap(_.loopState)
66+
.foreach(state => chkpt.save(SerializedState.LOOP_STATE_KEY, state))
6267
// checkpoint itself should not be serialized, thus we register it after serialization
6368
dp.ecmManager.checkpoints(checkpointId) = chkpt
6469
logger.info("Serialized DP state")

‎amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/UpdateExecutorHandler.scala‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
package org.apache.texera.amber.engine.architecture.worker.promisehandlers
2121

2222
import com.twitter.util.Future
23+
import org.apache.texera.amber.core.executor.LateBoundExecutor
2324
import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{
2425
AsyncRPCContext,
2526
UpdateExecutorRequest
@@ -44,11 +45,15 @@ trait UpdateExecutorHandler {
4445
)
4546
// Close the existing executor (if any) before replacing it to avoid resource leaks.
4647
val oldExecutor = dp.executor
48+
// The iteration's state message went by before this swap, and no second one follows: hand
49+
// it to the replacement, or a late-bound one would never bind its `$K` placeholders.
50+
val loopState = Option(oldExecutor).flatMap(_.loopState)
4751
if (oldExecutor != null) {
4852
oldExecutor.close()
4953
}
5054
setupExecutor(request.newExecInitInfo, workerIdx, cachedTotalWorkerCount)
5155
dp.executor.open()
56+
loopState.foreach(LateBoundExecutor.adoptLoopState(dp.executor, _))
5257
EmptyReturn()
5358
}
5459

‎amber/src/main/scala/org/apache/texera/amber/engine/common/SerializedState.scala‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,9 @@ object SerializedState {
2828
val IN_FLIGHT_MSG_KEY = "Amber_Inflight_Messages"
2929
val DP_QUEUED_MSG_KEY = "Amber_DP_Queued_Messages"
3030
val OUTPUT_MSG_KEY = "Amber_Output_Messages"
31+
// The loop state the executor had consumed: the executor is rebuilt on restore, and the
32+
// iteration's state message is not replayed, so a late-bound executor binds to this one.
33+
val LOOP_STATE_KEY = "Amber_Loop_State"
3134

3235
def fromObject[T <: AnyRef](obj: T, serialization: Serialization): SerializedState = {
3336
val bytes = serialization.serialize(obj).get

‎amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import io.dropwizard.configuration.{EnvironmentVariableSubstitutor, Substituting
2626
import io.dropwizard.setup.{Bootstrap, Environment}
2727
import io.dropwizard.websockets.WebsocketBundle
2828
import org.apache.texera.common.config.{ApplicationConfig, StorageConfig}
29+
import org.apache.texera.amber.core.state.StateReferenceModule
2930
import org.apache.texera.amber.core.storage.DocumentFactory
3031
import org.apache.texera.amber.core.virtualidentity.ExecutionIdentity
3132
import org.apache.texera.amber.core.workflow.{PhysicalPlan, WorkflowContext}
@@ -136,6 +137,8 @@ class ComputingUnitMaster extends io.dropwizard.Application[Configuration] with
136137
)
137138
// register scala module to dropwizard default object mapper
138139
bootstrap.getObjectMapper.registerModule(DefaultScalaModule)
140+
// rewrite `$K` loop-variable references in the operators of REST request bodies
141+
bootstrap.getObjectMapper.registerModule(new StateReferenceModule())
139142
}
140143

141144
override def run(configuration: Configuration, environment: Environment): Unit = {

‎amber/src/main/scala/org/apache/texera/web/TexeraWebApplication.scala‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import io.dropwizard.setup.{Bootstrap, Environment}
2828
import io.dropwizard.websockets.WebsocketBundle
2929
import org.apache.texera.common.config.StorageConfig
3030
import org.apache.texera.amber.engine.common.Utils
31+
import org.apache.texera.amber.core.state.StateReferenceModule
3132
import org.apache.texera.amber.util.ObjectMapperUtils
3233
import org.apache.texera.auth.SessionUser
3334
import org.apache.texera.dao.SqlServer
@@ -98,6 +99,8 @@ class TexeraWebApplication
9899
bootstrap.addBundle(new WebsocketBundle(classOf[CollaborationResource]))
99100
// register scala module to dropwizard default object mapper
100101
bootstrap.getObjectMapper.registerModule(DefaultScalaModule)
102+
// rewrite `$K` loop-variable references in the operators of REST request bodies
103+
bootstrap.getObjectMapper.registerModule(new StateReferenceModule())
101104
}
102105

103106
override def run(configuration: TexeraWebConfiguration, environment: Environment): Unit = {

‎amber/src/test/integration/org/apache/texera/amber/engine/e2e/LoopIntegrationSpec.scala‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,11 @@ import org.apache.texera.amber.engine.e2e.TestUtils.{
4141
workflowContext
4242
}
4343
import org.apache.texera.amber.operator.LogicalOp
44+
import org.apache.texera.amber.operator.filter.{
45+
ComparisonType,
46+
FilterPredicate,
47+
SpecializedFilterOpDesc
48+
}
4449
import org.apache.texera.amber.operator.limit.LimitOpDesc
4550
import org.apache.texera.amber.operator.loop.{LoopEndOpDesc, LoopStartOpDesc}
4651
import org.apache.texera.amber.operator.sleep.SleepOpDesc
@@ -188,6 +193,13 @@ class LoopIntegrationSpec
188193
op
189194
}
190195

196+
/** `attribute = value`; a `$name` value refers to a loop variable (bound at the worker). */
197+
private def filterEquals(attribute: String, value: String): SpecializedFilterOpDesc = {
198+
val op = new SpecializedFilterOpDesc()
199+
op.predicates = List(new FilterPredicate(attribute, ComparisonType.EQUAL_TO, value))
200+
op
201+
}
202+
191203
// NOTE: a JavaUDF (runtime-compiled Java) case cannot run in this suite.
192204
// The integration tests run forkless under sbt (fork := false), where
193205
// `javax.tools` sees only the sbt launcher on `java.class.path` -- the
@@ -317,6 +329,33 @@ class LoopIntegrationSpec
317329
)
318330
}
319331

332+
it should "bind a $i loop-variable reference in a JVM operator's property on every iteration" in {
333+
// TextInput("0","1","2") -> LoopStart(i = 0 / table.iloc[i]) -> Filter(line = $i) -> LoopEnd.
334+
//
335+
// The Filter's predicate value is the LITERAL "$i" set directly on the descriptor (no
336+
// parse-time rewrite ran), so this exercises the worker's literal detection: ExecFactory
337+
// hands the worker a LateBoundExecutor, which binds "$i" to the iteration's `i` when the
338+
// LoopStart's state message arrives -- the region's input reader replays states before
339+
// tuples -- and only then instantiates the real Filter. Iteration i emits the single row
340+
// "i" and the Filter compares it against i, so every iteration's row passes and the
341+
// LoopEnd accumulates 3 rows. A constant "0" instead of "$i" would pass only the first
342+
// iteration's row and give 1; a reference that never rebinds to the new i would too.
343+
val src = textInput("0\n1\n2")
344+
val start = loopStart("i = 0", "table.iloc[i]")
345+
val mid = filterEquals("line", "$i")
346+
val end = loopEnd("i += 1", "i < len(table)")
347+
val materialized = runAndGetMaterializedRowCounts(
348+
List(src, start, mid, end),
349+
List(link(src, start), link(start, mid), link(mid, end))
350+
)
351+
val endRows = materialized.getOrElse(end.operatorIdentifier, -1L)
352+
assert(
353+
endRows == 3,
354+
s"Filter(line = $$i) must pass its own iteration's row on all 3 iterations: " +
355+
s"expected 3, got $endRows (all: $materialized)"
356+
)
357+
}
358+
320359
it should "run a nested loop whose inner body chains multiple JVM operators" in {
321360
// TextInput -> OuterStart -> InnerStart -> Limit -> Sleep(0) -> InnerEnd -> OuterEnd.
322361
//

‎amber/src/test/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessorRPCHandlerInitializerSpec.scala‎

Lines changed: 53 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,24 @@
2020
package org.apache.texera.amber.engine.architecture.worker
2121

2222
import org.apache.texera.amber.core.executor.{
23+
LateBoundExecutor,
2324
OpExecInitInfo,
2425
OperatorExecutor,
2526
OpExecWithClassName,
2627
OpExecWithCode
2728
}
29+
import org.apache.texera.amber.core.state.State
2830
import org.apache.texera.amber.core.tuple.{Tuple, TupleLike}
29-
import org.apache.texera.amber.core.virtualidentity.ActorVirtualIdentity
31+
import org.apache.texera.amber.core.virtualidentity.{
32+
ActorVirtualIdentity,
33+
OperatorIdentity,
34+
PhysicalOpIdentity
35+
}
36+
import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{
37+
AsyncRPCContext,
38+
UpdateExecutorRequest
39+
}
40+
import org.apache.texera.amber.engine.common.virtualidentity.util.COORDINATOR
3041
import org.apache.texera.amber.engine.architecture.worker.WorkflowWorker.{
3142
DPInputQueueElement,
3243
MainThreadDelegateMessage
@@ -131,6 +142,47 @@ class DataProcessorRPCHandlerInitializerSpec extends AnyFlatSpec {
131142
assert(failure.getMessage == "Empty executor initialization info")
132143
assert(initializer.dp.executor == null)
133144
}
145+
146+
behavior of "DataProcessorRPCHandlerInitializer.updateExecutor"
147+
148+
private def update(initializer: DataProcessorRPCHandlerInitializer, desc: String): Unit =
149+
initializer.updateExecutor(
150+
UpdateExecutorRequest(
151+
PhysicalOpIdentity(OperatorIdentity("op"), "main"),
152+
OpExecWithClassName(classOf[DescribedExec].getName, desc)
153+
),
154+
AsyncRPCContext(COORDINATOR, workerId)
155+
)
156+
157+
it should "bind a late-bound replacement to the loop state its predecessor already consumed" in {
158+
// Reconfiguring an operator mid-iteration: the iteration's state message went by before
159+
// the swap and no second one follows, so the replacement must take it over or every
160+
// remaining tuple fails on an unbound `$i`.
161+
val initializer = newInitializer()
162+
val desc = """{"name":"$i"}"""
163+
initializer.setupExecutor(OpExecWithClassName(classOf[DescribedExec].getName, desc), 0, 1)
164+
val state = State(Map("i" -> "bound"))
165+
initializer.dp.executor.processState(state, 0)
166+
167+
update(initializer, desc)
168+
169+
val replacement = initializer.dp.executor.asInstanceOf[LateBoundExecutor]
170+
assert(replacement.isBound)
171+
assert(replacement.boundExecutor.get.asInstanceOf[DescribedExec].desc == """{"name":"bound"}""")
172+
assert(replacement.loopState.contains(state))
173+
}
174+
175+
it should "carry the loop state over to a replacement that refers to no loop variable" in {
176+
val initializer = newInitializer()
177+
initializer.setupExecutor(OpExecWithClassName(classOf[DescribedExec].getName, "first"), 0, 1)
178+
val state = State(Map("i" -> 1L))
179+
initializer.dp.executor.registerState(state)
180+
181+
update(initializer, "second")
182+
183+
assert(initializer.dp.executor.asInstanceOf[DescribedExec].desc == "second")
184+
assert(initializer.dp.executor.loopState.contains(state))
185+
}
134186
}
135187

136188
private object DataProcessorRPCHandlerInitializerSpec {

‎amber/src/test/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessorSpec.scala‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -260,6 +260,7 @@ class DataProcessorSpec extends AnyFlatSpec with MockFactory with Matchers with
260260
}
261261
.anyNumberOfTimes()
262262
val inputState = State(Map("field1" -> 1))
263+
(executor.registerState _).expects(inputState).once()
263264
(
264265
(
265266
state: State,
@@ -305,6 +306,7 @@ class DataProcessorSpec extends AnyFlatSpec with MockFactory with Matchers with
305306
}
306307
.anyNumberOfTimes()
307308
val inputState = State(Map("i" -> 1))
309+
(executor.registerState _).expects(inputState).once()
308310
(
309311
(
310312
state: State,
@@ -345,6 +347,7 @@ class DataProcessorSpec extends AnyFlatSpec with MockFactory with Matchers with
345347
}
346348
.anyNumberOfTimes()
347349
val inputState = State(Map("field1" -> 2))
350+
(executor.registerState _).expects(inputState).once()
348351
(
349352
(
350353
state: State,
@@ -377,6 +380,7 @@ class DataProcessorSpec extends AnyFlatSpec with MockFactory with Matchers with
377380
dp.stateManager.transitTo(READY)
378381
(outputHandler.apply _).expects(*).anyNumberOfTimes()
379382
val inputState = State(Map("field1" -> 3))
383+
(executor.registerState _).expects(inputState).once()
380384
(
381385
(
382386
state: State,
@@ -400,6 +404,53 @@ class DataProcessorSpec extends AnyFlatSpec with MockFactory with Matchers with
400404
dp.pauseManager.isPaused shouldBe true
401405
}
402406

407+
"data processor" should "register the loop state on the executor before invoking processState" in {
408+
// A real executor, not a mock: it records what `loopState` holds at the moment its
409+
// callback runs, which pins the ORDER (register first, then process) that lets a user
410+
// executor read the iteration's variables inside processState.
411+
class LoopStateSpy extends OperatorExecutor {
412+
val seen = scala.collection.mutable.ArrayBuffer[(Option[State], Int)]()
413+
override def processState(state: State, port: Int): Option[State] = {
414+
seen += ((loopState, port))
415+
super.processState(state, port)
416+
}
417+
override def processTuple(tuple: Tuple, port: Int): Iterator[TupleLike] =
418+
Iterator.single(tuple)
419+
}
420+
val spy = new LoopStateSpy
421+
val dp = mkDataProcessor
422+
dp.executor = spy
423+
dp.stateManager.transitTo(READY)
424+
val emitted = scala.collection.mutable.ArrayBuffer[WorkflowFIFOMessage]()
425+
(outputHandler.apply _)
426+
.expects(*)
427+
.onCall { (m: Either[MainThreadDelegateMessage, WorkflowFIFOMessage]) =>
428+
m.foreach(emitted += _); ()
429+
}
430+
.anyNumberOfTimes()
431+
val inputState = State(Map("i" -> 2L))
432+
dp.inputManager.addPort(inputPortId, schema, List.empty, List.empty)
433+
dp.inputGateway
434+
.getChannel(ChannelIdentity(senderWorkerId, testWorkerId, isControl = false))
435+
.setPortId(inputPortId)
436+
dp.outputManager.addPort(outputPortId, schema, None)
437+
dp.outputManager.addPartitionerWithPartitioning(
438+
PhysicalLink(testOpId, outputPortId, upstreamOpId, inputPortId),
439+
OneToOnePartitioning(1, Seq(ChannelIdentity(testWorkerId, senderWorkerId, isControl = false)))
440+
)
441+
assert(spy.loopState.isEmpty)
442+
dp.processDataPayload(
443+
ChannelIdentity(senderWorkerId, testWorkerId, isControl = false),
444+
StateFrame(inputState)
445+
)
446+
// The callback saw the state already registered, on its port ...
447+
assert(spy.seen.toList == List((Some(inputState), 0)))
448+
// ... it stays registered afterwards, and the default callback still forwards it.
449+
assert(spy.loopState.contains(inputState))
450+
assert(emitted.map(_.payload).contains(StateFrame(inputState)))
451+
dp.pauseManager.isPaused shouldBe false
452+
}
453+
403454
"data processor" should "handle an exception thrown while processing an input tuple" in {
404455
val dp = mkDataProcessor
405456
dp.executor = executor

0 commit comments

Comments
 (0)