Skip to content

Commit f126146

Browse files
committed
Merge remote-tracking branch 'origin/main' into feat/loop-variable-refs-frontend
2 parents 0b81d8c + 6a29c59 commit f126146

74 files changed

Lines changed: 3229 additions & 1071 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎amber/dev-requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,4 +42,4 @@ textual==8.2.8
4242

4343
# Reads bin/k8s/values.yaml in bin/k8s/tests/test_helm_values.sh. That check fails
4444
# rather than skipping when this is missing, so the suite cannot go green by accident.
45-
PyYAML==6.0.2
45+
PyYAML==6.0.3

‎amber/src/main/python/core/architecture/packaging/output_manager.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -124,11 +124,10 @@ def add_output_port(
124124
if port_id.internal is None:
125125
port_id.internal = False
126126

127-
if storage_uri_base is not None:
128-
self.set_up_port_storage_writer(port_id, storage_uri_base)
129-
130127
# each port can only be added and initialized once.
131128
if port_id not in self._ports:
129+
if storage_uri_base is not None:
130+
self.set_up_port_storage_writer(port_id, storage_uri_base)
132131
self._ports[port_id] = WorkerPort(schema)
133132

134133
def set_up_port_storage_writer(self, port_id: PortIdentity, storage_uri_base: str):

‎amber/src/main/python/core/models/state.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,9 @@ def _to_json_value(value: Any) -> Any:
9393
_PAYLOAD_MARKER: base64.b64encode(value).decode("ascii"),
9494
}
9595
if isinstance(value, dict):
96-
return {str(key): _to_json_value(inner) for key, inner in value.items()}
96+
if any(not isinstance(key, str) for key in value):
97+
raise TypeError("State dictionary keys must be strings")
98+
return {key: _to_json_value(inner) for key, inner in value.items()}
9799
if isinstance(value, (list, tuple)):
98100
return [_to_json_value(inner) for inner in value]
99101
raise TypeError(

‎amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveResource.scala‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import com.fasterxml.jackson.module.scala.DefaultScalaModule
2525
import com.typesafe.scalalogging.LazyLogging
2626
import io.dropwizard.auth.Auth
2727
import org.apache.texera.auth.SessionUser
28+
import org.apache.texera.dao.SqlStates
2829
import org.jooq.exception.DataAccessException
2930

3031
import javax.ws.rs._
@@ -107,7 +108,7 @@ class PveResource extends LazyLogging {
107108
if (updated) Response.ok(Map("veid" -> veid).asJava).build()
108109
else Response.status(Response.Status.NOT_FOUND).build()
109110
} catch {
110-
case e: DataAccessException if e.sqlState() == "23505" =>
111+
case e: DataAccessException if e.sqlState() == SqlStates.UNIQUE_VIOLATION =>
111112
Response
112113
.status(Response.Status.CONFLICT)
113114
.entity(s"""An environment named "$name" already exists.""")
@@ -145,7 +146,7 @@ class PveResource extends LazyLogging {
145146
val veid = PveManager.savePve(sessionUser.getUid.intValue(), name, packagesJson)
146147
Response.status(Response.Status.CREATED).entity(Map("veid" -> veid).asJava).build()
147148
} catch {
148-
case e: DataAccessException if e.sqlState() == "23505" =>
149+
case e: DataAccessException if e.sqlState() == SqlStates.UNIQUE_VIOLATION =>
149150
Response
150151
.status(Response.Status.CONFLICT)
151152
.entity(s"""An environment named "$name" already exists.""")

‎amber/src/main/scala/org/apache/texera/web/service/ExecutionReconfigurationService.scala‎

Lines changed: 39 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919

2020
package org.apache.texera.web.service
2121

22+
import com.fasterxml.jackson.databind.node.ObjectNode
23+
import org.apache.texera.amber.core.state.StateReferencing.literalReferences
2224
import org.apache.texera.amber.core.storage.RepositoryMountManager
2325
import org.apache.texera.amber.core.virtualidentity.ActorVirtualIdentity
2426
import org.apache.texera.amber.engine.architecture.coordinator.{UpdateExecutorCompleted, Workflow}
@@ -27,6 +29,8 @@ import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{
2729
WorkflowReconfigureRequest
2830
}
2931
import org.apache.texera.amber.engine.common.client.AmberClient
32+
import org.apache.texera.amber.operator.LogicalOp
33+
import org.apache.texera.amber.util.JSONUtils.objectMapper
3034
import org.apache.texera.web.SubscriptionManager
3135
import org.apache.texera.web.model.websocket.event.TexeraWebSocketEvent
3236
import org.apache.texera.web.model.websocket.request.ModifyLogicRequest
@@ -60,24 +64,47 @@ class ExecutionReconfigurationService(
6064
val newOp = modifyLogicRequest.operator
6165
val opId = newOp.operatorIdentifier
6266
val currentOp = workflow.logicalPlan.getOperator(opId)
63-
val reconfiguredPhysicalOp =
64-
currentOp.runtimeReconfiguration(
65-
workflow.context.workflowId,
66-
workflow.context.executionId,
67-
currentOp,
68-
newOp
67+
if (refersToLoopVariables(currentOp) || refersToLoopVariables(newOp)) {
68+
ModifyLogicResponse(
69+
opId.id,
70+
isValid = false,
71+
s"${currentOp.operatorInfo.userFriendlyName} refers to loop variables, so it can't be " +
72+
"changed while the workflow runs."
6973
)
70-
reconfiguredPhysicalOp match {
71-
case Failure(exception) => ModifyLogicResponse(opId.id, isValid = false, exception.getMessage)
72-
case Success(op) => {
73-
stateStore.reconfigurationStore.updateState(old =>
74-
old.copy(unscheduledReconfigurations = old.unscheduledReconfigurations :+ op)
74+
} else {
75+
val reconfiguredPhysicalOp =
76+
currentOp.runtimeReconfiguration(
77+
workflow.context.workflowId,
78+
workflow.context.executionId,
79+
currentOp,
80+
newOp
7581
)
76-
ModifyLogicResponse(opId.id, isValid = true, "")
82+
reconfiguredPhysicalOp match {
83+
case Failure(exception) =>
84+
ModifyLogicResponse(opId.id, isValid = false, exception.getMessage)
85+
case Success(op) => {
86+
stateStore.reconfigurationStore.updateState(old =>
87+
old.copy(unscheduledReconfigurations = old.unscheduledReconfigurations :+ op)
88+
)
89+
ModifyLogicResponse(opId.id, isValid = true, "")
90+
}
7791
}
7892
}
7993
}
8094

95+
/**
96+
* Whether `op`, in the deployed operator's place, refers to loop variables: its sidecar names
97+
* one (the compiler's final one on the deployed operator, the parse's typed ones on an edit),
98+
* or it sits inside a loop block and holds a whole-string `$name`. The rebuilt executor would
99+
* parse the placeholders after the loop state that writes them into its setting has arrived
100+
* (`OperatorExecutor.registerState`), so nothing would ever set them.
101+
*/
102+
private def refersToLoopVariables(op: LogicalOp): Boolean =
103+
op.stateReferences.nonEmpty || (
104+
workflow.logicalPlan.operatorsInsideLoopBlocks.contains(op.operatorIdentifier) &&
105+
literalReferences(objectMapper.valueToTree[ObjectNode](op)).nonEmpty
106+
)
107+
81108
// actually performs all reconfiguration requests the user made during pause
82109
// sends ModifyLogic messages to operators and workers,
83110
// see the Fries reconfiguration paper for the algorithm.

‎amber/src/test/python/core/architecture/packaging/test_output_manager.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -396,6 +396,17 @@ def test_port_can_only_be_added_once(self, output_manager):
396396
assert output_manager.get_port_ids() == [port_id]
397397
assert output_manager.get_port().get_schema() is schema_first
398398

399+
def test_duplicate_port_does_not_start_replacement_storage_writers(
400+
self, output_manager
401+
):
402+
output_manager.set_up_port_storage_writer = MagicMock()
403+
port_id = PortIdentity(id=0, internal=False)
404+
output_manager.add_output_port(port_id, MagicMock(), "vfs:///first")
405+
output_manager.add_output_port(port_id, MagicMock(), "vfs:///duplicate")
406+
output_manager.set_up_port_storage_writer.assert_called_once_with(
407+
port_id, "vfs:///first"
408+
)
409+
399410
def test_sets_up_storage_writer_only_when_uri_given(self, output_manager):
400411
output_manager.set_up_port_storage_writer = MagicMock()
401412
port_a = PortIdentity(id=0, internal=False)

‎amber/src/test/python/core/models/test_state.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,13 @@ class Custom:
9090
with pytest.raises(TypeError):
9191
State({"bad": Custom()}).to_json()
9292

93+
@pytest.mark.parametrize(
94+
"state", [State({1: "value"}), State({"nested": {1: "value"}})]
95+
)
96+
def test_to_json_rejects_non_string_keys(self, state):
97+
with pytest.raises(TypeError, match="keys must be strings"):
98+
state.to_json()
99+
93100
def test_tuple_round_trip(self):
94101
original = State({"i": 3, "label": "outer", "blob": b"\x01\x02"})
95102
decoded = State.from_tuple(original.to_tuple())

‎amber/src/test/scala/org/apache/texera/web/service/ExecutionReconfigurationServiceSpec.scala‎

Lines changed: 101 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ import org.apache.texera.amber.core.workflow.{
3737
OutputPort,
3838
PhysicalOp,
3939
PhysicalPlan,
40+
PortIdentity,
4041
WorkflowContext
4142
}
4243
import org.apache.texera.amber.engine.architecture.coordinator.{
@@ -48,11 +49,17 @@ import org.apache.texera.amber.engine.architecture.rpc.controlcommands.WorkflowR
4849
import org.apache.texera.amber.engine.architecture.rpc.controlreturns.EmptyReturn
4950
import org.apache.texera.amber.engine.architecture.rpc.coordinatorservice.CoordinatorServiceFs2Grpc
5051
import org.apache.texera.amber.engine.common.client.AmberClient
52+
import org.apache.texera.amber.operator.filter.{
53+
ComparisonType,
54+
FilterPredicate,
55+
SpecializedFilterOpDesc
56+
}
5157
import org.apache.texera.amber.operator.limit.LimitOpDesc
58+
import org.apache.texera.amber.operator.loop.{LoopEndOpDesc, LoopStartOpDesc}
5259
import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo}
5360
import org.apache.texera.amber.operator.{LogicalOp, StateTransferFunc}
5461
import org.apache.texera.amber.util.JSONUtils.objectMapper
55-
import org.apache.texera.common.compiler.model.LogicalPlan
62+
import org.apache.texera.common.compiler.model.{LogicalLink, LogicalPlan}
5663
import org.apache.texera.web.model.websocket.event.TexeraWebSocketEvent
5764
import org.apache.texera.web.model.websocket.request.ModifyLogicRequest
5865
import org.apache.texera.web.model.websocket.response.{
@@ -470,6 +477,99 @@ class ExecutionReconfigurationServiceSpec
470477
queued.map(_._1.id) shouldBe List(alreadyQueued.id)
471478
}
472479

480+
// ---------------------------------------------------------------------------
481+
// An operator that refers to loop variables
482+
// ---------------------------------------------------------------------------
483+
484+
private def loopStartOp(): LoopStartOpDesc = {
485+
val op = new LoopStartOpDesc
486+
op.initialization = "i = 0"
487+
op.output = "table.iloc[i]"
488+
op
489+
}
490+
491+
private def loopEndOp(): LoopEndOpDesc = {
492+
val op = new LoopEndOpDesc
493+
op.update = "i += 1"
494+
op.condition = "i < len(table)"
495+
op
496+
}
497+
498+
/** A service over a workflow whose plan is LoopStart -> `deployedOp` -> LoopEnd. */
499+
private def insideLoopBlock(deployedOp: LogicalOp): (ExecutionStateStore, WorkflowOnlyService) = {
500+
val start = loopStartOp()
501+
val end = loopEndOp()
502+
def link(from: LogicalOp, to: LogicalOp): LogicalLink =
503+
LogicalLink(from.operatorIdentifier, PortIdentity(), to.operatorIdentifier, PortIdentity())
504+
val workflow = Workflow(
505+
new WorkflowContext(workflowId = testWorkflowId, executionId = testExecutionId),
506+
LogicalPlan(
507+
List(start, deployedOp, end),
508+
List(link(start, deployedOp), link(deployedOp, end))
509+
),
510+
PhysicalPlan(Set.empty, Set.empty)
511+
)
512+
val stateStore = new ExecutionStateStore
513+
(stateStore, new WorkflowOnlyService(stateStore, workflow))
514+
}
515+
516+
private def filterOp(opId: String, value: String): SpecializedFilterOpDesc = {
517+
val op = new SpecializedFilterOpDesc
518+
op.setOperatorId(opId)
519+
op.predicates = List(new FilterPredicate("line", ComparisonType.EQUAL_TO, value))
520+
op
521+
}
522+
523+
private def refused(name: String): String =
524+
s"$name refers to loop variables, so it can't be changed while the workflow runs."
525+
526+
"modifyOperatorLogic" should
527+
"refuse to modify an operator whose deployed setting refers to a loop variable" in {
528+
// The references the compiler left on the deployed operator, as it does inside a loop block.
529+
val deployedOp = mkLimitOp("limit-a", 0)
530+
deployedOp.stateReferences = Map("/limit" -> "n")
531+
val (stateStore, service) = insideLoopBlock(deployedOp)
532+
533+
service.modifyOperatorLogic(ModifyLogicRequest(mkLimitOp("limit-a", 7))) shouldBe
534+
ModifyLogicResponse("limit-a", isValid = false, refused("Limit"))
535+
stateStore.reconfigurationStore.getState.unscheduledReconfigurations shouldBe empty
536+
}
537+
538+
it should "refuse an edit that makes an operator inside a loop block refer to a loop variable" in {
539+
val (typedStore, typed) = insideLoopBlock(mkLimitOp("limit-a", 1))
540+
// As the frontend sends it: "$n" in the Int property.
541+
val typedEdit = objectMapper.readValue(
542+
"""{"operatorID":"limit-a","limit":"$n","operatorType":"Limit"}""",
543+
classOf[LogicalOp]
544+
)
545+
typed.modifyOperatorLogic(ModifyLogicRequest(typedEdit)) shouldBe
546+
ModifyLogicResponse("limit-a", isValid = false, refused("Limit"))
547+
typedStore.reconfigurationStore.getState.unscheduledReconfigurations shouldBe empty
548+
549+
// A whole-string "$x" refers to a loop variable inside a loop block; the compiler never saw it.
550+
val (literalStore, literal) = insideLoopBlock(filterOp("filter-a", "Asia"))
551+
literal.modifyOperatorLogic(ModifyLogicRequest(filterOp("filter-a", "$x"))) shouldBe
552+
ModifyLogicResponse("filter-a", isValid = false, refused("Filter"))
553+
literalStore.reconfigurationStore.getState.unscheduledReconfigurations shouldBe empty
554+
}
555+
556+
it should "still modify an operator inside a loop block that refers to no loop variable, and a '$x' literal outside every block" in {
557+
val (insideStore, inside) = insideLoopBlock(mkLimitOp("limit-a", 1))
558+
inside.modifyOperatorLogic(ModifyLogicRequest(mkLimitOp("limit-a", 7))) shouldBe
559+
ModifyLogicResponse("limit-a", isValid = true, "")
560+
insideStore.reconfigurationStore.getState.unscheduledReconfigurations.map {
561+
case (op, _) => limitOf(op)
562+
} shouldBe List(7)
563+
564+
// Outside every block "$x" is the literal it is on main.
565+
val outsideStore = new ExecutionStateStore
566+
val outside =
567+
new WorkflowOnlyService(outsideStore, mkWorkflow(List(filterOp("filter-a", "Asia"))))
568+
outside.modifyOperatorLogic(ModifyLogicRequest(filterOp("filter-a", "$x"))) shouldBe
569+
ModifyLogicResponse("filter-a", isValid = true, "")
570+
outsideStore.reconfigurationStore.getState.unscheduledReconfigurations should have size 1
571+
}
572+
473573
"performReconfigurationOnResume" should
474574
"return without dispatching when no reconfigurations are pending" in {
475575
val stateStore = new ExecutionStateStore()

‎common/workflow-compiler/src/main/scala/org/apache/texera/common/compiler/WorkflowCompiler.scala‎

Lines changed: 30 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,10 @@ import org.apache.texera.common.compiler.WorkflowCompiler.{
3030
}
3131
import org.apache.texera.common.compiler.model.{LogicalPlan, LogicalPlanPojo}
3232
import org.apache.texera.amber.core.executor.OpExecWithCode
33-
import org.apache.texera.amber.core.state.StateReferencing.literalReferences
33+
import org.apache.texera.amber.core.state.StateReferencing.{
34+
literalReferences,
35+
noLoopVariableReferences
36+
}
3437
import org.apache.texera.amber.core.tuple.Schema
3538
import org.apache.texera.amber.core.virtualidentity.OperatorIdentity
3639
import org.apache.texera.amber.core.workflow.{
@@ -82,16 +85,19 @@ object WorkflowCompiler {
8285
/**
8386
* Makes `logicalOp`'s `stateReferences` sidecar final, before its physical plan serializes it
8487
* into the executor's descString. Inside a loop block it adds every whole-string `$name` value
85-
* of the operator's JSON (`StateReferencing.literalReferences`). Outside every block a string
86-
* such as "$AAPL" stays a literal, and a typed placeholder (the user wrote "$n" for an Int),
87-
* which nothing would bind, is an error; it stays in the sidecar, so a recompile reports it too.
88+
* of the operator's JSON (`StateReferencing.literalReferences`), and an entry in a property that
89+
* cannot hold a loop variable is an error (`referencesInNoLoopVariableProperties`). Outside
90+
* every block a string such as "$AAPL" stays a literal, and a typed placeholder (the user wrote
91+
* "$n" for an Int), which nothing would bind, is an error; it stays in the sidecar, so a
92+
* recompile reports it too.
8893
*/
8994
def normalizeStateReferences(logicalOp: LogicalOp, plan: LogicalPlan): Option[Throwable] =
9095
if (plan.operatorsInsideLoopBlocks.contains(logicalOp.operatorIdentifier)) {
9196
logicalOp.stateReferences =
9297
literalReferences(objectMapper.valueToTree[ObjectNode](logicalOp)) ++
9398
logicalOp.stateReferences
94-
inputsAheadOfLoopState(logicalOp, plan)
99+
referencesInNoLoopVariableProperties(logicalOp)
100+
.orElse(inputsAheadOfLoopState(logicalOp, plan))
95101
} else {
96102
Option.when(logicalOp.stateReferences.nonEmpty) {
97103
new IllegalArgumentException(
@@ -101,6 +107,25 @@ object WorkflowCompiler {
101107
}
102108
}
103109

110+
/**
111+
* The loop state writes a loop variable into the operator's setting only once the workflow runs,
112+
* but a property marked `NoLoopVariable` -- the name or type of an output column, or what an
113+
* input is partitioned on -- has been built into the plan by then: from the placeholder. A
114+
* Projection alias "$a" would output a column named "$a", null in every row; a SortPartitions
115+
* domain would stay 0 to 0. A reference there is an error instead.
116+
*/
117+
private def referencesInNoLoopVariableProperties(logicalOp: LogicalOp): Option[Throwable] = {
118+
val references = noLoopVariableReferences(logicalOp)
119+
Option.when(references.nonEmpty) {
120+
val those = if (references.size == 1) "that property" else "those properties"
121+
new IllegalArgumentException(
122+
s"${logicalOp.operatorInfo.userFriendlyName} cannot refer to loop variables " +
123+
s"(${formatReferences(references)}): its output schema or partitioning is built from " +
124+
s"$those when the workflow is compiled, before the loop runs"
125+
)
126+
}
127+
}
128+
104129
/**
105130
* The first tuple fails on a loop variable its state has not written yet, and the states of
106131
* the blocks around the operator come ahead of the tuples only on links from inside all of

0 commit comments

Comments
 (0)