Skip to content

Commit 55b495b

Browse files
committed
feat(workflow-compiling-service): report each operator's enclosing loop starts
1 parent 038e1c3 commit 55b495b

2 files changed

Lines changed: 140 additions & 4 deletions

File tree

‎workflow-compiling-service/src/main/scala/org/apache/texera/service/resource/WorkflowCompilationResource.scala‎

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -44,14 +44,27 @@ import org.apache.texera.amber.util.serde.PortIdentityKeySerializer
4444
)
4545
)
4646
trait WorkflowCompilationResponse
47+
48+
/**
49+
* @param operatorLoopStarts for each operator inside a loop block, the ids of the Loop Starts of
50+
* the blocks it is inside, sorted; an operator outside every block is
51+
* absent. The property panel offers `$name` loop-variable references
52+
* from it.
53+
*/
4754
case class WorkflowCompilationSuccess(
4855
physicalPlan: PhysicalPlan,
49-
operatorOutputSchemas: Map[String, Map[String, Option[List[Attribute]]]]
56+
operatorOutputSchemas: Map[String, Map[String, Option[List[Attribute]]]],
57+
operatorLoopStarts: Map[String, List[String]]
5058
) extends WorkflowCompilationResponse
5159

60+
/**
61+
* @param operatorLoopStarts as on [[WorkflowCompilationSuccess]]: it comes from the logical plan,
62+
* so a workflow that does not compile yet still reports it.
63+
*/
5264
case class WorkflowCompilationFailure(
5365
operatorErrors: Map[String, WorkflowFatalError],
54-
operatorOutputSchemas: Map[String, Map[String, Option[List[Attribute]]]]
66+
operatorOutputSchemas: Map[String, Map[String, Option[List[Attribute]]]],
67+
operatorLoopStarts: Map[String, List[String]]
5568
) extends WorkflowCompilationResponse
5669

5770
@Consumes(Array(MediaType.APPLICATION_JSON))
@@ -86,11 +99,19 @@ class WorkflowCompilationResource extends LazyLogging {
8699
(opId, portIdAndAttributes)
87100
}
88101

102+
// The Loop Starts around each operator inside a loop block, from the logical plan rather than
103+
// the physical one, which a failed compile does not have.
104+
val operatorLoopStarts = compilationResult.logicalPlan.enclosingLoopStarts.map {
105+
case (operatorIdentity, loopStarts) =>
106+
(operatorIdentity.id, loopStarts.map(_.id).toList.sorted)
107+
}
108+
89109
// Handle success case: No errors in the compilation result
90110
if (compilationResult.operatorIdToError.isEmpty && compilationResult.physicalPlan.nonEmpty) {
91111
WorkflowCompilationSuccess(
92112
physicalPlan = compilationResult.physicalPlan.get,
93-
operatorOutputSchemas = operatorOutputSchemas
113+
operatorOutputSchemas = operatorOutputSchemas,
114+
operatorLoopStarts = operatorLoopStarts
94115
)
95116
}
96117
// Handle failure case: Errors found during compilation
@@ -99,7 +120,8 @@ class WorkflowCompilationResource extends LazyLogging {
99120
operatorErrors = compilationResult.operatorIdToError.map {
100121
case (operatorIdentity, error) => (operatorIdentity.id, error)
101122
},
102-
operatorOutputSchemas = operatorOutputSchemas
123+
operatorOutputSchemas = operatorOutputSchemas,
124+
operatorLoopStarts = operatorLoopStarts
103125
)
104126
}
105127
}

‎workflow-compiling-service/src/test/scala/org/apache/texera/service/resource/WorkflowCompilationResourceSpec.scala‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,19 @@ import jakarta.ws.rs.client.Entity
2525
import jakarta.ws.rs.core.{MediaType, Response}
2626
import org.apache.texera.common.compiler.model.{LogicalLink, LogicalPlanPojo}
2727
import org.apache.texera.amber.core.workflow.PortIdentity
28+
import org.apache.texera.amber.operator.LogicalOp
29+
import org.apache.texera.amber.operator.limit.LimitOpDesc
30+
import org.apache.texera.amber.operator.loop.{LoopEndOpDesc, LoopStartOpDesc}
2831
import org.apache.texera.amber.operator.projection.{AttributeUnit, ProjectionOpDesc}
2932
import org.apache.texera.amber.operator.source.scan.csv.CSVScanSourceOpDesc
33+
import org.apache.texera.amber.operator.source.scan.text.TextInputSourceOpDesc
3034
import org.apache.texera.amber.util.JSONUtils.objectMapper
3135
import org.assertj.core.api.Assertions.assertThat
3236
import org.scalatest.BeforeAndAfterAll
3337
import org.scalatest.flatspec.AnyFlatSpec
3438

39+
import scala.jdk.CollectionConverters.{IterableHasAsScala, SetHasAsScala}
40+
3541
/**
3642
* Resource-layer tests for `/compile`. Owns only what the REST envelope
3743
* adds on top of compilation itself: HTTP status, the
@@ -172,4 +178,112 @@ class WorkflowCompilationResourceSpec extends AnyFlatSpec with BeforeAndAfterAll
172178
assertThat(rootNode.get("type").asText()).isEqualTo("failure")
173179
assertThat(rootNode.has("operatorErrors")).isTrue
174180
}
181+
182+
// -------------------- operatorLoopStarts --------------------
183+
184+
private def textInputOp(): TextInputSourceOpDesc = {
185+
val op = new TextInputSourceOpDesc()
186+
op.textInput = "0\n1"
187+
op
188+
}
189+
190+
private def loopStartOp(): LoopStartOpDesc = {
191+
val op = new LoopStartOpDesc()
192+
op.initialization = "i = 0"
193+
op.output = "table.iloc[i]"
194+
op
195+
}
196+
197+
private def loopEndOp(): LoopEndOpDesc = {
198+
val op = new LoopEndOpDesc()
199+
op.update = "i += 1"
200+
op.condition = "i < len(table)"
201+
op
202+
}
203+
204+
private def limitOp(): LimitOpDesc = {
205+
val op = new LimitOpDesc()
206+
op.limit = 1
207+
op
208+
}
209+
210+
private def chainedPlan(operators: LogicalOp*): LogicalPlanPojo =
211+
LogicalPlanPojo(
212+
operators = operators.toList,
213+
links = operators
214+
.sliding(2)
215+
.map(pair =>
216+
LogicalLink(
217+
pair.head.operatorIdentifier,
218+
PortIdentity(0),
219+
pair.last.operatorIdentifier,
220+
PortIdentity(0)
221+
)
222+
)
223+
.toList,
224+
opsToViewResult = List.empty,
225+
opsToReuseResult = List.empty
226+
)
227+
228+
/** The response's `type` and its `operatorLoopStarts`, read from the raw JSON on the wire. */
229+
private def loopStartsOf(response: Response): (String, Map[String, List[String]]) = {
230+
assertThat(response.getStatus).isEqualTo(200)
231+
val body = objectMapper.readTree(response.readEntity(classOf[String]))
232+
assert(body.has("operatorLoopStarts"), s"no operatorLoopStarts in $body")
233+
val loopStarts = body
234+
.get("operatorLoopStarts")
235+
.properties()
236+
.asScala
237+
.map(entry => entry.getKey -> entry.getValue.asScala.map(_.asText()).toList)
238+
.toMap
239+
(body.get("type").asText(), loopStarts)
240+
}
241+
242+
it should "report the loop starts around each operator inside a loop block, and none for one outside" in {
243+
// text -> outer -> inner -> body -> innerEnd -> outerEnd -> after
244+
val text = textInputOp()
245+
val outer = loopStartOp()
246+
val inner = loopStartOp()
247+
val body = limitOp()
248+
val innerEnd = loopEndOp()
249+
val outerEnd = loopEndOp()
250+
val after = limitOp()
251+
252+
val (responseType, loopStarts) =
253+
loopStartsOf(postCompile(chainedPlan(text, outer, inner, body, innerEnd, outerEnd, after)))
254+
255+
assert(responseType == "success")
256+
val (outerId, innerId) = (outer.operatorIdentifier.id, inner.operatorIdentifier.id)
257+
// Sorted, so the JSON is the same on every compile of the same workflow.
258+
assert(
259+
loopStarts == Map(
260+
body.operatorIdentifier.id -> List(outerId, innerId).sorted,
261+
inner.operatorIdentifier.id -> List(outerId),
262+
innerEnd.operatorIdentifier.id -> List(outerId)
263+
)
264+
)
265+
}
266+
267+
it should "report the loop starts around each operator inside a loop block when the compile fails" in {
268+
// The editor compiles half-built workflows: a Projection on a column the input does not have
269+
// fails the compile, and the panel still needs to know it is inside the block.
270+
val text = textInputOp()
271+
val start = loopStartOp()
272+
val broken = projectOp(List("no_such_column"))
273+
val end = loopEndOp()
274+
275+
val (responseType, loopStarts) =
276+
loopStartsOf(postCompile(chainedPlan(text, start, broken, end)))
277+
278+
assert(responseType == "failure")
279+
assert(loopStarts == Map(broken.operatorIdentifier.id -> List(start.operatorIdentifier.id)))
280+
}
281+
282+
it should "report no loop starts for a plan without a loop block" in {
283+
val (responseType, loopStarts) =
284+
loopStartsOf(postCompile(chainedPlan(csvOp(realCsvPath), projectOp(List("Region")))))
285+
286+
assert(responseType == "success")
287+
assert(loopStarts.isEmpty)
288+
}
175289
}

0 commit comments

Comments
 (0)