Skip to content

Commit 9a31f76

Browse files
committed
test(amber): add X-shaped multi-region workflow e2e integration test
Adds MultiRegionWorkflowIntegrationSpec, an @IntegrationTEST that runs a big X-shaped workflow end-to-end across multiple regions: two CSV sources fan into a hash join (build/probe) feeding a Python sort UDF, and also into a union; the join and union are the two terminal (materialized) outputs. Cross-region data delivery used to go through cache-source operators, which this branch removes; #3425 replaced them with input-port materialization reader threads. Until now that path had no end-to-end coverage -- only scheduling unit tests counted regions on a physical plan. This spec guards it: - asserts the plan spans >= 2 regions (via the same CostBasedScheduleGenerator the controller uses), so the join's build/probe dependency forces a materialized cross-region boundary; - verifies correctness through that boundary: the self-join on the unique 'Order ID' key yields 100 rows, the Python UDF returns them sorted, and the union concatenates both sources into 200 rows. Python-tagged, so it runs in the amber-integration CI job.
1 parent e89fe4a commit 9a31f76

1 file changed

Lines changed: 286 additions & 0 deletions

File tree

Lines changed: 286 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,286 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.texera.amber.engine.e2e
21+
22+
import com.twitter.util.{Await, Duration, Promise, Return}
23+
import com.typesafe.scalalogging.Logger
24+
import org.apache.pekko.actor.{ActorSystem, Props}
25+
import org.apache.pekko.testkit.{ImplicitSender, TestKit}
26+
import org.apache.pekko.util.Timeout
27+
import org.apache.texera.amber.clustering.SingleNodeListener
28+
import org.apache.texera.amber.core.workflow.PortIdentity
29+
import org.apache.texera.amber.engine.architecture.controller.{
30+
ControllerConfig,
31+
ExecutionStateUpdate
32+
}
33+
import org.apache.texera.amber.engine.architecture.rpc.controlcommands.EmptyRequest
34+
import org.apache.texera.amber.engine.architecture.rpc.controlreturns.WorkflowAggregatedState.COMPLETED
35+
import org.apache.texera.amber.engine.architecture.scheduling.CostBasedScheduleGenerator
36+
import org.apache.texera.amber.engine.common.AmberRuntime
37+
import org.apache.texera.amber.engine.common.client.AmberClient
38+
import org.apache.texera.amber.engine.common.virtualidentity.util.CONTROLLER
39+
import org.apache.texera.amber.engine.e2e.TestUtils.{
40+
buildWorkflow,
41+
cleanupWorkflowExecutionData,
42+
initiateTexeraDBForTestCases,
43+
runWorkflowAndReadTerminalResults,
44+
setUpWorkflowExecutionData
45+
}
46+
import org.apache.texera.amber.operator.TestOperators
47+
import org.apache.texera.amber.operator.source.scan.text.TextInputSourceOpDesc
48+
import org.apache.texera.amber.operator.union.UnionOpDesc
49+
import org.apache.texera.amber.tags.IntegrationTest
50+
import org.apache.texera.workflow.LogicalLink
51+
import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach, Outcome, Retries}
52+
import org.scalatest.flatspec.AnyFlatSpecLike
53+
54+
import scala.concurrent.duration._
55+
56+
/**
57+
* End-to-end coverage for a workflow that executes across MULTIPLE regions.
58+
*
59+
* Cross-region data used to be delivered by dedicated cache-source operators.
60+
* #3425 replaced those with input-port materialization reader threads, and the
61+
* now-dead cache-source plumbing is being removed. Prior to this spec the only
62+
* multi-region coverage was in the scheduling unit tests (region counting on a
63+
* physical plan); nothing exercised a real multi-region run end-to-end, so a
64+
* regression in the materialization read/write path could pass CI unnoticed.
65+
*
66+
* This spec drives a big "X"-shaped workflow to guard that path:
67+
*
68+
* {{{
69+
* csvLeft ─┬─▶ join.build (port 0) ─┐
70+
* │ join ─▶ pythonSort ─▶ (terminal)
71+
* csvRight ─┼─▶ join.probe (port 1) ─┘
72+
* │
73+
* csvLeft ─┼─▶ union ───────────────────────────────────▶ (terminal)
74+
* csvRight ─┘ (two links fan into union's single port)
75+
* }}}
76+
*
77+
* The hash join's probe-depends-on-build ordering forces the build input to be
78+
* materialized, cutting the plan into >=2 regions whose boundary is crossed by
79+
* the reader-thread path. A Python UDF (sort) makes it a real end-to-end run
80+
* with worker subprocesses, so it is class-level `@IntegrationTest` tagged and
81+
* routed to the `amber-integration` CI job (which provisions Python deps); the
82+
* lighter `amber` job excludes this tag.
83+
*/
84+
@IntegrationTest
85+
class MultiRegionWorkflowIntegrationSpec
86+
extends TestKit(ActorSystem("MultiRegionWorkflowIntegrationSpec", AmberRuntime.pekkoConfig))
87+
with ImplicitSender
88+
with AnyFlatSpecLike
89+
with BeforeAndAfterAll
90+
with BeforeAndAfterEach
91+
with Retries {
92+
93+
/**
94+
* Retry each test once if it fails. Mirrors the other e2e integration specs:
95+
* in CI there is a small chance the run does not observe "COMPLETED", so a
96+
* single retry stabilizes the suite until that root cause is fixed.
97+
*/
98+
override def withFixture(test: NoArgTest): Outcome =
99+
withRetry { super.withFixture(test) }
100+
101+
implicit val timeout: Timeout = Timeout(5.seconds)
102+
103+
private val logger = Logger("MultiRegionWorkflowIntegrationSpecLogger")
104+
private val specId = 5
105+
private val ctx = TestUtils.workflowContext(specId)
106+
107+
// country_sales_small.csv has 100 rows with a unique "Order ID" per row.
108+
private val sourceRowCount = 100
109+
110+
// Buffer every input tuple, then emit them ordered by "Order ID" once the
111+
// input port is exhausted. This makes the UDF a genuine sort and forces a real
112+
// Python worker to process data that crossed a region boundary.
113+
private val sortByOrderIdCode =
114+
"""
115+
|from pytexera import *
116+
|
117+
|class ProcessTupleOperator(UDFOperatorV2):
118+
| def open(self) -> None:
119+
| self.buffer = []
120+
|
121+
| @overrides
122+
| def process_tuple(self, tuple_: Tuple, port: int) -> Iterator[Optional[TupleLike]]:
123+
| self.buffer.append(tuple_)
124+
| yield from []
125+
|
126+
| @overrides
127+
| def on_finish(self, port: int) -> Iterator[Optional[TupleLike]]:
128+
| for row in sorted(self.buffer, key=lambda t: t["Order ID"]):
129+
| yield row
130+
|""".stripMargin
131+
132+
override protected def beforeEach(): Unit = {
133+
setUpWorkflowExecutionData(specId)
134+
}
135+
136+
override protected def afterEach(): Unit = {
137+
cleanupWorkflowExecutionData(specId)
138+
}
139+
140+
override def beforeAll(): Unit = {
141+
system.actorOf(Props[SingleNodeListener](), "cluster-info")
142+
// These test cases access postgres in CI, but occasionally the jdbc driver cannot be found during CI run.
143+
// Explicitly load the JDBC driver to avoid flaky CI failures.
144+
Class.forName("org.postgresql.Driver")
145+
initiateTexeraDBForTestCases()
146+
warmupOnce()
147+
}
148+
149+
override def afterAll(): Unit = {
150+
TestKit.shutdownActorSystem(system)
151+
}
152+
153+
/**
154+
* Runs a TextInput -> Python UDF workflow once before the timed test so the
155+
* Python worker cold-start is paid here, not inside the timed run. Capped and
156+
* wrapped so warmup can never fail or hang the suite.
157+
*/
158+
private def warmupOnce(): Unit = {
159+
val warmupCap = Duration.fromSeconds(60)
160+
setUpWorkflowExecutionData(specId)
161+
var client: AmberClient = null
162+
try {
163+
val src = new TextInputSourceOpDesc()
164+
src.textInput = "warmup"
165+
val udf = TestOperators.pythonOpDesc()
166+
val warmupCtx = TestUtils.workflowContext(specId)
167+
val workflow = buildWorkflow(
168+
List(src, udf),
169+
List(
170+
LogicalLink(
171+
src.operatorIdentifier,
172+
PortIdentity(),
173+
udf.operatorIdentifier,
174+
PortIdentity()
175+
)
176+
),
177+
warmupCtx
178+
)
179+
client = new AmberClient(
180+
system,
181+
workflow.context,
182+
workflow.physicalPlan,
183+
ControllerConfig.default,
184+
_ => {}
185+
)
186+
val completion = Promise[Unit]()
187+
client.registerCallback[ExecutionStateUpdate](evt => {
188+
if (evt.state == COMPLETED) completion.updateIfEmpty(Return(()))
189+
})
190+
Await.result(
191+
client.controllerInterface.startWorkflow(EmptyRequest(), ()),
192+
warmupCap
193+
)
194+
Await.result(completion, warmupCap)
195+
} catch {
196+
case e: Throwable =>
197+
logger.warn(
198+
s"warmup workflow did not finish within ${warmupCap}; tests will run cold and rely on Retries: ${e.getMessage}"
199+
)
200+
} finally {
201+
if (client != null) {
202+
try client.shutdown()
203+
catch { case _: Throwable => () }
204+
}
205+
cleanupWorkflowExecutionData(specId)
206+
}
207+
}
208+
209+
"Engine" should "execute an X-shaped multi-region workflow (join + union + python) end-to-end" in {
210+
val csvLeft = TestOperators.smallCsvScanOpDesc()
211+
val csvRight = TestOperators.smallCsvScanOpDesc()
212+
// Self-join on the unique "Order ID" key: each row matches exactly one row
213+
// from the other source, so the join emits exactly `sourceRowCount` rows.
214+
val join = TestOperators.joinOpDesc("Order ID", "Order ID")
215+
val pythonSort = TestOperators.pythonOpDesc()
216+
pythonSort.code = sortByOrderIdCode
217+
val union = new UnionOpDesc()
218+
219+
val workflow = buildWorkflow(
220+
List(csvLeft, csvRight, join, pythonSort, union),
221+
List(
222+
// Left arm of the X: the two sources feed the join (build/probe) ...
223+
LogicalLink(
224+
csvLeft.operatorIdentifier,
225+
PortIdentity(),
226+
join.operatorIdentifier,
227+
PortIdentity()
228+
),
229+
LogicalLink(
230+
csvRight.operatorIdentifier,
231+
PortIdentity(),
232+
join.operatorIdentifier,
233+
PortIdentity(1)
234+
),
235+
LogicalLink(
236+
join.operatorIdentifier,
237+
PortIdentity(),
238+
pythonSort.operatorIdentifier,
239+
PortIdentity()
240+
),
241+
// ... and also fan into the union's single input port (the crossing).
242+
LogicalLink(
243+
csvLeft.operatorIdentifier,
244+
PortIdentity(),
245+
union.operatorIdentifier,
246+
PortIdentity()
247+
),
248+
LogicalLink(
249+
csvRight.operatorIdentifier,
250+
PortIdentity(),
251+
union.operatorIdentifier,
252+
PortIdentity()
253+
)
254+
),
255+
ctx
256+
)
257+
258+
// The join's build/probe dependency cuts the plan into multiple regions
259+
// whose boundary is crossed by the input-port materialization reader path.
260+
// Assert that with the same generator the controller uses at runtime.
261+
val (schedule, _) = new CostBasedScheduleGenerator(
262+
workflow.context,
263+
workflow.physicalPlan,
264+
CONTROLLER
265+
).generate()
266+
assert(
267+
schedule.getRegions.size >= 2,
268+
s"expected the X-shaped workflow to span multiple regions, got ${schedule.getRegions.size}"
269+
)
270+
271+
val results = runWorkflowAndReadTerminalResults(system, workflow, Duration.fromMinutes(2))
272+
273+
val sortedRows = results(pythonSort.operatorIdentifier)
274+
val unionRows = results(union.operatorIdentifier)
275+
276+
// Join branch: self-join on a unique key emits one row per source row, and
277+
// the Python UDF returns them ordered by "Order ID".
278+
assert(sortedRows.size == sourceRowCount)
279+
val orderIds = sortedRows.map(_.getField("Order ID").asInstanceOf[Number].longValue())
280+
assert(orderIds == orderIds.sorted, "python UDF output should be sorted by Order ID")
281+
282+
// Union branch: concatenation of both sources, which share a schema.
283+
assert(unionRows.size == 2 * sourceRowCount)
284+
}
285+
286+
}

0 commit comments

Comments
 (0)