|
| 1 | +/* |
| 2 | + * Licensed to the Apache Software Foundation (ASF) under one |
| 3 | + * or more contributor license agreements. See the NOTICE file |
| 4 | + * distributed with this work for additional information |
| 5 | + * regarding copyright ownership. The ASF licenses this file |
| 6 | + * to you under the Apache License, Version 2.0 (the |
| 7 | + * "License"); you may not use this file except in compliance |
| 8 | + * with the License. You may obtain a copy of the License at |
| 9 | + * |
| 10 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | + * |
| 12 | + * Unless required by applicable law or agreed to in writing, |
| 13 | + * software distributed under the License is distributed on an |
| 14 | + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 15 | + * KIND, either express or implied. See the License for the |
| 16 | + * specific language governing permissions and limitations |
| 17 | + * under the License. |
| 18 | + */ |
| 19 | + |
| 20 | +package org.apache.texera.amber.engine.e2e |
| 21 | + |
| 22 | +import com.twitter.util.{Await, Duration, Promise, Return} |
| 23 | +import com.typesafe.scalalogging.Logger |
| 24 | +import org.apache.pekko.actor.{ActorSystem, Props} |
| 25 | +import org.apache.pekko.testkit.{ImplicitSender, TestKit} |
| 26 | +import org.apache.pekko.util.Timeout |
| 27 | +import org.apache.texera.amber.clustering.SingleNodeListener |
| 28 | +import org.apache.texera.amber.core.workflow.PortIdentity |
| 29 | +import org.apache.texera.amber.engine.architecture.controller.{ |
| 30 | + ControllerConfig, |
| 31 | + ExecutionStateUpdate |
| 32 | +} |
| 33 | +import org.apache.texera.amber.engine.architecture.rpc.controlcommands.EmptyRequest |
| 34 | +import org.apache.texera.amber.engine.architecture.rpc.controlreturns.WorkflowAggregatedState.COMPLETED |
| 35 | +import org.apache.texera.amber.engine.architecture.scheduling.CostBasedScheduleGenerator |
| 36 | +import org.apache.texera.amber.engine.common.AmberRuntime |
| 37 | +import org.apache.texera.amber.engine.common.client.AmberClient |
| 38 | +import org.apache.texera.amber.engine.common.virtualidentity.util.CONTROLLER |
| 39 | +import org.apache.texera.amber.engine.e2e.TestUtils.{ |
| 40 | + buildWorkflow, |
| 41 | + cleanupWorkflowExecutionData, |
| 42 | + initiateTexeraDBForTestCases, |
| 43 | + runWorkflowAndReadTerminalResults, |
| 44 | + setUpWorkflowExecutionData |
| 45 | +} |
| 46 | +import org.apache.texera.amber.operator.TestOperators |
| 47 | +import org.apache.texera.amber.operator.source.scan.text.TextInputSourceOpDesc |
| 48 | +import org.apache.texera.amber.operator.union.UnionOpDesc |
| 49 | +import org.apache.texera.amber.tags.IntegrationTest |
| 50 | +import org.apache.texera.workflow.LogicalLink |
| 51 | +import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach, Outcome, Retries} |
| 52 | +import org.scalatest.flatspec.AnyFlatSpecLike |
| 53 | + |
| 54 | +import scala.concurrent.duration._ |
| 55 | + |
| 56 | +/** |
| 57 | + * End-to-end coverage for a workflow that executes across MULTIPLE regions. |
| 58 | + * |
| 59 | + * Cross-region data used to be delivered by dedicated cache-source operators. |
| 60 | + * #3425 replaced those with input-port materialization reader threads, and the |
| 61 | + * now-dead cache-source plumbing is being removed. Prior to this spec the only |
| 62 | + * multi-region coverage was in the scheduling unit tests (region counting on a |
| 63 | + * physical plan); nothing exercised a real multi-region run end-to-end, so a |
| 64 | + * regression in the materialization read/write path could pass CI unnoticed. |
| 65 | + * |
| 66 | + * This spec drives a big "X"-shaped workflow to guard that path: |
| 67 | + * |
| 68 | + * {{{ |
| 69 | + * csvLeft ─┬─▶ join.build (port 0) ─┐ |
| 70 | + * │ join ─▶ pythonSort ─▶ (terminal) |
| 71 | + * csvRight ─┼─▶ join.probe (port 1) ─┘ |
| 72 | + * │ |
| 73 | + * csvLeft ─┼─▶ union ───────────────────────────────────▶ (terminal) |
| 74 | + * csvRight ─┘ (two links fan into union's single port) |
| 75 | + * }}} |
| 76 | + * |
| 77 | + * The hash join's probe-depends-on-build ordering forces the build input to be |
| 78 | + * materialized, cutting the plan into >=2 regions whose boundary is crossed by |
| 79 | + * the reader-thread path. A Python UDF (sort) makes it a real end-to-end run |
| 80 | + * with worker subprocesses, so it is class-level `@IntegrationTest` tagged and |
| 81 | + * routed to the `amber-integration` CI job (which provisions Python deps); the |
| 82 | + * lighter `amber` job excludes this tag. |
| 83 | + */ |
| 84 | +@IntegrationTest |
| 85 | +class MultiRegionWorkflowIntegrationSpec |
| 86 | + extends TestKit(ActorSystem("MultiRegionWorkflowIntegrationSpec", AmberRuntime.pekkoConfig)) |
| 87 | + with ImplicitSender |
| 88 | + with AnyFlatSpecLike |
| 89 | + with BeforeAndAfterAll |
| 90 | + with BeforeAndAfterEach |
| 91 | + with Retries { |
| 92 | + |
| 93 | + /** |
| 94 | + * Retry each test once if it fails. Mirrors the other e2e integration specs: |
| 95 | + * in CI there is a small chance the run does not observe "COMPLETED", so a |
| 96 | + * single retry stabilizes the suite until that root cause is fixed. |
| 97 | + */ |
| 98 | + override def withFixture(test: NoArgTest): Outcome = |
| 99 | + withRetry { super.withFixture(test) } |
| 100 | + |
| 101 | + implicit val timeout: Timeout = Timeout(5.seconds) |
| 102 | + |
| 103 | + private val logger = Logger("MultiRegionWorkflowIntegrationSpecLogger") |
| 104 | + private val specId = 5 |
| 105 | + private val ctx = TestUtils.workflowContext(specId) |
| 106 | + |
| 107 | + // country_sales_small.csv has 100 rows with a unique "Order ID" per row. |
| 108 | + private val sourceRowCount = 100 |
| 109 | + |
| 110 | + // Buffer every input tuple, then emit them ordered by "Order ID" once the |
| 111 | + // input port is exhausted. This makes the UDF a genuine sort and forces a real |
| 112 | + // Python worker to process data that crossed a region boundary. |
| 113 | + private val sortByOrderIdCode = |
| 114 | + """ |
| 115 | + |from pytexera import * |
| 116 | + | |
| 117 | + |class ProcessTupleOperator(UDFOperatorV2): |
| 118 | + | def open(self) -> None: |
| 119 | + | self.buffer = [] |
| 120 | + | |
| 121 | + | @overrides |
| 122 | + | def process_tuple(self, tuple_: Tuple, port: int) -> Iterator[Optional[TupleLike]]: |
| 123 | + | self.buffer.append(tuple_) |
| 124 | + | yield from [] |
| 125 | + | |
| 126 | + | @overrides |
| 127 | + | def on_finish(self, port: int) -> Iterator[Optional[TupleLike]]: |
| 128 | + | for row in sorted(self.buffer, key=lambda t: t["Order ID"]): |
| 129 | + | yield row |
| 130 | + |""".stripMargin |
| 131 | + |
| 132 | + override protected def beforeEach(): Unit = { |
| 133 | + setUpWorkflowExecutionData(specId) |
| 134 | + } |
| 135 | + |
| 136 | + override protected def afterEach(): Unit = { |
| 137 | + cleanupWorkflowExecutionData(specId) |
| 138 | + } |
| 139 | + |
| 140 | + override def beforeAll(): Unit = { |
| 141 | + system.actorOf(Props[SingleNodeListener](), "cluster-info") |
| 142 | + // These test cases access postgres in CI, but occasionally the jdbc driver cannot be found during CI run. |
| 143 | + // Explicitly load the JDBC driver to avoid flaky CI failures. |
| 144 | + Class.forName("org.postgresql.Driver") |
| 145 | + initiateTexeraDBForTestCases() |
| 146 | + warmupOnce() |
| 147 | + } |
| 148 | + |
| 149 | + override def afterAll(): Unit = { |
| 150 | + TestKit.shutdownActorSystem(system) |
| 151 | + } |
| 152 | + |
| 153 | + /** |
| 154 | + * Runs a TextInput -> Python UDF workflow once before the timed test so the |
| 155 | + * Python worker cold-start is paid here, not inside the timed run. Capped and |
| 156 | + * wrapped so warmup can never fail or hang the suite. |
| 157 | + */ |
| 158 | + private def warmupOnce(): Unit = { |
| 159 | + val warmupCap = Duration.fromSeconds(60) |
| 160 | + setUpWorkflowExecutionData(specId) |
| 161 | + var client: AmberClient = null |
| 162 | + try { |
| 163 | + val src = new TextInputSourceOpDesc() |
| 164 | + src.textInput = "warmup" |
| 165 | + val udf = TestOperators.pythonOpDesc() |
| 166 | + val warmupCtx = TestUtils.workflowContext(specId) |
| 167 | + val workflow = buildWorkflow( |
| 168 | + List(src, udf), |
| 169 | + List( |
| 170 | + LogicalLink( |
| 171 | + src.operatorIdentifier, |
| 172 | + PortIdentity(), |
| 173 | + udf.operatorIdentifier, |
| 174 | + PortIdentity() |
| 175 | + ) |
| 176 | + ), |
| 177 | + warmupCtx |
| 178 | + ) |
| 179 | + client = new AmberClient( |
| 180 | + system, |
| 181 | + workflow.context, |
| 182 | + workflow.physicalPlan, |
| 183 | + ControllerConfig.default, |
| 184 | + _ => {} |
| 185 | + ) |
| 186 | + val completion = Promise[Unit]() |
| 187 | + client.registerCallback[ExecutionStateUpdate](evt => { |
| 188 | + if (evt.state == COMPLETED) completion.updateIfEmpty(Return(())) |
| 189 | + }) |
| 190 | + Await.result( |
| 191 | + client.controllerInterface.startWorkflow(EmptyRequest(), ()), |
| 192 | + warmupCap |
| 193 | + ) |
| 194 | + Await.result(completion, warmupCap) |
| 195 | + } catch { |
| 196 | + case e: Throwable => |
| 197 | + logger.warn( |
| 198 | + s"warmup workflow did not finish within ${warmupCap}; tests will run cold and rely on Retries: ${e.getMessage}" |
| 199 | + ) |
| 200 | + } finally { |
| 201 | + if (client != null) { |
| 202 | + try client.shutdown() |
| 203 | + catch { case _: Throwable => () } |
| 204 | + } |
| 205 | + cleanupWorkflowExecutionData(specId) |
| 206 | + } |
| 207 | + } |
| 208 | + |
| 209 | + "Engine" should "execute an X-shaped multi-region workflow (join + union + python) end-to-end" in { |
| 210 | + val csvLeft = TestOperators.smallCsvScanOpDesc() |
| 211 | + val csvRight = TestOperators.smallCsvScanOpDesc() |
| 212 | + // Self-join on the unique "Order ID" key: each row matches exactly one row |
| 213 | + // from the other source, so the join emits exactly `sourceRowCount` rows. |
| 214 | + val join = TestOperators.joinOpDesc("Order ID", "Order ID") |
| 215 | + val pythonSort = TestOperators.pythonOpDesc() |
| 216 | + pythonSort.code = sortByOrderIdCode |
| 217 | + val union = new UnionOpDesc() |
| 218 | + |
| 219 | + val workflow = buildWorkflow( |
| 220 | + List(csvLeft, csvRight, join, pythonSort, union), |
| 221 | + List( |
| 222 | + // Left arm of the X: the two sources feed the join (build/probe) ... |
| 223 | + LogicalLink( |
| 224 | + csvLeft.operatorIdentifier, |
| 225 | + PortIdentity(), |
| 226 | + join.operatorIdentifier, |
| 227 | + PortIdentity() |
| 228 | + ), |
| 229 | + LogicalLink( |
| 230 | + csvRight.operatorIdentifier, |
| 231 | + PortIdentity(), |
| 232 | + join.operatorIdentifier, |
| 233 | + PortIdentity(1) |
| 234 | + ), |
| 235 | + LogicalLink( |
| 236 | + join.operatorIdentifier, |
| 237 | + PortIdentity(), |
| 238 | + pythonSort.operatorIdentifier, |
| 239 | + PortIdentity() |
| 240 | + ), |
| 241 | + // ... and also fan into the union's single input port (the crossing). |
| 242 | + LogicalLink( |
| 243 | + csvLeft.operatorIdentifier, |
| 244 | + PortIdentity(), |
| 245 | + union.operatorIdentifier, |
| 246 | + PortIdentity() |
| 247 | + ), |
| 248 | + LogicalLink( |
| 249 | + csvRight.operatorIdentifier, |
| 250 | + PortIdentity(), |
| 251 | + union.operatorIdentifier, |
| 252 | + PortIdentity() |
| 253 | + ) |
| 254 | + ), |
| 255 | + ctx |
| 256 | + ) |
| 257 | + |
| 258 | + // The join's build/probe dependency cuts the plan into multiple regions |
| 259 | + // whose boundary is crossed by the input-port materialization reader path. |
| 260 | + // Assert that with the same generator the controller uses at runtime. |
| 261 | + val (schedule, _) = new CostBasedScheduleGenerator( |
| 262 | + workflow.context, |
| 263 | + workflow.physicalPlan, |
| 264 | + CONTROLLER |
| 265 | + ).generate() |
| 266 | + assert( |
| 267 | + schedule.getRegions.size >= 2, |
| 268 | + s"expected the X-shaped workflow to span multiple regions, got ${schedule.getRegions.size}" |
| 269 | + ) |
| 270 | + |
| 271 | + val results = runWorkflowAndReadTerminalResults(system, workflow, Duration.fromMinutes(2)) |
| 272 | + |
| 273 | + val sortedRows = results(pythonSort.operatorIdentifier) |
| 274 | + val unionRows = results(union.operatorIdentifier) |
| 275 | + |
| 276 | + // Join branch: self-join on a unique key emits one row per source row, and |
| 277 | + // the Python UDF returns them ordered by "Order ID". |
| 278 | + assert(sortedRows.size == sourceRowCount) |
| 279 | + val orderIds = sortedRows.map(_.getField("Order ID").asInstanceOf[Number].longValue()) |
| 280 | + assert(orderIds == orderIds.sorted, "python UDF output should be sorted by Order ID") |
| 281 | + |
| 282 | + // Union branch: concatenation of both sources, which share a schema. |
| 283 | + assert(unionRows.size == 2 * sourceRowCount) |
| 284 | + } |
| 285 | + |
| 286 | +} |
0 commit comments