Skip to content

Commit 84e0253

Browse files
committed
fix(amber): add timeouts and retries to e2e tests
1 parent 4b9e0d0 commit 84e0253

2 files changed

Lines changed: 91 additions & 25 deletions

File tree

‎amber/src/test/scala/org/apache/amber/engine/e2e/DataProcessingSpec.scala‎

Lines changed: 46 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -126,9 +126,42 @@ class DataProcessingSpec
126126
completion.setDone()
127127
}
128128
})
129-
Await.result(client.controllerInterface.startWorkflow(EmptyRequest(), ()))
130-
Await.result(completion, Duration.fromMinutes(1))
131-
results
129+
try {
130+
Await.result(client.controllerInterface.startWorkflow(EmptyRequest(), ()))
131+
Await.result(completion, Duration.fromMinutes(1))
132+
results
133+
} finally {
134+
client.shutdown()
135+
}
136+
}
137+
138+
/**
139+
* In the CI environment, there is a chance that executeWorkflow does not receive "COMPLETED" status.
140+
* Until we find the root cause of this issue, we use a retry mechanism here to stablize CI runs.
141+
*/
142+
def executeWorkflowWithRetries(
143+
workflow: Workflow,
144+
maxRetries: Int = 3
145+
): Map[OperatorIdentity, List[Tuple]] = {
146+
var attempt = 0
147+
while (attempt <= maxRetries) {
148+
try {
149+
return executeWorkflow(workflow)
150+
} catch {
151+
case _: com.twitter.util.TimeoutException =>
152+
attempt += 1
153+
if (attempt > maxRetries) {
154+
throw new com.twitter.util.TimeoutException("executeWorkflow timed out after retries")
155+
}
156+
// Need to reset test texera_db state before retry
157+
cleanupWorkflowExecutionData()
158+
setUpWorkflowExecutionData()
159+
case otherError: Throwable =>
160+
throw otherError
161+
}
162+
}
163+
// Code should not reach here.
164+
throw new RuntimeException("unreachable")
132165
}
133166

134167
"Engine" should "execute headerlessCsv workflow normally" in {
@@ -138,7 +171,7 @@ class DataProcessingSpec
138171
List(),
139172
workflowContext
140173
)
141-
val results = executeWorkflow(workflow)(headerlessCsvOpDesc.operatorIdentifier)
174+
val results = executeWorkflowWithRetries(workflow)(headerlessCsvOpDesc.operatorIdentifier)
142175

143176
assert(results.size == 100)
144177
}
@@ -150,7 +183,7 @@ class DataProcessingSpec
150183
List(),
151184
workflowContext
152185
)
153-
val results = executeWorkflow(workflow)(headerlessCsvOpDesc.operatorIdentifier)
186+
val results = executeWorkflowWithRetries(workflow)(headerlessCsvOpDesc.operatorIdentifier)
154187

155188
assert(results.size == 100)
156189
}
@@ -162,7 +195,7 @@ class DataProcessingSpec
162195
List(),
163196
workflowContext
164197
)
165-
val results = executeWorkflow(workflow)(jsonlOp.operatorIdentifier)
198+
val results = executeWorkflowWithRetries(workflow)(jsonlOp.operatorIdentifier)
166199

167200
assert(results.size == 100)
168201

@@ -185,7 +218,7 @@ class DataProcessingSpec
185218
List(),
186219
workflowContext
187220
)
188-
val results = executeWorkflow(workflow)(jsonlOp.operatorIdentifier)
221+
val results = executeWorkflowWithRetries(workflow)(jsonlOp.operatorIdentifier)
189222

190223
assert(results.size == 1000)
191224

@@ -216,7 +249,7 @@ class DataProcessingSpec
216249
),
217250
workflowContext
218251
)
219-
executeWorkflow(workflow)
252+
executeWorkflowWithRetries(workflow)
220253
}
221254

222255
"Engine" should "execute csv workflow normally" in {
@@ -226,7 +259,7 @@ class DataProcessingSpec
226259
List(),
227260
workflowContext
228261
)
229-
executeWorkflow(workflow)
262+
executeWorkflowWithRetries(workflow)
230263
}
231264

232265
"Engine" should "execute csv->keyword workflow normally" in {
@@ -244,7 +277,7 @@ class DataProcessingSpec
244277
),
245278
workflowContext
246279
)
247-
executeWorkflow(workflow)
280+
executeWorkflowWithRetries(workflow)
248281
}
249282

250283
"Engine" should "execute csv->keyword->count workflow normally" in {
@@ -270,7 +303,7 @@ class DataProcessingSpec
270303
),
271304
workflowContext
272305
)
273-
executeWorkflow(workflow)
306+
executeWorkflowWithRetries(workflow)
274307
}
275308

276309
"Engine" should "execute csv->keyword->averageAndGroupBy workflow normally" in {
@@ -300,7 +333,7 @@ class DataProcessingSpec
300333
),
301334
workflowContext
302335
)
303-
executeWorkflow(workflow)
336+
executeWorkflowWithRetries(workflow)
304337
}
305338

306339
"Engine" should "execute csv->(csv->)->join workflow normally" in {
@@ -329,6 +362,6 @@ class DataProcessingSpec
329362
),
330363
workflowContext
331364
)
332-
executeWorkflow(workflow)
365+
executeWorkflowWithRetries(workflow)
333366
}
334367
}

‎amber/src/test/scala/org/apache/amber/engine/e2e/PauseSpec.scala‎

Lines changed: 45 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ package org.apache.amber.engine.e2e
2222
import akka.actor.{ActorSystem, Props}
2323
import akka.testkit.{ImplicitSender, TestKit}
2424
import akka.util.Timeout
25-
import com.twitter.util.{Await, Promise}
25+
import com.twitter.util.{Await, Duration, Promise}
2626
import com.typesafe.scalalogging.Logger
2727
import org.apache.amber.clustering.SingleNodeListener
2828
import org.apache.amber.core.workflow.{PortIdentity, WorkflowContext}
@@ -95,21 +95,54 @@ class PauseSpec
9595
completion.setDone()
9696
}
9797
})
98-
Await.result(client.controllerInterface.startWorkflow(EmptyRequest(), ()))
99-
Await.result(client.controllerInterface.pauseWorkflow(EmptyRequest(), ()))
100-
Thread.sleep(4000)
101-
Await.result(client.controllerInterface.resumeWorkflow(EmptyRequest(), ()))
102-
Thread.sleep(400)
103-
Await.result(client.controllerInterface.pauseWorkflow(EmptyRequest(), ()))
104-
Thread.sleep(4000)
105-
Await.result(client.controllerInterface.resumeWorkflow(EmptyRequest(), ()))
106-
Await.result(completion)
98+
try {
99+
Await.result(client.controllerInterface.startWorkflow(EmptyRequest(), ()))
100+
Await.result(client.controllerInterface.pauseWorkflow(EmptyRequest(), ()))
101+
Thread.sleep(4000)
102+
Await.result(client.controllerInterface.resumeWorkflow(EmptyRequest(), ()))
103+
Thread.sleep(400)
104+
Await.result(client.controllerInterface.pauseWorkflow(EmptyRequest(), ()))
105+
Thread.sleep(4000)
106+
Await.result(client.controllerInterface.resumeWorkflow(EmptyRequest(), ()))
107+
Await.result(completion, Duration.fromMinutes(1))
108+
} finally {
109+
client.shutdown()
110+
}
111+
}
112+
113+
/**
114+
* In the CI environment, there is a chance that shouldPause does not receive "COMPLETED" status.
115+
* Until we find the root cause of this issue, we use a retry mechanism here to stablize CI runs.
116+
*/
117+
def shouldPauseWithRetries(
118+
operators: List[LogicalOp],
119+
links: List[LogicalLink],
120+
maxRetries: Int = 3
121+
): Unit = {
122+
var attempt = 0
123+
while (attempt <= maxRetries) {
124+
try {
125+
shouldPause(operators, links)
126+
return
127+
} catch {
128+
case _: com.twitter.util.TimeoutException =>
129+
attempt += 1
130+
if (attempt > maxRetries) {
131+
throw new com.twitter.util.TimeoutException("shouldPause timed out after retries")
132+
}
133+
// Need to reset test texera_db state before retry
134+
cleanupWorkflowExecutionData()
135+
setUpWorkflowExecutionData()
136+
case otherError: Throwable =>
137+
throw otherError
138+
}
139+
}
107140
}
108141

109142
"Engine" should "be able to pause csv workflow" in {
110143
val csvOpDesc = TestOperators.mediumCsvScanOpDesc()
111144
logger.info(s"csv-id ${csvOpDesc.operatorIdentifier}")
112-
shouldPause(
145+
shouldPauseWithRetries(
113146
List(csvOpDesc),
114147
List()
115148
)
@@ -121,7 +154,7 @@ class PauseSpec
121154
logger.info(
122155
s"csv-id ${csvOpDesc.operatorIdentifier}, keyword-id ${keywordOpDesc.operatorIdentifier}"
123156
)
124-
shouldPause(
157+
shouldPauseWithRetries(
125158
List(csvOpDesc, keywordOpDesc),
126159
List(
127160
LogicalLink(

0 commit comments

Comments
 (0)