@@ -126,9 +126,42 @@ class DataProcessingSpec
126126 completion.setDone()
127127 }
128128 })
129- Await .result(client.controllerInterface.startWorkflow(EmptyRequest (), ()))
130- Await .result(completion, Duration .fromMinutes(1 ))
131- results
129+ try {
130+ Await .result(client.controllerInterface.startWorkflow(EmptyRequest (), ()))
131+ Await .result(completion, Duration .fromMinutes(1 ))
132+ results
133+ } finally {
134+ client.shutdown()
135+ }
136+ }
137+
138+ /**
139+ * In the CI environment, there is a chance that executeWorkflow does not receive "COMPLETED" status.
140+ * Until we find the root cause of this issue, we use a retry mechanism here to stablize CI runs.
141+ */
142+ def executeWorkflowWithRetries (
143+ workflow : Workflow ,
144+ maxRetries : Int = 3
145+ ): Map [OperatorIdentity , List [Tuple ]] = {
146+ var attempt = 0
147+ while (attempt <= maxRetries) {
148+ try {
149+ return executeWorkflow(workflow)
150+ } catch {
151+ case _ : com.twitter.util.TimeoutException =>
152+ attempt += 1
153+ if (attempt > maxRetries) {
154+ throw new com.twitter.util.TimeoutException (" executeWorkflow timed out after retries" )
155+ }
156+ // Need to reset test texera_db state before retry
157+ cleanupWorkflowExecutionData()
158+ setUpWorkflowExecutionData()
159+ case otherError : Throwable =>
160+ throw otherError
161+ }
162+ }
163+ // Code should not reach here.
164+ throw new RuntimeException (" unreachable" )
132165 }
133166
134167 " Engine" should " execute headerlessCsv workflow normally" in {
@@ -138,7 +171,7 @@ class DataProcessingSpec
138171 List (),
139172 workflowContext
140173 )
141- val results = executeWorkflow (workflow)(headerlessCsvOpDesc.operatorIdentifier)
174+ val results = executeWorkflowWithRetries (workflow)(headerlessCsvOpDesc.operatorIdentifier)
142175
143176 assert(results.size == 100 )
144177 }
@@ -150,7 +183,7 @@ class DataProcessingSpec
150183 List (),
151184 workflowContext
152185 )
153- val results = executeWorkflow (workflow)(headerlessCsvOpDesc.operatorIdentifier)
186+ val results = executeWorkflowWithRetries (workflow)(headerlessCsvOpDesc.operatorIdentifier)
154187
155188 assert(results.size == 100 )
156189 }
@@ -162,7 +195,7 @@ class DataProcessingSpec
162195 List (),
163196 workflowContext
164197 )
165- val results = executeWorkflow (workflow)(jsonlOp.operatorIdentifier)
198+ val results = executeWorkflowWithRetries (workflow)(jsonlOp.operatorIdentifier)
166199
167200 assert(results.size == 100 )
168201
@@ -185,7 +218,7 @@ class DataProcessingSpec
185218 List (),
186219 workflowContext
187220 )
188- val results = executeWorkflow (workflow)(jsonlOp.operatorIdentifier)
221+ val results = executeWorkflowWithRetries (workflow)(jsonlOp.operatorIdentifier)
189222
190223 assert(results.size == 1000 )
191224
@@ -216,7 +249,7 @@ class DataProcessingSpec
216249 ),
217250 workflowContext
218251 )
219- executeWorkflow (workflow)
252+ executeWorkflowWithRetries (workflow)
220253 }
221254
222255 " Engine" should " execute csv workflow normally" in {
@@ -226,7 +259,7 @@ class DataProcessingSpec
226259 List (),
227260 workflowContext
228261 )
229- executeWorkflow (workflow)
262+ executeWorkflowWithRetries (workflow)
230263 }
231264
232265 " Engine" should " execute csv->keyword workflow normally" in {
@@ -244,7 +277,7 @@ class DataProcessingSpec
244277 ),
245278 workflowContext
246279 )
247- executeWorkflow (workflow)
280+ executeWorkflowWithRetries (workflow)
248281 }
249282
250283 " Engine" should " execute csv->keyword->count workflow normally" in {
@@ -270,7 +303,7 @@ class DataProcessingSpec
270303 ),
271304 workflowContext
272305 )
273- executeWorkflow (workflow)
306+ executeWorkflowWithRetries (workflow)
274307 }
275308
276309 " Engine" should " execute csv->keyword->averageAndGroupBy workflow normally" in {
@@ -300,7 +333,7 @@ class DataProcessingSpec
300333 ),
301334 workflowContext
302335 )
303- executeWorkflow (workflow)
336+ executeWorkflowWithRetries (workflow)
304337 }
305338
306339 " Engine" should " execute csv->(csv->)->join workflow normally" in {
@@ -329,6 +362,6 @@ class DataProcessingSpec
329362 ),
330363 workflowContext
331364 )
332- executeWorkflow (workflow)
365+ executeWorkflowWithRetries (workflow)
333366 }
334367}
0 commit comments