Skip to content

Commit abca306

Browse files
authored
Prevent Duplicate Sink Operator Generation for Shared Storage URIs (#3264)
This PR addresses an issue in the workflow scheduler where duplicate sink operators are generated when multiple blocking downstream operators cause the split operator to produce multiple sinks with the same storage URI. With these changes, an additional check is added during the physical plan construction. If a sink operator with the given URI already exists, the scheduler will skip generating another one, thereby preventing redundancy. The PR will fix the issue #3263. <img width="1528" alt="Screenshot 2025-02-13 at 4 22 20 PM" src="https://github.com/user-attachments/assets/c8061e3a-50a7-4acf-a518-04cde6af9090" />
1 parent af8d2d0 commit abca306

1 file changed

Lines changed: 38 additions & 26 deletions

File tree

‎core/amber/src/main/scala/edu/uci/ics/amber/engine/architecture/scheduling/ScheduleGenerator.scala‎

Lines changed: 38 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package edu.uci.ics.amber.engine.architecture.scheduling
22

3+
import edu.uci.ics.amber.core.executor.OpExecSink
34
import edu.uci.ics.amber.core.storage.{DocumentFactory, VFSURIFactory}
45
import edu.uci.ics.amber.core.workflow.{PhysicalOp, PhysicalPlan, WorkflowContext}
56
import edu.uci.ics.amber.engine.architecture.scheduling.ScheduleGenerator.replaceVertex
@@ -151,7 +152,6 @@ abstract class ScheduleGenerator(
151152
var newPhysicalPlan = physicalPlan
152153
.removeLink(physicalLink)
153154

154-
// create cache writer and link
155155
// create the uri of the materialization storage
156156
val storageUri = VFSURIFactory.createMaterializedResultURI(
157157
workflowContext.workflowId,
@@ -166,32 +166,44 @@ abstract class ScheduleGenerator(
166166
storageUri,
167167
fromPortOutputMode
168168
)
169-
val sourceToWriterLink =
170-
PhysicalLink(
171-
fromOp.id,
172-
fromPortId,
173-
matWriterPhysicalOp.id,
174-
matWriterPhysicalOp.inputPorts.keys.head
169+
170+
// Check if an operator with the same storageUri already exists
171+
val existingOperator = newPhysicalPlan.operators.find {
172+
case op if op.opExecInitInfo.isInstanceOf[OpExecSink] =>
173+
val OpExecSink(uri, _, _) = op.opExecInitInfo
174+
uri == storageUri.toString
175+
case _ => false
176+
}
177+
178+
if (existingOperator.isEmpty) {
179+
// create cache writer and link
180+
val sourceToWriterLink =
181+
PhysicalLink(
182+
fromOp.id,
183+
fromPortId,
184+
matWriterPhysicalOp.id,
185+
matWriterPhysicalOp.inputPorts.keys.head
186+
)
187+
newPhysicalPlan = newPhysicalPlan
188+
.addOperator(matWriterPhysicalOp)
189+
.addLink(sourceToWriterLink)
190+
191+
// sink has exactly one input port and one output port
192+
val schema = newPhysicalPlan
193+
.getOperator(matWriterPhysicalOp.id)
194+
.outputPorts(matWriterPhysicalOp.outputPorts.keys.head)
195+
._3
196+
.toOption
197+
.get
198+
// create the document
199+
DocumentFactory.createDocument(storageUri, schema)
200+
WorkflowExecutionsResource.insertOperatorPortResultUri(
201+
workflowContext.executionId,
202+
physicalLink.fromOpId.logicalOpId,
203+
physicalLink.fromPortId,
204+
storageUri
175205
)
176-
newPhysicalPlan = newPhysicalPlan
177-
.addOperator(matWriterPhysicalOp)
178-
.addLink(sourceToWriterLink)
179-
180-
// sink has exactly one input port and one output port
181-
val schema = newPhysicalPlan
182-
.getOperator(matWriterPhysicalOp.id)
183-
.outputPorts(matWriterPhysicalOp.outputPorts.keys.head)
184-
._3
185-
.toOption
186-
.get
187-
// create the document
188-
DocumentFactory.createDocument(storageUri, schema)
189-
WorkflowExecutionsResource.insertOperatorPortResultUri(
190-
workflowContext.executionId,
191-
physicalLink.fromOpId.logicalOpId,
192-
physicalLink.fromPortId,
193-
storageUri
194-
)
206+
}
195207

196208
// create cache reader and link
197209
val matReaderPhysicalOp: PhysicalOp = SpecialPhysicalOpFactory.newSourcePhysicalOp(

0 commit comments

Comments
 (0)