Skip to content

Commit f6d6041

Browse files
authored
[fix](iceberg) Fix some issues in batch mode (apache#51185)
### What problem does this PR solve? Problem Summary: 1. When adding splits to the queue, a retry mechanism is added to prevent blocking and sticking. 2. We use a separate thread to fetch splits. When the SQL execution ends early (such as `limit`), this thread needs to be handled. Now it is exited by throwing an exception. Moreover, this thread should use an independent thread pool to prevent system freezes caused by resource contention. 3. When there are multiple exceptions, overwriting would occur previously. Now we use `addSuppressed` to prevent exception loss. 4. When initializing the split assignment, a timeout mechanism is added to prevent the situation where the system gets stuck due to failing to obtain splits. 5. When the split assignment stop, if an exception is not null, an exception should be thrown externally.
1 parent f138847 commit f6d6041

8 files changed

Lines changed: 508 additions & 44 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ public class ExternalMetaCacheMgr {
8484
private ExecutorService rowCountRefreshExecutor;
8585
private ExecutorService commonRefreshExecutor;
8686
private ExecutorService fileListingExecutor;
87+
// This executor is used to schedule the getting split tasks
8788
private ExecutorService scheduleExecutor;
8889

8990
// catalog id -> HiveMetaStoreCache

‎fe/fe-core/src/main/java/org/apache/doris/datasource/SplitAssignment.java‎

Lines changed: 37 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import java.util.concurrent.BlockingQueue;
3737
import java.util.concurrent.ConcurrentHashMap;
3838
import java.util.concurrent.LinkedBlockingQueue;
39+
import java.util.concurrent.TimeUnit;
3940
import java.util.concurrent.atomic.AtomicBoolean;
4041

4142
/**
@@ -76,20 +77,28 @@ public SplitAssignment(
7677
public void init() throws UserException {
7778
splitGenerator.startSplit(backendPolicy.numBackends());
7879
synchronized (assignLock) {
79-
while (sampleSplit == null && waitFirstSplit()) {
80+
final int waitIntervalTimeMillis = 100;
81+
final int initTimeoutMillis = 30000; // 30s
82+
int waitTotalTime = 0;
83+
while (sampleSplit == null && needMoreSplit()) {
8084
try {
81-
assignLock.wait(100);
85+
assignLock.wait(waitIntervalTimeMillis);
8286
} catch (InterruptedException e) {
8387
throw new UserException(e.getMessage(), e);
8488
}
89+
waitTotalTime += waitIntervalTimeMillis;
90+
if (waitTotalTime > initTimeoutMillis) {
91+
throw new UserException("Failed to get first split after waiting for "
92+
+ (waitTotalTime / 1000) + " seconds.");
93+
}
8594
}
8695
}
8796
if (exception != null) {
8897
throw exception;
8998
}
9099
}
91100

92-
private boolean waitFirstSplit() {
101+
public boolean needMoreSplit() {
93102
return !scheduleFinished.get() && !isStopped.get() && exception == null;
94103
}
95104

@@ -100,10 +109,16 @@ private void appendBatch(Multimap<Backend, Split> batch) throws UserException {
100109
for (Split split : splits) {
101110
locations.add(splitToScanRange.getScanRange(backend, locationProperties, split, pathPartitionKeys));
102111
}
103-
try {
104-
assignment.computeIfAbsent(backend, be -> new LinkedBlockingQueue<>(10000)).put(locations);
105-
} catch (Exception e) {
106-
throw new UserException("Failed to offer batch split", e);
112+
while (needMoreSplit()) {
113+
BlockingQueue<Collection<TScanRangeLocations>> queue =
114+
assignment.computeIfAbsent(backend, be -> new LinkedBlockingQueue<>(10000));
115+
try {
116+
if (queue.offer(locations, 100, TimeUnit.MILLISECONDS)) {
117+
return;
118+
}
119+
} catch (InterruptedException e) {
120+
addUserException(new UserException("Failed to offer batch split by interrupted", e));
121+
}
107122
}
108123
}
109124
}
@@ -120,7 +135,7 @@ public Split getSampleSplit() {
120135
return sampleSplit;
121136
}
122137

123-
public void addToQueue(List<Split> splits) {
138+
public void addToQueue(List<Split> splits) throws UserException {
124139
if (splits.isEmpty()) {
125140
return;
126141
}
@@ -130,19 +145,9 @@ public void addToQueue(List<Split> splits) {
130145
sampleSplit = splits.get(0);
131146
assignLock.notify();
132147
}
133-
try {
134-
batch = backendPolicy.computeScanRangeAssignment(splits);
135-
} catch (UserException e) {
136-
exception = e;
137-
}
138-
}
139-
if (batch != null) {
140-
try {
141-
appendBatch(batch);
142-
} catch (UserException e) {
143-
exception = e;
144-
}
148+
batch = backendPolicy.computeScanRangeAssignment(splits);
145149
}
150+
appendBatch(batch);
146151
}
147152

148153
private void notifyAssignment() {
@@ -164,10 +169,18 @@ public BlockingQueue<Collection<TScanRangeLocations>> getAssignedSplits(Backend
164169
}
165170

166171
public void setException(UserException e) {
167-
exception = e;
172+
addUserException(e);
168173
notifyAssignment();
169174
}
170175

176+
private void addUserException(UserException e) {
177+
if (exception != null) {
178+
exception.addSuppressed(e);
179+
} else {
180+
exception = e;
181+
}
182+
}
183+
171184
public void finishSchedule() {
172185
scheduleFinished.set(true);
173186
notifyAssignment();
@@ -187,6 +200,9 @@ public void stop() {
187200
}
188201
});
189202
notifyAssignment();
203+
if (exception != null) {
204+
throw new RuntimeException(exception);
205+
}
190206
}
191207

192208
public boolean isStop() {

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -181,11 +181,11 @@ protected void initLocalObjectsImpl() {
181181
}
182182
HiveMetadataOps hiveOps = ExternalMetadataOperations.newHiveMetadataOps(hiveConf, jdbcClientConfig, this);
183183
threadPoolWithPreAuth = ThreadPoolManager.newDaemonFixedThreadPoolWithPreAuth(
184-
ICEBERG_CATALOG_EXECUTOR_THREAD_NUM,
185-
Integer.MAX_VALUE,
186-
String.format("hms_iceberg_catalog_%s_executor_pool", name),
187-
true,
188-
preExecutionAuthenticator);
184+
ICEBERG_CATALOG_EXECUTOR_THREAD_NUM,
185+
Integer.MAX_VALUE,
186+
String.format("hms_iceberg_catalog_%s_executor_pool", name),
187+
true,
188+
preExecutionAuthenticator);
189189
FileSystemProvider fileSystemProvider = new FileSystemProviderImpl(Env.getCurrentEnv().getExtMetaCacheMgr(),
190190
this.bindBrokerName(), this.catalogProperty.getHadoopProperties());
191191
this.fileSystemExecutor = ThreadPoolManager.newDaemonFixedThreadPool(FILE_SYSTEM_EXECUTOR_THREAD_NUM,

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -225,7 +225,9 @@ public void startSplit(int numBackends) {
225225
if (allFiles.size() > numSplitsPerPartition.get()) {
226226
numSplitsPerPartition.set(allFiles.size());
227227
}
228-
splitAssignment.addToQueue(allFiles);
228+
if (splitAssignment.needMoreSplit()) {
229+
splitAssignment.addToQueue(allFiles);
230+
}
229231
} catch (Exception e) {
230232
batchException.set(new UserException(e.getMessage(), e));
231233
} finally {

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@
7777
import java.util.concurrent.ConcurrentHashMap;
7878
import java.util.concurrent.CountDownLatch;
7979
import java.util.concurrent.Executor;
80+
import java.util.concurrent.ExecutorService;
8081
import java.util.concurrent.Semaphore;
8182
import java.util.concurrent.atomic.AtomicInteger;
8283
import java.util.concurrent.atomic.AtomicLong;
@@ -417,6 +418,7 @@ public void startSplit(int numBackends) {
417418
return;
418419
}
419420
AtomicInteger numFinishedPartitions = new AtomicInteger(0);
421+
ExecutorService scheduleExecutor = Env.getCurrentEnv().getExtMetaCacheMgr().getScheduleExecutor();
420422
CompletableFuture.runAsync(() -> {
421423
for (HivePartition partition : prunedPartitions) {
422424
if (batchException.get() != null || splitAssignment.isStop()) {
@@ -435,8 +437,10 @@ public void startSplit(int numBackends) {
435437
if (allFiles.size() > numSplitsPerPartition.get()) {
436438
numSplitsPerPartition.set(allFiles.size());
437439
}
438-
splitAssignment.addToQueue(allFiles);
439-
} catch (IOException e) {
440+
if (splitAssignment.needMoreSplit()) {
441+
splitAssignment.addToQueue(allFiles);
442+
}
443+
} catch (Exception e) {
440444
batchException.set(new UserException(e.getMessage(), e));
441445
} finally {
442446
splittersOnFlight.release();
@@ -447,12 +451,12 @@ public void startSplit(int numBackends) {
447451
splitAssignment.finishSchedule();
448452
}
449453
}
450-
});
454+
}, scheduleExecutor);
451455
}
452456
if (batchException.get() != null) {
453457
splitAssignment.setException(batchException.get());
454458
}
455-
});
459+
}, scheduleExecutor);
456460
}
457461

458462
@Override

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java‎

Lines changed: 22 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.apache.doris.analysis.TableSnapshot;
2323
import org.apache.doris.analysis.TupleDescriptor;
2424
import org.apache.doris.catalog.Column;
25+
import org.apache.doris.catalog.Env;
2526
import org.apache.doris.catalog.TableIf;
2627
import org.apache.doris.common.DdlException;
2728
import org.apache.doris.common.UserException;
@@ -78,6 +79,7 @@
7879
import java.util.OptionalLong;
7980
import java.util.concurrent.CompletableFuture;
8081
import java.util.concurrent.ConcurrentHashMap;
82+
import java.util.concurrent.atomic.AtomicReference;
8183

8284
public class IcebergScanNode extends FileQueryScanNode {
8385

@@ -229,19 +231,21 @@ public void startSplit(int numBackends) throws UserException {
229231
public void doStartSplit() {
230232
TableScan scan = createTableScan();
231233
CompletableFuture.runAsync(() -> {
234+
AtomicReference<CloseableIterable<FileScanTask>> taskRef = new AtomicReference<>();
232235
try {
233236
preExecutionAuthenticator.execute(
234237
() -> {
235238
CloseableIterable<FileScanTask> fileScanTasks = planFileScanTask(scan);
236-
237-
// 1. this task should stop when all splits are assigned
238-
// 2. if we want to stop this plan, we can close the fileScanTasks to stop
239-
splitAssignment.addCloseable(fileScanTasks);
240-
241-
fileScanTasks.forEach(fileScanTask ->
242-
splitAssignment.addToQueue(Lists.newArrayList(createIcebergSplit(fileScanTask))));
243-
244-
return null;
239+
taskRef.set(fileScanTasks);
240+
241+
CloseableIterator<FileScanTask> iterator = fileScanTasks.iterator();
242+
while (splitAssignment.needMoreSplit() && iterator.hasNext()) {
243+
try {
244+
splitAssignment.addToQueue(Lists.newArrayList(createIcebergSplit(iterator.next())));
245+
} catch (UserException e) {
246+
throw new RuntimeException(e);
247+
}
248+
}
245249
}
246250
);
247251
splitAssignment.finishSchedule();
@@ -252,8 +256,16 @@ public void doStartSplit() {
252256
} else {
253257
splitAssignment.setException(new UserException(e.getMessage(), e));
254258
}
259+
} finally {
260+
if (taskRef.get() != null) {
261+
try {
262+
taskRef.get().close();
263+
} catch (IOException e) {
264+
// ignore
265+
}
266+
}
255267
}
256-
});
268+
}, Env.getCurrentEnv().getExtMetaCacheMgr().getScheduleExecutor());
257269
}
258270

259271
@VisibleForTesting

‎fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -267,8 +267,10 @@ public void startSplit(int numBackends) {
267267
createTableBatchReadSession(requiredBatchPartitionSpecs);
268268
List<Split> batchSplit = getSplitByTableSession(tableBatchReadSession);
269269

270-
splitAssignment.addToQueue(batchSplit);
271-
} catch (IOException e) {
270+
if (splitAssignment.needMoreSplit()) {
271+
splitAssignment.addToQueue(batchSplit);
272+
}
273+
} catch (Exception e) {
272274
batchException.set(new UserException(e.getMessage(), e));
273275
} finally {
274276
if (batchException.get() != null) {
@@ -288,7 +290,7 @@ public void startSplit(int numBackends) {
288290
splitAssignment.setException(batchException.get());
289291
}
290292
}
291-
});
293+
}, scheduleExecutor);
292294
}
293295

294296
@Override

0 commit comments

Comments
 (0)