Skip to content

Commit 4e7a1c6

Browse files
wuwenchihubgeter
authored andcommitted
[fix](iceberg) Fix some issues in batch mode (apache#51185)
### What problem does this PR solve? Problem Summary: 1. When adding splits to the queue, a retry mechanism is added to prevent blocking and sticking. 2. We use a separate thread to fetch splits. When the SQL execution ends early (such as `limit`), this thread needs to be handled. Now it is exited by throwing an exception. Moreover, this thread should use an independent thread pool to prevent system freezes caused by resource contention. 3. When there are multiple exceptions, overwriting would occur previously. Now we use `addSuppressed` to prevent exception loss. 4. When initializing the split assignment, a timeout mechanism is added to prevent the situation where the system gets stuck due to failing to obtain splits. 5. When the split assignment stop, if an exception is not null, an exception should be thrown externally. (cherry picked from commit f6d6041)
1 parent 8e86fe6 commit 4e7a1c6

8 files changed

Lines changed: 508 additions & 44 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ public class ExternalMetaCacheMgr {
8484
private ExecutorService rowCountRefreshExecutor;
8585
private ExecutorService commonRefreshExecutor;
8686
private ExecutorService fileListingExecutor;
87+
// This executor is used to schedule the getting split tasks
8788
private ExecutorService scheduleExecutor;
8889

8990
// catalog id -> HiveMetaStoreCache

‎fe/fe-core/src/main/java/org/apache/doris/datasource/SplitAssignment.java‎

Lines changed: 37 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import java.util.concurrent.BlockingQueue;
3737
import java.util.concurrent.ConcurrentHashMap;
3838
import java.util.concurrent.LinkedBlockingQueue;
39+
import java.util.concurrent.TimeUnit;
3940
import java.util.concurrent.atomic.AtomicBoolean;
4041

4142
/**
@@ -76,20 +77,28 @@ public SplitAssignment(
7677
public void init() throws UserException {
7778
splitGenerator.startSplit(backendPolicy.numBackends());
7879
synchronized (assignLock) {
79-
while (sampleSplit == null && waitFirstSplit()) {
80+
final int waitIntervalTimeMillis = 100;
81+
final int initTimeoutMillis = 30000; // 30s
82+
int waitTotalTime = 0;
83+
while (sampleSplit == null && needMoreSplit()) {
8084
try {
81-
assignLock.wait(100);
85+
assignLock.wait(waitIntervalTimeMillis);
8286
} catch (InterruptedException e) {
8387
throw new UserException(e.getMessage(), e);
8488
}
89+
waitTotalTime += waitIntervalTimeMillis;
90+
if (waitTotalTime > initTimeoutMillis) {
91+
throw new UserException("Failed to get first split after waiting for "
92+
+ (waitTotalTime / 1000) + " seconds.");
93+
}
8594
}
8695
}
8796
if (exception != null) {
8897
throw exception;
8998
}
9099
}
91100

92-
private boolean waitFirstSplit() {
101+
public boolean needMoreSplit() {
93102
return !scheduleFinished.get() && !isStopped.get() && exception == null;
94103
}
95104

@@ -100,10 +109,16 @@ private void appendBatch(Multimap<Backend, Split> batch) throws UserException {
100109
for (Split split : splits) {
101110
locations.add(splitToScanRange.getScanRange(backend, locationProperties, split, pathPartitionKeys));
102111
}
103-
try {
104-
assignment.computeIfAbsent(backend, be -> new LinkedBlockingQueue<>(10000)).put(locations);
105-
} catch (Exception e) {
106-
throw new UserException("Failed to offer batch split", e);
112+
while (needMoreSplit()) {
113+
BlockingQueue<Collection<TScanRangeLocations>> queue =
114+
assignment.computeIfAbsent(backend, be -> new LinkedBlockingQueue<>(10000));
115+
try {
116+
if (queue.offer(locations, 100, TimeUnit.MILLISECONDS)) {
117+
return;
118+
}
119+
} catch (InterruptedException e) {
120+
addUserException(new UserException("Failed to offer batch split by interrupted", e));
121+
}
107122
}
108123
}
109124
}
@@ -120,7 +135,7 @@ public Split getSampleSplit() {
120135
return sampleSplit;
121136
}
122137

123-
public void addToQueue(List<Split> splits) {
138+
public void addToQueue(List<Split> splits) throws UserException {
124139
if (splits.isEmpty()) {
125140
return;
126141
}
@@ -130,19 +145,9 @@ public void addToQueue(List<Split> splits) {
130145
sampleSplit = splits.get(0);
131146
assignLock.notify();
132147
}
133-
try {
134-
batch = backendPolicy.computeScanRangeAssignment(splits);
135-
} catch (UserException e) {
136-
exception = e;
137-
}
138-
}
139-
if (batch != null) {
140-
try {
141-
appendBatch(batch);
142-
} catch (UserException e) {
143-
exception = e;
144-
}
148+
batch = backendPolicy.computeScanRangeAssignment(splits);
145149
}
150+
appendBatch(batch);
146151
}
147152

148153
private void notifyAssignment() {
@@ -164,10 +169,18 @@ public BlockingQueue<Collection<TScanRangeLocations>> getAssignedSplits(Backend
164169
}
165170

166171
public void setException(UserException e) {
167-
exception = e;
172+
addUserException(e);
168173
notifyAssignment();
169174
}
170175

176+
private void addUserException(UserException e) {
177+
if (exception != null) {
178+
exception.addSuppressed(e);
179+
} else {
180+
exception = e;
181+
}
182+
}
183+
171184
public void finishSchedule() {
172185
scheduleFinished.set(true);
173186
notifyAssignment();
@@ -187,6 +200,9 @@ public void stop() {
187200
}
188201
});
189202
notifyAssignment();
203+
if (exception != null) {
204+
throw new RuntimeException(exception);
205+
}
190206
}
191207

192208
public boolean isStop() {

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -181,11 +181,11 @@ protected void initLocalObjectsImpl() {
181181
}
182182
HiveMetadataOps hiveOps = ExternalMetadataOperations.newHiveMetadataOps(hiveConf, jdbcClientConfig, this);
183183
threadPoolWithPreAuth = ThreadPoolManager.newDaemonFixedThreadPoolWithPreAuth(
184-
ICEBERG_CATALOG_EXECUTOR_THREAD_NUM,
185-
Integer.MAX_VALUE,
186-
String.format("hms_iceberg_catalog_%s_executor_pool", name),
187-
true,
188-
preExecutionAuthenticator);
184+
ICEBERG_CATALOG_EXECUTOR_THREAD_NUM,
185+
Integer.MAX_VALUE,
186+
String.format("hms_iceberg_catalog_%s_executor_pool", name),
187+
true,
188+
preExecutionAuthenticator);
189189
FileSystemProvider fileSystemProvider = new FileSystemProviderImpl(Env.getCurrentEnv().getExtMetaCacheMgr(),
190190
this.bindBrokerName(), this.catalogProperty.getHadoopProperties());
191191
this.fileSystemExecutor = ThreadPoolManager.newDaemonFixedThreadPool(FILE_SYSTEM_EXECUTOR_THREAD_NUM,

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -233,7 +233,9 @@ public void startSplit(int numBackends) {
233233
if (allFiles.size() > numSplitsPerPartition.get()) {
234234
numSplitsPerPartition.set(allFiles.size());
235235
}
236-
splitAssignment.addToQueue(allFiles);
236+
if (splitAssignment.needMoreSplit()) {
237+
splitAssignment.addToQueue(allFiles);
238+
}
237239
} catch (Exception e) {
238240
batchException.set(new UserException(e.getMessage(), e));
239241
} finally {

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@
7878
import java.util.concurrent.ConcurrentHashMap;
7979
import java.util.concurrent.CountDownLatch;
8080
import java.util.concurrent.Executor;
81+
import java.util.concurrent.ExecutorService;
8182
import java.util.concurrent.Semaphore;
8283
import java.util.concurrent.atomic.AtomicInteger;
8384
import java.util.concurrent.atomic.AtomicLong;
@@ -430,6 +431,7 @@ public void startSplit(int numBackends) {
430431
return;
431432
}
432433
AtomicInteger numFinishedPartitions = new AtomicInteger(0);
434+
ExecutorService scheduleExecutor = Env.getCurrentEnv().getExtMetaCacheMgr().getScheduleExecutor();
433435
CompletableFuture.runAsync(() -> {
434436
for (HivePartition partition : prunedPartitions) {
435437
if (batchException.get() != null || splitAssignment.isStop()) {
@@ -448,8 +450,10 @@ public void startSplit(int numBackends) {
448450
if (allFiles.size() > numSplitsPerPartition.get()) {
449451
numSplitsPerPartition.set(allFiles.size());
450452
}
451-
splitAssignment.addToQueue(allFiles);
452-
} catch (IOException e) {
453+
if (splitAssignment.needMoreSplit()) {
454+
splitAssignment.addToQueue(allFiles);
455+
}
456+
} catch (Exception e) {
453457
batchException.set(new UserException(e.getMessage(), e));
454458
} finally {
455459
splittersOnFlight.release();
@@ -460,12 +464,12 @@ public void startSplit(int numBackends) {
460464
splitAssignment.finishSchedule();
461465
}
462466
}
463-
});
467+
}, scheduleExecutor);
464468
}
465469
if (batchException.get() != null) {
466470
splitAssignment.setException(batchException.get());
467471
}
468-
});
472+
}, scheduleExecutor);
469473
}
470474

471475
@Override

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java‎

Lines changed: 22 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.apache.doris.analysis.TableSnapshot;
2323
import org.apache.doris.analysis.TupleDescriptor;
2424
import org.apache.doris.catalog.Column;
25+
import org.apache.doris.catalog.Env;
2526
import org.apache.doris.catalog.TableIf;
2627
import org.apache.doris.common.DdlException;
2728
import org.apache.doris.common.UserException;
@@ -79,6 +80,7 @@
7980
import java.util.OptionalLong;
8081
import java.util.concurrent.CompletableFuture;
8182
import java.util.concurrent.ConcurrentHashMap;
83+
import java.util.concurrent.atomic.AtomicReference;
8284

8385
public class IcebergScanNode extends FileQueryScanNode {
8486

@@ -231,19 +233,21 @@ public void startSplit(int numBackends) throws UserException {
231233
public void doStartSplit() {
232234
TableScan scan = createTableScan();
233235
CompletableFuture.runAsync(() -> {
236+
AtomicReference<CloseableIterable<FileScanTask>> taskRef = new AtomicReference<>();
234237
try {
235238
preExecutionAuthenticator.execute(
236239
() -> {
237240
CloseableIterable<FileScanTask> fileScanTasks = planFileScanTask(scan);
238-
239-
// 1. this task should stop when all splits are assigned
240-
// 2. if we want to stop this plan, we can close the fileScanTasks to stop
241-
splitAssignment.addCloseable(fileScanTasks);
242-
243-
fileScanTasks.forEach(fileScanTask ->
244-
splitAssignment.addToQueue(Lists.newArrayList(createIcebergSplit(fileScanTask))));
245-
246-
return null;
241+
taskRef.set(fileScanTasks);
242+
243+
CloseableIterator<FileScanTask> iterator = fileScanTasks.iterator();
244+
while (splitAssignment.needMoreSplit() && iterator.hasNext()) {
245+
try {
246+
splitAssignment.addToQueue(Lists.newArrayList(createIcebergSplit(iterator.next())));
247+
} catch (UserException e) {
248+
throw new RuntimeException(e);
249+
}
250+
}
247251
}
248252
);
249253
splitAssignment.finishSchedule();
@@ -254,8 +258,16 @@ public void doStartSplit() {
254258
} else {
255259
splitAssignment.setException(new UserException(e.getMessage(), e));
256260
}
261+
} finally {
262+
if (taskRef.get() != null) {
263+
try {
264+
taskRef.get().close();
265+
} catch (IOException e) {
266+
// ignore
267+
}
268+
}
257269
}
258-
});
270+
}, Env.getCurrentEnv().getExtMetaCacheMgr().getScheduleExecutor());
259271
}
260272

261273
@VisibleForTesting

‎fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -267,8 +267,10 @@ public void startSplit(int numBackends) {
267267
createTableBatchReadSession(requiredBatchPartitionSpecs);
268268
List<Split> batchSplit = getSplitByTableSession(tableBatchReadSession);
269269

270-
splitAssignment.addToQueue(batchSplit);
271-
} catch (IOException e) {
270+
if (splitAssignment.needMoreSplit()) {
271+
splitAssignment.addToQueue(batchSplit);
272+
}
273+
} catch (Exception e) {
272274
batchException.set(new UserException(e.getMessage(), e));
273275
} finally {
274276
if (batchException.get() != null) {
@@ -288,7 +290,7 @@ public void startSplit(int numBackends) {
288290
splitAssignment.setException(batchException.get());
289291
}
290292
}
291-
});
293+
}, scheduleExecutor);
292294
}
293295

294296
@Override

0 commit comments

Comments
 (0)