Skip to content

Commit fb1ef62

Browse files
committed
Revert "[fix](iceberg) Fix some issues in batch mode (apache#51185)"
This reverts commit 4e7a1c6.
1 parent 4e7a1c6 commit fb1ef62

8 files changed

Lines changed: 44 additions & 508 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,6 @@ public class ExternalMetaCacheMgr {
8484
private ExecutorService rowCountRefreshExecutor;
8585
private ExecutorService commonRefreshExecutor;
8686
private ExecutorService fileListingExecutor;
87-
// This executor is used to schedule the getting split tasks
8887
private ExecutorService scheduleExecutor;
8988

9089
// catalog id -> HiveMetaStoreCache

‎fe/fe-core/src/main/java/org/apache/doris/datasource/SplitAssignment.java‎

Lines changed: 21 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@
3636
import java.util.concurrent.BlockingQueue;
3737
import java.util.concurrent.ConcurrentHashMap;
3838
import java.util.concurrent.LinkedBlockingQueue;
39-
import java.util.concurrent.TimeUnit;
4039
import java.util.concurrent.atomic.AtomicBoolean;
4140

4241
/**
@@ -77,28 +76,20 @@ public SplitAssignment(
7776
public void init() throws UserException {
7877
splitGenerator.startSplit(backendPolicy.numBackends());
7978
synchronized (assignLock) {
80-
final int waitIntervalTimeMillis = 100;
81-
final int initTimeoutMillis = 30000; // 30s
82-
int waitTotalTime = 0;
83-
while (sampleSplit == null && needMoreSplit()) {
79+
while (sampleSplit == null && waitFirstSplit()) {
8480
try {
85-
assignLock.wait(waitIntervalTimeMillis);
81+
assignLock.wait(100);
8682
} catch (InterruptedException e) {
8783
throw new UserException(e.getMessage(), e);
8884
}
89-
waitTotalTime += waitIntervalTimeMillis;
90-
if (waitTotalTime > initTimeoutMillis) {
91-
throw new UserException("Failed to get first split after waiting for "
92-
+ (waitTotalTime / 1000) + " seconds.");
93-
}
9485
}
9586
}
9687
if (exception != null) {
9788
throw exception;
9889
}
9990
}
10091

101-
public boolean needMoreSplit() {
92+
private boolean waitFirstSplit() {
10293
return !scheduleFinished.get() && !isStopped.get() && exception == null;
10394
}
10495

@@ -109,16 +100,10 @@ private void appendBatch(Multimap<Backend, Split> batch) throws UserException {
109100
for (Split split : splits) {
110101
locations.add(splitToScanRange.getScanRange(backend, locationProperties, split, pathPartitionKeys));
111102
}
112-
while (needMoreSplit()) {
113-
BlockingQueue<Collection<TScanRangeLocations>> queue =
114-
assignment.computeIfAbsent(backend, be -> new LinkedBlockingQueue<>(10000));
115-
try {
116-
if (queue.offer(locations, 100, TimeUnit.MILLISECONDS)) {
117-
return;
118-
}
119-
} catch (InterruptedException e) {
120-
addUserException(new UserException("Failed to offer batch split by interrupted", e));
121-
}
103+
try {
104+
assignment.computeIfAbsent(backend, be -> new LinkedBlockingQueue<>(10000)).put(locations);
105+
} catch (Exception e) {
106+
throw new UserException("Failed to offer batch split", e);
122107
}
123108
}
124109
}
@@ -135,7 +120,7 @@ public Split getSampleSplit() {
135120
return sampleSplit;
136121
}
137122

138-
public void addToQueue(List<Split> splits) throws UserException {
123+
public void addToQueue(List<Split> splits) {
139124
if (splits.isEmpty()) {
140125
return;
141126
}
@@ -145,9 +130,19 @@ public void addToQueue(List<Split> splits) throws UserException {
145130
sampleSplit = splits.get(0);
146131
assignLock.notify();
147132
}
148-
batch = backendPolicy.computeScanRangeAssignment(splits);
133+
try {
134+
batch = backendPolicy.computeScanRangeAssignment(splits);
135+
} catch (UserException e) {
136+
exception = e;
137+
}
138+
}
139+
if (batch != null) {
140+
try {
141+
appendBatch(batch);
142+
} catch (UserException e) {
143+
exception = e;
144+
}
149145
}
150-
appendBatch(batch);
151146
}
152147

153148
private void notifyAssignment() {
@@ -169,18 +164,10 @@ public BlockingQueue<Collection<TScanRangeLocations>> getAssignedSplits(Backend
169164
}
170165

171166
public void setException(UserException e) {
172-
addUserException(e);
167+
exception = e;
173168
notifyAssignment();
174169
}
175170

176-
private void addUserException(UserException e) {
177-
if (exception != null) {
178-
exception.addSuppressed(e);
179-
} else {
180-
exception = e;
181-
}
182-
}
183-
184171
public void finishSchedule() {
185172
scheduleFinished.set(true);
186173
notifyAssignment();
@@ -200,9 +187,6 @@ public void stop() {
200187
}
201188
});
202189
notifyAssignment();
203-
if (exception != null) {
204-
throw new RuntimeException(exception);
205-
}
206190
}
207191

208192
public boolean isStop() {

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -181,11 +181,11 @@ protected void initLocalObjectsImpl() {
181181
}
182182
HiveMetadataOps hiveOps = ExternalMetadataOperations.newHiveMetadataOps(hiveConf, jdbcClientConfig, this);
183183
threadPoolWithPreAuth = ThreadPoolManager.newDaemonFixedThreadPoolWithPreAuth(
184-
ICEBERG_CATALOG_EXECUTOR_THREAD_NUM,
185-
Integer.MAX_VALUE,
186-
String.format("hms_iceberg_catalog_%s_executor_pool", name),
187-
true,
188-
preExecutionAuthenticator);
184+
ICEBERG_CATALOG_EXECUTOR_THREAD_NUM,
185+
Integer.MAX_VALUE,
186+
String.format("hms_iceberg_catalog_%s_executor_pool", name),
187+
true,
188+
preExecutionAuthenticator);
189189
FileSystemProvider fileSystemProvider = new FileSystemProviderImpl(Env.getCurrentEnv().getExtMetaCacheMgr(),
190190
this.bindBrokerName(), this.catalogProperty.getHadoopProperties());
191191
this.fileSystemExecutor = ThreadPoolManager.newDaemonFixedThreadPool(FILE_SYSTEM_EXECUTOR_THREAD_NUM,

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -233,9 +233,7 @@ public void startSplit(int numBackends) {
233233
if (allFiles.size() > numSplitsPerPartition.get()) {
234234
numSplitsPerPartition.set(allFiles.size());
235235
}
236-
if (splitAssignment.needMoreSplit()) {
237-
splitAssignment.addToQueue(allFiles);
238-
}
236+
splitAssignment.addToQueue(allFiles);
239237
} catch (Exception e) {
240238
batchException.set(new UserException(e.getMessage(), e));
241239
} finally {

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,6 @@
7878
import java.util.concurrent.ConcurrentHashMap;
7979
import java.util.concurrent.CountDownLatch;
8080
import java.util.concurrent.Executor;
81-
import java.util.concurrent.ExecutorService;
8281
import java.util.concurrent.Semaphore;
8382
import java.util.concurrent.atomic.AtomicInteger;
8483
import java.util.concurrent.atomic.AtomicLong;
@@ -431,7 +430,6 @@ public void startSplit(int numBackends) {
431430
return;
432431
}
433432
AtomicInteger numFinishedPartitions = new AtomicInteger(0);
434-
ExecutorService scheduleExecutor = Env.getCurrentEnv().getExtMetaCacheMgr().getScheduleExecutor();
435433
CompletableFuture.runAsync(() -> {
436434
for (HivePartition partition : prunedPartitions) {
437435
if (batchException.get() != null || splitAssignment.isStop()) {
@@ -450,10 +448,8 @@ public void startSplit(int numBackends) {
450448
if (allFiles.size() > numSplitsPerPartition.get()) {
451449
numSplitsPerPartition.set(allFiles.size());
452450
}
453-
if (splitAssignment.needMoreSplit()) {
454-
splitAssignment.addToQueue(allFiles);
455-
}
456-
} catch (Exception e) {
451+
splitAssignment.addToQueue(allFiles);
452+
} catch (IOException e) {
457453
batchException.set(new UserException(e.getMessage(), e));
458454
} finally {
459455
splittersOnFlight.release();
@@ -464,12 +460,12 @@ public void startSplit(int numBackends) {
464460
splitAssignment.finishSchedule();
465461
}
466462
}
467-
}, scheduleExecutor);
463+
});
468464
}
469465
if (batchException.get() != null) {
470466
splitAssignment.setException(batchException.get());
471467
}
472-
}, scheduleExecutor);
468+
});
473469
}
474470

475471
@Override

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java‎

Lines changed: 10 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
import org.apache.doris.analysis.TableSnapshot;
2323
import org.apache.doris.analysis.TupleDescriptor;
2424
import org.apache.doris.catalog.Column;
25-
import org.apache.doris.catalog.Env;
2625
import org.apache.doris.catalog.TableIf;
2726
import org.apache.doris.common.DdlException;
2827
import org.apache.doris.common.UserException;
@@ -80,7 +79,6 @@
8079
import java.util.OptionalLong;
8180
import java.util.concurrent.CompletableFuture;
8281
import java.util.concurrent.ConcurrentHashMap;
83-
import java.util.concurrent.atomic.AtomicReference;
8482

8583
public class IcebergScanNode extends FileQueryScanNode {
8684

@@ -233,21 +231,19 @@ public void startSplit(int numBackends) throws UserException {
233231
public void doStartSplit() {
234232
TableScan scan = createTableScan();
235233
CompletableFuture.runAsync(() -> {
236-
AtomicReference<CloseableIterable<FileScanTask>> taskRef = new AtomicReference<>();
237234
try {
238235
preExecutionAuthenticator.execute(
239236
() -> {
240237
CloseableIterable<FileScanTask> fileScanTasks = planFileScanTask(scan);
241-
taskRef.set(fileScanTasks);
242-
243-
CloseableIterator<FileScanTask> iterator = fileScanTasks.iterator();
244-
while (splitAssignment.needMoreSplit() && iterator.hasNext()) {
245-
try {
246-
splitAssignment.addToQueue(Lists.newArrayList(createIcebergSplit(iterator.next())));
247-
} catch (UserException e) {
248-
throw new RuntimeException(e);
249-
}
250-
}
238+
239+
// 1. this task should stop when all splits are assigned
240+
// 2. if we want to stop this plan, we can close the fileScanTasks to stop
241+
splitAssignment.addCloseable(fileScanTasks);
242+
243+
fileScanTasks.forEach(fileScanTask ->
244+
splitAssignment.addToQueue(Lists.newArrayList(createIcebergSplit(fileScanTask))));
245+
246+
return null;
251247
}
252248
);
253249
splitAssignment.finishSchedule();
@@ -258,16 +254,8 @@ public void doStartSplit() {
258254
} else {
259255
splitAssignment.setException(new UserException(e.getMessage(), e));
260256
}
261-
} finally {
262-
if (taskRef.get() != null) {
263-
try {
264-
taskRef.get().close();
265-
} catch (IOException e) {
266-
// ignore
267-
}
268-
}
269257
}
270-
}, Env.getCurrentEnv().getExtMetaCacheMgr().getScheduleExecutor());
258+
});
271259
}
272260

273261
@VisibleForTesting

‎fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -267,10 +267,8 @@ public void startSplit(int numBackends) {
267267
createTableBatchReadSession(requiredBatchPartitionSpecs);
268268
List<Split> batchSplit = getSplitByTableSession(tableBatchReadSession);
269269

270-
if (splitAssignment.needMoreSplit()) {
271-
splitAssignment.addToQueue(batchSplit);
272-
}
273-
} catch (Exception e) {
270+
splitAssignment.addToQueue(batchSplit);
271+
} catch (IOException e) {
274272
batchException.set(new UserException(e.getMessage(), e));
275273
} finally {
276274
if (batchException.get() != null) {
@@ -290,7 +288,7 @@ public void startSplit(int numBackends) {
290288
splitAssignment.setException(batchException.get());
291289
}
292290
}
293-
}, scheduleExecutor);
291+
});
294292
}
295293

296294
@Override

0 commit comments

Comments
 (0)