Skip to content

Commit 7e8a562

Browse files
committed
[fix](fe) Preserve streaming scan ownership and count split policy
### What problem does this PR solve? Issue Number: N/A Related PR: #69298, #69986 Problem Summary: Cancelled Iceberg planning can retain an independent table-generation lease when task-group lookup throws before producer cleanup is registered. Resolve that executor before acquiring the lease. Hive and Iceberg also eagerly allocate a configured maximum batch capacity, so a large valid setting can fail with OOM even for empty work; share a bounded initial allocation while preserving the configured batch maximum. Hive streaming COUNT(*) decides whether to split one partition at a time, producing excessive ranges and footer reads when the selected scan already has enough files. Use the existing concurrent cancellable producers for bounded metadata lookahead, make the scan-wide decision after all lookahead workers finish, and reuse captured metadata before lazily listing further partitions. Preserve singleton listing, fair task admission, interruption, range backpressure and the eager COUNT policy. ### Release note Avoid excessive Hive COUNT(*) scan ranges, premature memory allocation for large split batch settings, and leaked Iceberg planning references after cancellation. ### Check List (For Author) - Test: Unit tests and packaging - Reproduced the Iceberg lease leak and Hive COUNT range regression before repair; isolated the allocation failure in a separate 64MB JDK process. - All 218 selected FE tests passed with no failures, errors or skips, using run-fe-ut.sh. - Covered both scanner versions, Parquet/ORC, metadata reuse without file caching, few-file splitting, maximum batch settings, metadata failure and lookahead cancellation. - Full FE packaging and Checkstyle passed using build.sh --fe -j4; git diff --check passed. - Complete P0 and external-service regression were not run locally in this cycle. - Behavior changed: Yes. Streaming COUNT(*) uses the selected scan's split decision, large batch limits allocate only for actual work, and failed admission acquires no independent generation reference. Scheduling and cancellation continue through existing producers. - Does this need documentation: No
1 parent 663c74d commit 7e8a562

5 files changed

Lines changed: 306 additions & 21 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/datasource/SplitProducer.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525

2626
import java.io.Closeable;
2727
import java.io.IOException;
28+
import java.util.ArrayList;
2829
import java.util.Collection;
2930
import java.util.Collections;
3031
import java.util.Iterator;
@@ -38,6 +39,12 @@
3839
* full. Each producer retains at most one assigned batch and resumes when a consumer frees space.
3940
*/
4041
public class SplitProducer implements Runnable, Closeable {
42+
private static final int INITIAL_BATCH_CAPACITY = 128;
43+
44+
/** Allocate for actual work without eagerly reserving an arbitrarily large configured maximum. */
45+
public static List<Split> newBatch(int batchSize) {
46+
return new ArrayList<>(Math.min(batchSize, INITIAL_BATCH_CAPACITY));
47+
}
4148

4249
public interface BatchSource extends Closeable {
4350
List<Split> getNextBatch() throws Exception;

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java‎

Lines changed: 103 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -87,8 +87,10 @@
8787
import java.util.Map;
8888
import java.util.Objects;
8989
import java.util.Random;
90+
import java.util.concurrent.ConcurrentLinkedQueue;
9091
import java.util.concurrent.Executor;
9192
import java.util.concurrent.atomic.AtomicInteger;
93+
import java.util.concurrent.atomic.AtomicLong;
9294
import java.util.stream.Collectors;
9395

9496
public class HiveScanNode extends FileQueryScanNode {
@@ -248,7 +250,7 @@ protected SplitAssignment createSplitAssignment(Map<String, String> properties,
248250
}
249251

250252
@Override
251-
public void startSplit(int numBackends) {
253+
public void startSplit(int numBackends) throws UserException {
252254
ConnectContext context = ConnectContext.get();
253255
StatementContext statement = context == null ? null : context.getStatementContext();
254256
if (statement != null) {
@@ -274,17 +276,91 @@ public void startSplit(int numBackends) {
274276
int producers = parallelListing
275277
? Math.min(prunedPartitions.size(), Config.max_external_cache_loader_thread_pool_size) : 1;
276278
AtomicInteger nextPartition = new AtomicInteger();
277-
AtomicInteger remaining = new AtomicInteger(producers);
278279
long started = System.currentTimeMillis();
280+
if (!usesCountFileSplitPolicy()) {
281+
startSplitProducers(cache, executor, broker, numBackends, producers, nextPartition, null, started);
282+
return;
283+
}
284+
int parallelism = sessionVariable.getParallelExecInstanceNum(scanContext.getClusterName());
285+
CountSplitPlan countPlan = new CountSplitPlan();
286+
splitAssignment.addCloseable(countPlan);
287+
AtomicInteger remaining = new AtomicInteger(producers);
288+
// The count policy needs only a scan-wide lower bound. Reuse the same cancellable workers
289+
// and singleton partition loads, then publish one decision before starting range producers.
290+
for (int i = 0; i < producers; i++) {
291+
new SplitProducer(splitAssignment, executor, () -> {
292+
prefetchCountFiles(countPlan, cache, broker, parallelism, numBackends, nextPartition);
293+
return Collections.emptyList();
294+
}, failure -> {
295+
if (failure != null) {
296+
splitAssignment.setException(createSplitPlanningException(failure));
297+
}
298+
if (remaining.decrementAndGet() == 0) {
299+
if (splitAssignment.needMoreSplit()) {
300+
countPlan.needSplit = FileSplitter.needSplitForCountPushdown(
301+
parallelism, numBackends, countPlan.fileCount.get());
302+
startSplitProducers(cache, executor, broker, numBackends,
303+
producers, nextPartition, countPlan, started);
304+
} else {
305+
closeCountSplitPlan(countPlan);
306+
splitAssignment.finishSchedule();
307+
}
308+
}
309+
}).start();
310+
}
311+
}
312+
313+
private void prefetchCountFiles(CountSplitPlan plan, HiveExternalMetaCache cache, String broker,
314+
int parallelism, int numBackends, AtomicInteger nextPartition)
315+
throws IOException, UserException {
316+
while (splitAssignment.needMoreSplit()
317+
&& FileSplitter.needSplitForCountPushdown(parallelism, numBackends, plan.fileCount.get())) {
318+
int index = nextPartition.getAndIncrement();
319+
if (index >= prunedPartitions.size()) {
320+
return;
321+
}
322+
List<FileCacheValue> loaded = getFileCachesByPartitions(cache,
323+
Collections.singletonList(prunedPartitions.get(index)), broker, true);
324+
for (FileCacheValue value : loaded) {
325+
if (value.getFiles() != null && !value.getFiles().isEmpty()) {
326+
plan.prefetched.add(value);
327+
plan.fileCount.addAndGet(value.getFiles().size());
328+
}
329+
}
330+
}
331+
}
332+
333+
private static class CountSplitPlan implements Closeable {
334+
private final ConcurrentLinkedQueue<FileCacheValue> prefetched = new ConcurrentLinkedQueue<>();
335+
private final AtomicLong fileCount = new AtomicLong();
336+
private boolean needSplit;
337+
338+
@Override
339+
public void close() {
340+
prefetched.clear();
341+
}
342+
}
343+
344+
private void closeCountSplitPlan(CountSplitPlan plan) {
345+
splitAssignment.removeCloseable(plan);
346+
plan.close();
347+
}
348+
349+
private void startSplitProducers(HiveExternalMetaCache cache, Executor executor, String broker,
350+
int numBackends, int producers, AtomicInteger nextPartition, CountSplitPlan countPlan, long started) {
351+
AtomicInteger remaining = new AtomicInteger(producers);
279352
for (int i = 0; i < producers; i++) {
280353
new SplitProducer(splitAssignment, executor,
281-
createSplitBatchSource(cache, nextPartition, broker, numBackends), failure -> {
354+
createSplitBatchSource(cache, nextPartition, broker, numBackends, countPlan), failure -> {
282355
try {
283356
if (failure != null) {
284357
splitAssignment.setException(createSplitPlanningException(failure));
285358
}
286359
} finally {
287360
if (remaining.decrementAndGet() == 0) {
361+
if (countPlan != null) {
362+
closeCountSplitPlan(countPlan);
363+
}
288364
if (getSummaryProfile() != null) {
289365
getSummaryProfile().addExternalTableGetPartitionFilesTime(
290366
System.currentTimeMillis() - started);
@@ -298,6 +374,11 @@ public void startSplit(int numBackends) {
298374

299375
SplitProducer.BatchSource createSplitBatchSource(HiveExternalMetaCache cache, AtomicInteger nextPartition,
300376
String broker, int numBackends) {
377+
return createSplitBatchSource(cache, nextPartition, broker, numBackends, null);
378+
}
379+
380+
private SplitProducer.BatchSource createSplitBatchSource(HiveExternalMetaCache cache, AtomicInteger nextPartition,
381+
String broker, int numBackends, CountSplitPlan countPlan) {
301382
return new SplitProducer.BatchSource() {
302383
private Iterator<FileCacheValue> fileCaches = Collections.emptyIterator();
303384
private Iterator<HiveExternalMetaCache.HiveFileStatus> files = Collections.emptyIterator();
@@ -309,7 +390,7 @@ SplitProducer.BatchSource createSplitBatchSource(HiveExternalMetaCache cache, At
309390

310391
@Override
311392
public List<Split> getNextBatch() throws IOException, UserException {
312-
List<Split> batch = new ArrayList<>(splitAssignment.getSplitBatchSize());
393+
List<Split> batch = SplitProducer.newBatch(splitAssignment.getSplitBatchSize());
313394
while (batch.size() < splitAssignment.getSplitBatchSize() && splitAssignment.needMoreSplit()) {
314395
if (ranges.hasNext()) {
315396
batch.add(ranges.next());
@@ -349,14 +430,20 @@ private boolean loadNextFileRanges() throws IOException, UserException {
349430
// Publish its accumulated count before moving to the next selected partition.
350431
updatePartitionSplitEstimate();
351432
partitionSplitCount = 0;
352-
int index = nextPartition.getAndIncrement();
353-
if (index >= prunedPartitions.size()) {
354-
return false;
433+
FileCacheValue prefetched = countPlan == null ? null : countPlan.prefetched.poll();
434+
List<FileCacheValue> loaded;
435+
if (prefetched != null) {
436+
loaded = Collections.singletonList(prefetched);
437+
} else {
438+
int index = nextPartition.getAndIncrement();
439+
if (index >= prunedPartitions.size()) {
440+
return false;
441+
}
442+
loaded = getFileCachesByPartitions(cache,
443+
Collections.singletonList(prunedPartitions.get(index)), broker, true);
355444
}
356-
List<FileCacheValue> loaded = getFileCachesByPartitions(cache,
357-
Collections.singletonList(prunedPartitions.get(index)), broker, true);
358445
targetSize = determineTargetFileSplitSize(loaded, true);
359-
needSplit = needSplitFiles(loaded, numBackends);
446+
needSplit = countPlan == null ? needSplitFiles(loaded, numBackends) : countPlan.needSplit;
360447
fileCaches = loaded.iterator();
361448
}
362449
return false;
@@ -511,8 +598,7 @@ private List<FileCacheValue> getFileCachesByPartitions(HiveExternalMetaCache cac
511598

512599
private boolean needSplitFiles(List<FileCacheValue> fileCaches, int numBackends) throws UserException {
513600
boolean needSplit = true;
514-
if (isTableLevelCountStarPushdown()
515-
&& !(hmsTable.isHiveTransactionalTable() && hmsTable.isFullAcidTable())) {
601+
if (usesCountFileSplitPolicy()) {
516602
int totalFileNum = 0;
517603
for (FileCacheValue fileCacheValue : fileCaches) {
518604
if (fileCacheValue.getFiles() != null) {
@@ -525,6 +611,11 @@ private boolean needSplitFiles(List<FileCacheValue> fileCaches, int numBackends)
525611
return needSplit;
526612
}
527613

614+
private boolean usesCountFileSplitPolicy() throws UserException {
615+
return isTableLevelCountStarPushdown()
616+
&& !(hmsTable.isHiveTransactionalTable() && hmsTable.isFullAcidTable());
617+
}
618+
528619
private long retainedHiveFileCount(List<FileCacheValue> fileCaches) {
529620
long fileCount = fileCaches.size();
530621
for (FileCacheValue fileCache : fileCaches) {

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1740,6 +1740,7 @@ public void startSplit(int numBackends) throws UserException {
17401740

17411741
public void doStartSplit() throws UserException {
17421742
TableScan scan = createTableScan();
1743+
Executor executor = getSplitTaskExecutor();
17431744
Closeable generationLease = IcebergUtils.retainTableGenerationForAsyncPlanning(
17441745
source.getTargetTable(), frozenGenerationSource);
17451746
long started = System.currentTimeMillis();
@@ -1754,7 +1755,7 @@ public List<Split> getNextBatch() throws Exception {
17541755
tasks = planFileScanTask(scan);
17551756
iterator = tasks.iterator();
17561757
}
1757-
List<Split> batch = new ArrayList<>(splitAssignment.getSplitBatchSize());
1758+
List<Split> batch = SplitProducer.newBatch(splitAssignment.getSplitBatchSize());
17581759
while (batch.size() < splitAssignment.getSplitBatchSize()
17591760
&& splitAssignment.needMoreSplit() && iterator.hasNext()) {
17601761
batch.add(createIcebergSplit(iterator.next()));
@@ -1774,7 +1775,7 @@ public void close() throws IOException {
17741775
}
17751776
}
17761777
};
1777-
new SplitProducer(splitAssignment, getSplitTaskExecutor(), batches, failure -> {
1778+
new SplitProducer(splitAssignment, executor, batches, failure -> {
17781779
if (failure != null) {
17791780
Optional<NotSupportedException> unsupported = checkNotSupportedException(failure);
17801781
splitAssignment.setException(new UserException(

0 commit comments

Comments
 (0)