Skip to content

Commit 47241e4

Browse files
committed
[fix](fe) Retry split fetches while file listing is still running
### What problem does this PR solve? Issue Number: N/A Related PR: #69683 Problem Summary: With the new batch-mode defaults, a Hive scan can dispatch after its first partition produces a split. If another partition is still listing files, a BE may exhaust its assigned ranges and reach the one-second split-fetch wait limit. FE previously returned a fatal timeout even though split production was still running. FE now returns a retryable INCOMPLETE status only for this temporary gap; BE retries after checking query cancellation and releases its FE RPC client between attempts. Completed scans still return EOF, while producer errors and cancellation retain their failure paths. The wait setting remains a per-RPC bound and now rejects nonpositive values to avoid a tight retry loop. ### Release note External batch scans no longer fail solely because another partition takes longer than one split-fetch wait interval to list files. ### Check List (For Author) - Test: Unit Test: SplitSourceTest 37/37 passed with the repository test script. FE Maven build and Checkstyle passed through build.sh --fe; the outer script returned nonzero during output packaging. BE clang-format 16 check and git diff --check passed. BE compilation was not run locally. - Behavior changed: Yes. A temporary split gap produces a retryable response instead of a fatal timeout. - Does this need documentation: No. The session variable description was updated in code.
1 parent 650df2a commit 47241e4

5 files changed

Lines changed: 108 additions & 19 deletions

File tree

‎be/src/exec/scan/split_source_connector.cpp‎

Lines changed: 25 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -48,26 +48,36 @@ Status RemoteSplitSourceConnector::get_next(bool* has_next, TFileRangeDesc* rang
4848
*has_next = false;
4949
if (_scan_index == _scan_ranges.size() && !_last_batch) {
5050
SCOPED_TIMER(_get_split_timer);
51-
Status coord_status;
52-
// No need to set timeout because on FE side, there is a max fetch time
53-
FrontendServiceConnection coord(_state->exec_env()->frontend_client_cache(),
54-
_state->get_query_ctx()->coord_addr, &coord_status);
55-
RETURN_IF_ERROR(coord_status);
5651
TFetchSplitBatchRequest request;
5752
request.__set_split_source_id(_split_source_id);
5853
request.__set_max_num_splits(config::remote_split_source_batch_size);
5954
TFetchSplitBatchResult result;
60-
try {
61-
coord->fetchSplitBatch(result, request);
62-
if (result.__isset.status && result.status.status_code != TStatusCode::OK) {
63-
return Status::IOError<false>("Failed to get batch of split source: {}",
64-
result.status.error_msgs.empty()
65-
? "unknown error"
66-
: result.status.error_msgs[0]);
55+
do {
56+
RETURN_IF_CANCELLED(_state);
57+
Status coord_status;
58+
// Each FE call is bounded by the split-fetch wait interval. Release the client between
59+
// retries so a slow listing does not reserve a connection for the whole query.
60+
FrontendServiceConnection coord(_state->exec_env()->frontend_client_cache(),
61+
_state->get_query_ctx()->coord_addr, &coord_status);
62+
RETURN_IF_ERROR(coord_status);
63+
try {
64+
coord->fetchSplitBatch(result, request);
65+
if (result.__isset.status && result.status.status_code == TStatusCode::INCOMPLETE) {
66+
// One Hive partition may have supplied the first range while another is still
67+
// listing files. FE's bounded wait is not EOF or a query failure in that case.
68+
continue;
69+
}
70+
if (result.__isset.status && result.status.status_code != TStatusCode::OK) {
71+
return Status::IOError<false>("Failed to get batch of split source: {}",
72+
result.status.error_msgs.empty()
73+
? "unknown error"
74+
: result.status.error_msgs[0]);
75+
}
76+
} catch (std::exception& e) {
77+
return Status::IOError<false>("Failed to get batch of split source: {}", e.what());
6778
}
68-
} catch (std::exception& e) {
69-
return Status::IOError<false>("Failed to get batch of split source: {}", e.what());
70-
}
79+
break;
80+
} while (true);
7181
_last_batch = result.splits.empty();
7282
_merge_ranges<TScanRangeLocations>(_scan_ranges, result.splits);
7383
_scan_index = 0;

‎fe/fe-core/src/main/java/org/apache/doris/datasource/split/SplitSource.java‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,14 @@
4141
* the split source when its related scan node is GC.
4242
*/
4343
public class SplitSource {
44+
/** The producer is still running, but this fetch did not receive a split within its RPC wait window. */
45+
public static class SplitPendingException extends UserException {
46+
public SplitPendingException(long maxWaitTime) {
47+
super("Timeout waiting for the next split after " + maxWaitTime + " ms; production is still running",
48+
null, false, false);
49+
}
50+
}
51+
4452
private static final AtomicLong UNIQUE_ID_GENERATOR = new AtomicLong(0);
4553

4654
private final long uniqueId;
@@ -56,6 +64,7 @@ public SplitSource(Backend backend, SplitAssignment splitAssignment, long maxWai
5664
}
5765

5866
public SplitSource(Backend backend, SplitAssignment splitAssignment, long maxWaitTime, int initialBatchSize) {
67+
Preconditions.checkArgument(maxWaitTime > 0, "Split fetch wait time must be positive");
5968
Preconditions.checkArgument(initialBatchSize > 0, "Initial split batch size must be positive");
6069
this.initialBatchSize = initialBatchSize;
6170
this.uniqueId = UNIQUE_ID_GENERATOR.getAndIncrement();
@@ -118,7 +127,18 @@ public synchronized List<TScanRangeLocations> getNextBatch(int maxBatchSize) thr
118127
}
119128
long remainingNanos = waitNanos - (System.nanoTime() - startTime);
120129
if (remainingNanos <= 0) {
121-
throw new UserException("Timeout. Max wait time(ms): " + maxWaitTime);
130+
// A temporary gap between producer batches is neither EOF nor a query failure.
131+
// Let the BE retry after this bounded long poll rather than failing a slow listing.
132+
SplitAssignment.AssignmentQueue pending = splitAssignment.getAssignedSplits(backend);
133+
if (splitAssignment.isStop()) {
134+
isLastBatch.set(true);
135+
break;
136+
}
137+
if (pending == null) {
138+
isLastBatch.set(true);
139+
break;
140+
}
141+
throw new SplitPendingException(maxWaitTime);
122142
}
123143
try {
124144
TScanRangeLocations split = splitAssignment.awaitSplit(backend, remainingNanos);

‎fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2669,10 +2669,16 @@ public void checkSplitQueueCapacity(String value) throws Exception {
26692669

26702670
@VarAttrDef.VarAttr(
26712671
name = FETCH_SPLITS_MAX_WAIT_TIME,
2672-
description = "The max wait time of getting splits in batch mode.",
2672+
checker = "checkFetchSplitsMaxWaitTime",
2673+
description = "Maximum wait time per split-fetch RPC in batch mode. A running producer can be retried "
2674+
+ "after this interval; completion, cancellation and production errors are reported separately.",
26732675
needForward = true)
26742676
public long fetchSplitsMaxWaitTime = 1000;
26752677

2678+
public void checkFetchSplitsMaxWaitTime(String value) throws Exception {
2679+
checkFieldLongValue(FETCH_SPLITS_MAX_WAIT_TIME, 1, value);
2680+
}
2681+
26762682
@VarAttrDef.VarAttr(
26772683
name = ENABLE_PARQUET_LAZY_MAT,
26782684
fuzzy = true,

‎fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1119,6 +1119,10 @@ public TFetchSplitBatchResult fetchSplitBatch(TFetchSplitBatchRequest request) t
11191119
result.setSplits(locations);
11201120
result.status = new TStatus(TStatusCode.OK);
11211121
return result;
1122+
} catch (SplitSource.SplitPendingException e) {
1123+
// The backend will retry; an empty OK batch would incorrectly mean EOF.
1124+
result.status = new TStatus(TStatusCode.INCOMPLETE);
1125+
return result;
11221126
} catch (Exception e) {
11231127
LOG.warn("failed to fetch split batch with source id {}", request.getSplitSourceId(), e);
11241128
result.status = new TStatus(TStatusCode.INTERNAL_ERROR);

‎fe/fe-core/src/test/java/org/apache/doris/datasource/split/SplitSourceTest.java‎

Lines changed: 51 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -394,10 +394,59 @@ void availablePartialBatchDoesNotWaitForProducer() throws Exception {
394394
}
395395

396396
@Test
397-
void timeoutIsAnErrorRatherThanEof() {
397+
void waitWindowExpiryIsPendingRatherThanEof() {
398398
SplitSource source = new SplitSource(backend, assignment, 20);
399-
UserException failure = Assertions.assertThrows(UserException.class, () -> source.getNextBatch(1));
399+
UserException failure = Assertions.assertThrows(SplitSource.SplitPendingException.class,
400+
() -> source.getNextBatch(1));
400401
Assertions.assertTrue(failure.getMessage().contains("Timeout"));
402+
Assertions.assertThrows(IllegalArgumentException.class, () -> new SplitSource(backend, assignment, 0));
403+
SessionVariable variables = new SessionVariable();
404+
Assertions.assertThrows(Exception.class, () -> variables.checkFetchSplitsMaxWaitTime("0"));
405+
}
406+
407+
@Test
408+
void unfinishedListingAfterFirstSplitIsPendingNotEof() throws Exception {
409+
SplitSource source = new SplitSource(backend, assignment, 20, 1);
410+
assignment.addToQueue(splits.subList(0, 1));
411+
Assertions.assertEquals(ranges.subList(0, 1), source.getNextBatch(1));
412+
413+
// Another partition is still listing beyond one fetch wait window.
414+
Assertions.assertThrows(SplitSource.SplitPendingException.class, () -> source.getNextBatch(1));
415+
assignment.addToQueue(splits.subList(1, 2));
416+
assignment.finishSchedule();
417+
Assertions.assertEquals(ranges.subList(1, 2), source.getNextBatch(1));
418+
Assertions.assertTrue(source.getNextBatch(1).isEmpty());
419+
}
420+
421+
@Test
422+
void completedListingReturnsEof() throws Exception {
423+
SplitSource source = new SplitSource(backend, assignment, 20);
424+
assignment.finishSchedule();
425+
Assertions.assertTrue(source.getNextBatch(1).isEmpty());
426+
}
427+
428+
@Test
429+
void backendWithoutInitialAssignmentCanRetryUntilListingFinishes() throws Exception {
430+
Backend otherBackend = new Backend(2L, "127.0.0.2", 9050);
431+
SplitSource first = new SplitSource(backend, assignment, 20, 1);
432+
SplitSource second = new SplitSource(otherBackend, assignment, 20, 1);
433+
assignment.addToQueue(splits.subList(0, 1));
434+
Assertions.assertEquals(ranges.subList(0, 1), first.getNextBatch(1));
435+
436+
Assertions.assertThrows(SplitSource.SplitPendingException.class, () -> second.getNextBatch(1));
437+
Assertions.assertEquals(1, assignment.offerBatch(otherBackend, ranges.subList(1, 2), 0, () -> {}));
438+
assignment.finishSchedule();
439+
Assertions.assertEquals(ranges.subList(1, 2), second.getNextBatch(1));
440+
Assertions.assertTrue(second.getNextBatch(1).isEmpty());
441+
}
442+
443+
@Test
444+
void producerFailureAfterPendingFetchIsStillAnError() throws Exception {
445+
SplitSource source = new SplitSource(backend, assignment, 20);
446+
Assertions.assertThrows(SplitSource.SplitPendingException.class, () -> source.getNextBatch(1));
447+
UserException failure = new UserException("listing failed");
448+
assignment.setException(failure);
449+
Assertions.assertSame(failure, Assertions.assertThrows(UserException.class, () -> source.getNextBatch(1)));
401450
}
402451

403452
@Test

0 commit comments

Comments
 (0)