Skip to content

Commit 663c74d

Browse files
committed
[fix](fe) Preserve Hive estimates and validate scan scheduler sizes
### What problem does this PR solve? Issue Number: N/A Related PR: #69298, #69986 Problem Summary: Hive streaming batches may span several partitions and a single partition may span many batches. Using the outgoing batch size as a per-partition split estimate therefore undercounts large partitions and overcounts several small ones. Accumulate generated splits locally for the current selected partition, publish the existing global maximum at yields and partition boundaries, and reset only when advancing partitions. Keep lazy enumeration, the ten-split floor and int saturation. The three new external scan scheduler sizes also accepted nonpositive startup values, failing later in scheduler construction or the first query group. Add a pure positive-integer validator to ordinary and custom startup configuration loading while preserving empty defaults, immutability and the exclusion of runtime callbacks with side effects. ### Release note Correct Hive split estimates during streaming planning and reject nonpositive external scan scheduler sizes at startup. ### Check List (For Author) - Test: Unit tests and packaging - Reproduced both configuration load failures and both split-estimate failures before repair. - All 147 related FE tests passed: 13 fe-common and 134 fe-core tests. - Full FE packaging and Checkstyle passed; git diff --check passed. - No local complete P0 or external-service regression in this cycle. - Behavior changed: Yes. Partition estimates accumulate independently of outgoing batch sizes, and nonpositive scheduler sizes are rejected during startup before the invalid field is updated. Lazy enumeration and existing scheduling/cancellation behavior are preserved. - Does this need documentation: No
1 parent 68ef8c2 commit 663c74d

5 files changed

Lines changed: 139 additions & 7 deletions

File tree

‎fe/fe-common/src/main/java/org/apache/doris/common/Config.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2472,13 +2472,15 @@ public class Config extends ConfigBase {
24722472
@ConfField(mutable = false, masterOnly = false)
24732473
public static int max_external_cache_loader_thread_pool_size = 64;
24742474

2475-
@ConfField(description = "Workers for fair split production")
2475+
@ConfField(callback = PositiveIntConfHandler.class, description = "Positive worker count for fair split production")
24762476
public static int external_scan_split_threads = 64;
24772477

2478-
@ConfField(description = "Workers for independent metadata tasks")
2478+
@ConfField(callback = PositiveIntConfHandler.class,
2479+
description = "Positive worker count for independent metadata tasks")
24792480
public static int external_scan_metadata_threads = 64;
24802481

2481-
@ConfField(description = "Maximum queued tasks per query in each external scan scheduler")
2482+
@ConfField(callback = PositiveIntConfHandler.class,
2483+
description = "Positive maximum queued tasks per query in each external scan scheduler")
24822484
public static int external_scan_task_queue_capacity = 128;
24832485

24842486
/**

‎fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,21 @@ private static String checkedOption(Field field, String confVal) throws ConfigEx
104104
}
105105
}
106106

107+
public static class PositiveIntConfHandler implements ConfHandler {
108+
@Override
109+
public void handle(Field field, String confVal) throws Exception {
110+
setConfigField(field, checkedValue(field, confVal));
111+
}
112+
113+
private static String checkedValue(Field field, String confVal) throws ConfigException {
114+
int value = Integer.parseInt(confVal.trim());
115+
if (value <= 0) {
116+
throw new ConfigException("Config '" + field.getName() + "' must be greater than 0");
117+
}
118+
return Integer.toString(value);
119+
}
120+
}
121+
107122
static class CommaSeparatedIntersectConfHandler implements ConfHandler {
108123
@Override
109124
public void handle(Field field, String newVal) throws Exception {
@@ -296,13 +311,16 @@ private static void setFields(Properties props, boolean isLdapConfig) throws Exc
296311
}
297312

298313
try {
299-
// Runtime callbacks can have side effects; only options validation is safe here.
314+
// Runtime callbacks can have side effects; only pure value validators run here.
300315
if (anno.callback() == OptionsConfHandler.class) {
301316
confVal = OptionsConfHandler.checkedOption(f, confVal);
302317
}
303318
if (confVal.isEmpty()) {
304319
continue;
305320
}
321+
if (anno.callback() == PositiveIntConfHandler.class) {
322+
confVal = PositiveIntConfHandler.checkedValue(f, confVal);
323+
}
306324
setConfigField(f, confVal);
307325
} catch (Exception e) {
308326
String msg = String.format("Failed to set config, name: %s, value: %s", f.getName(), confVal);

‎fe/fe-common/src/test/java/org/apache/doris/common/ConfigTest.java‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,17 @@
2222
import org.junit.Test;
2323
import org.junit.jupiter.api.Assertions;
2424

25+
import java.lang.reflect.Field;
2526
import java.nio.file.Files;
2627
import java.nio.file.Path;
2728
import java.util.List;
2829
import java.util.Map;
2930

3031
public class ConfigTest {
32+
private static final String[] EXTERNAL_SCAN_SCHEDULER_CONFIGS = {
33+
"external_scan_split_threads", "external_scan_metadata_threads", "external_scan_task_queue_capacity"
34+
};
35+
3136
@BeforeClass
3237
public static void setUp() throws Exception {
3338
Config config = new Config();
@@ -153,6 +158,64 @@ public void testOptionsConfigIsValidatedInCustomStartupFile() throws Exception {
153158
}
154159
}
155160

161+
@Test
162+
public void testExternalScanSchedulerRejectsNonPositiveStartupValues() throws Exception {
163+
assertExternalScanSchedulerRejectsNonPositiveValues(false);
164+
}
165+
166+
@Test
167+
public void testExternalScanSchedulerRejectsNonPositiveCustomStartupValues() throws Exception {
168+
assertExternalScanSchedulerRejectsNonPositiveValues(true);
169+
}
170+
171+
private void assertExternalScanSchedulerRejectsNonPositiveValues(boolean custom) throws Exception {
172+
Config config = new Config();
173+
config.init(configFile("").toString());
174+
for (String name : EXTERNAL_SCAN_SCHEDULER_CONFIGS) {
175+
Field field = Config.class.getField(name);
176+
int original = field.getInt(null);
177+
try {
178+
for (String value : new String[] {"0", "-1"}) {
179+
Path invalid = configFile(name + "=" + value + "\n");
180+
IllegalArgumentException error = Assertions.assertThrows(IllegalArgumentException.class, () -> {
181+
if (custom) {
182+
config.initCustom(invalid.toString());
183+
} else {
184+
config.init(invalid.toString());
185+
}
186+
});
187+
Assertions.assertTrue(error.getMessage().contains(name));
188+
Assertions.assertInstanceOf(ConfigException.class, error.getCause());
189+
Assertions.assertTrue(error.getCause().getMessage().contains("must be greater than 0"));
190+
Assertions.assertEquals(original, field.getInt(null));
191+
}
192+
} finally {
193+
field.setInt(null, original);
194+
}
195+
}
196+
}
197+
198+
@Test
199+
public void testExternalScanSchedulerAcceptsPositiveStartupValuesAndStaysImmutable() throws Exception {
200+
Config config = new Config();
201+
for (String name : EXTERNAL_SCAN_SCHEDULER_CONFIGS) {
202+
Field field = Config.class.getField(name);
203+
int original = field.getInt(null);
204+
try {
205+
config.init(configFile(name + "=1\n").toString());
206+
Assertions.assertEquals(1, field.getInt(null));
207+
config.initCustom(configFile(name + "= 128 \n").toString());
208+
Assertions.assertEquals(128, field.getInt(null));
209+
config.init(configFile(name + "=\n").toString());
210+
Assertions.assertEquals(128, field.getInt(null));
211+
Assertions.assertThrows(ConfigException.class, () -> ConfigBase.setMutableConfig(name, "8"));
212+
Assertions.assertEquals(128, field.getInt(null));
213+
} finally {
214+
field.setInt(null, original);
215+
}
216+
}
217+
}
218+
156219
private static Path configFile(String contents) throws Exception {
157220
Path file = Files.createTempFile("fe_ut_options_", ".conf");
158221
file.toFile().deleteOnExit();

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -305,23 +305,30 @@ SplitProducer.BatchSource createSplitBatchSource(HiveExternalMetaCache cache, At
305305
private FileCacheValue currentCache;
306306
private long targetSize;
307307
private boolean needSplit;
308+
private long partitionSplitCount;
308309

309310
@Override
310311
public List<Split> getNextBatch() throws IOException, UserException {
311312
List<Split> batch = new ArrayList<>(splitAssignment.getSplitBatchSize());
312313
while (batch.size() < splitAssignment.getSplitBatchSize() && splitAssignment.needMoreSplit()) {
313314
if (ranges.hasNext()) {
314315
batch.add(ranges.next());
316+
partitionSplitCount++;
315317
continue;
316318
}
317319
if (!loadNextFileRanges()) {
318320
break;
319321
}
320322
}
321-
numSplitsPerPartition.accumulateAndGet(batch.size(), Math::max);
323+
updatePartitionSplitEstimate();
322324
return batch;
323325
}
324326

327+
private void updatePartitionSplitEstimate() {
328+
numSplitsPerPartition.accumulateAndGet(
329+
(int) Math.min(Integer.MAX_VALUE, partitionSplitCount), Math::max);
330+
}
331+
325332
private boolean loadNextFileRanges() throws IOException, UserException {
326333
while (splitAssignment.needMoreSplit()) {
327334
if (files.hasNext()) {
@@ -338,6 +345,10 @@ private boolean loadNextFileRanges() throws IOException, UserException {
338345
: currentCache.getFiles().iterator();
339346
continue;
340347
}
348+
// One producer batch can span partitions, and one partition can span batches.
349+
// Publish its accumulated count before moving to the next selected partition.
350+
updatePartitionSplitEstimate();
351+
partitionSplitCount = 0;
341352
int index = nextPartition.getAndIncrement();
342353
if (index >= prunedPartitions.size()) {
343354
return false;

‎fe/fe-core/src/test/java/org/apache/doris/datasource/hive/source/HiveStreamingSplitTest.java‎

Lines changed: 40 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@
6060

6161
import java.lang.reflect.Field;
6262
import java.util.ArrayList;
63+
import java.util.Arrays;
6364
import java.util.Collections;
6465
import java.util.HashSet;
6566
import java.util.List;
@@ -90,6 +91,36 @@ void realModeGateAndDispatchPreservePartitionOverlapAndEveryRange() throws Excep
9091
}
9192
}
9293

94+
@Test
95+
void partitionEstimateAccumulatesAcrossSmallProducerBatches() throws Exception {
96+
try (Fixture f = new Fixture(1, 1, 1)) {
97+
Assertions.assertTrue(f.node.isBatchMode());
98+
f.node.startSplit(1);
99+
f.release.countDown();
100+
Assertions.assertEquals(17, f.drain().size());
101+
Assertions.assertEquals(17, f.node.numApproximateSplits());
102+
}
103+
}
104+
105+
@Test
106+
void partitionEstimateDoesNotCountSeveralPartitionsAsOne() throws Exception {
107+
try (Fixture f = new Fixture(3, 1, 64)) {
108+
f.fileLengths[0] = 17;
109+
f.fileLengths[1] = 4;
110+
f.fileLengths[2] = 8;
111+
Assertions.assertTrue(f.node.isBatchMode());
112+
f.release.countDown();
113+
SplitProducer.BatchSource source = f.node.createSplitBatchSource(f.cache, new AtomicInteger(), "", 1);
114+
try {
115+
Assertions.assertEquals(29, source.getNextBatch().size());
116+
Assertions.assertTrue(source.getNextBatch().isEmpty());
117+
Assertions.assertEquals(51, f.node.numApproximateSplits());
118+
} finally {
119+
source.close();
120+
}
121+
}
122+
}
123+
93124
@Test
94125
void fileCountGateUsesOneProducerAndHonorsSmallConsumerBatch() throws Exception {
95126
try (Fixture f = new Fixture(2, 1024)) {
@@ -357,11 +388,18 @@ private static class Fixture implements AutoCloseable {
357388
final StatementContext statement = new StatementContext(context, null);
358389
final MockedStatic<org.apache.doris.catalog.Env> mockedEnv;
359390
final TestNode node;
391+
final long[] fileLengths;
360392

361393
Fixture(int partitionCount, int partitionThreshold) throws Exception {
394+
this(partitionCount, partitionThreshold, 7);
395+
}
396+
397+
Fixture(int partitionCount, int partitionThreshold, int producerBatchSize) throws Exception {
398+
fileLengths = new long[partitionCount];
399+
Arrays.fill(fileLengths, 17);
362400
Config.max_external_cache_loader_thread_pool_size = 2;
363401
variables.numPartitionsInBatchMode = partitionThreshold;
364-
variables.splitBatchSize = 7;
402+
variables.splitBatchSize = producerBatchSize;
365403
variables.splitQueueCapacity = 1;
366404
variables.enableFileScannerV2 = false;
367405
variables.setFileSplitSize(1);
@@ -423,7 +461,7 @@ private static class Fixture implements AutoCloseable {
423461
HiveExternalMetaCache.HiveFileStatus file(int partition) {
424462
HiveExternalMetaCache.HiveFileStatus status = new HiveExternalMetaCache.HiveFileStatus();
425463
status.setPath(LocationPath.of("hdfs://test/" + partition));
426-
status.setLength(17);
464+
status.setLength(fileLengths[partition]);
427465
return status;
428466
}
429467

0 commit comments

Comments
 (0)