Skip to content

Commit b76e398

Browse files
committed
[fix](fe) Keep Hive split estimates independent of partition batches
### What problem does this PR solve? Issue Number: None Related PR: #69683 Problem Summary: Hive split generation now schedules several partitions per task, but recorded the whole task output as the split count of one partition. Extrapolating this across all pruned partitions inflated approximate split/file-size estimates: six partitions with ten splits each reported 360 instead of 60. Count actual output by partition values within each batch and atomically retain the largest per-partition count. Preserve the existing minimum and cover uneven/empty partitions, sampling, unpartitioned scans, and different scheduling batch sizes. Correct the legacy-FE closure comment and verify that its existing timeout error is propagated without retries or an unsupported close request. ### Release note Correct approximate Hive split counts and derived file-size estimates during batched external scans. ### Check List (For Author) - Test: 209 related FE unit tests passed through run-fe-ut.sh. The unchanged-production reproduction showed 360 instead of 60 and 116 instead of 100; all four new cases now pass. Full FE packaging and Checkstyle passed through build.sh --fe. 61 related BE ASAN tests passed through run-be-ut.sh, including the new old-FE timeout/no-retry case. BE test validation uses the existing local-only focused CMake target and source-specific compiler-warning adjustment, restored afterwards and excluded from this commit. - Unit Test - Behavior changed: Yes; approximate Hive estimates are independent of scheduling batch size and preserve the per-partition maximum. - Does this need documentation: No Changed-line clang-tidy remains blocked by the existing core/types.h:578 unmatched NOLINTEND and the pre-included gtest/internal/gtest-internal.h:56 float.h diagnostic.
1 parent abcbb5c commit b76e398

4 files changed

Lines changed: 209 additions & 4 deletions

File tree

‎be/src/exec/scan/split_source_connector.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -219,7 +219,7 @@ Status RemoteSplitSourceConnector::_fetch_batch(TFetchSplitBatchResult* result,
219219

220220
Status RemoteSplitSourceConnector::close() {
221221
stop();
222-
// Older FEs have no bounded-queue release dependency and do not understand this request.
222+
// Older FEs do not advertise support for the source-close request.
223223
if (!_supports_close || _close_requested.exchange(true, std::memory_order_acq_rel)) {
224224
return Status::OK();
225225
}

‎be/test/exec/scan/file_scanner_v2_test.cpp‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1671,6 +1671,32 @@ TEST(FileScannerV2Test, OldFrontendDescriptorDoesNotSendAnUnsupportedCloseReques
16711671
EXPECT_TRUE(source.close().ok());
16721672
}
16731673

1674+
TEST(FileScannerV2Test, OldFrontendTimeoutRemainsAnErrorWithoutRetry) {
1675+
RuntimeState state {TQueryOptions(), TQueryGlobals()};
1676+
ScriptedRemoteSplitSource source(&state, false);
1677+
int fetched = 0;
1678+
source.fetch = [&](TFetchSplitBatchResult* result, const TFetchSplitBatchRequest& request) {
1679+
EXPECT_FALSE(request.__isset.close_source);
1680+
if (++fetched > 1) {
1681+
return Status::InternalError("the old FE timeout must not be retried");
1682+
}
1683+
TStatus thrift_status;
1684+
thrift_status.__set_status_code(TStatusCode::INTERNAL_ERROR);
1685+
thrift_status.__set_error_msgs({"Timeout. Max wait time(ms): 1000"});
1686+
result->__set_status(thrift_status);
1687+
return Status::OK();
1688+
};
1689+
bool has_next = true;
1690+
TFileRangeDesc range;
1691+
Status status = source.get_next(&has_next, &range);
1692+
EXPECT_FALSE(status.ok());
1693+
EXPECT_NE(status.to_string().find("Timeout. Max wait time(ms): 1000"), std::string::npos);
1694+
EXPECT_FALSE(has_next);
1695+
EXPECT_EQ(fetched, 1);
1696+
EXPECT_TRUE(source.close().ok());
1697+
EXPECT_EQ(fetched, 1);
1698+
}
1699+
16741700
TEST(FileScannerV2Test, OwnerCloseDoesNotWaitForTheFetchLockOrSurfaceALateFetchError) {
16751701
RuntimeState state {TQueryOptions(), TQueryGlobals()};
16761702
ScriptedRemoteSplitSource source(&state);

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -263,9 +263,7 @@ public void startSplit(int numBackends) {
263263
getFileSplitByPartitions(
264264
cache, partitions, allFiles, bindBrokerName,
265265
numBackends, true);
266-
if (allFiles.size() > numSplitsPerPartition.get()) {
267-
numSplitsPerPartition.set(allFiles.size());
268-
}
266+
updateNumSplitsPerPartition(allFiles);
269267
if (splitAssignment.needMoreSplit()) {
270268
splitAssignment.addToQueue(allFiles);
271269
}
@@ -319,6 +317,17 @@ public int numApproximateSplits() {
319317
return numSplitsPerPartition.get() * prunedPartitions.size();
320318
}
321319

320+
private void updateNumSplitsPerPartition(List<Split> splits) {
321+
// Scheduling batches contain several partitions; extrapolate the largest single partition.
322+
Map<List<String>, Integer> partitionSplitCounts = Maps.newHashMap();
323+
int maxSplitCount = 0;
324+
for (Split split : splits) {
325+
int count = partitionSplitCounts.merge(((FileSplit) split).getPartitionValues(), 1, Integer::sum);
326+
maxSplitCount = Math.max(maxSplitCount, count);
327+
}
328+
numSplitsPerPartition.accumulateAndGet(maxSplitCount, Math::max);
329+
}
330+
322331
private void getFileSplitByPartitions(HiveExternalMetaCache cache, List<HivePartition> partitions,
323332
List<Split> allFiles, String bindBrokerName, int numBackends,
324333
boolean isBatchMode) throws IOException, UserException {
Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.datasource.hive.source;
19+
20+
import org.apache.doris.analysis.TableSample;
21+
import org.apache.doris.analysis.TupleDescriptor;
22+
import org.apache.doris.analysis.TupleId;
23+
import org.apache.doris.catalog.Env;
24+
import org.apache.doris.common.util.LocationPath;
25+
import org.apache.doris.datasource.ExternalMetaCacheMgr;
26+
import org.apache.doris.datasource.FileQueryScanNode;
27+
import org.apache.doris.datasource.FileSplitter;
28+
import org.apache.doris.datasource.SplitAssignment;
29+
import org.apache.doris.datasource.hive.HMSExternalCatalog;
30+
import org.apache.doris.datasource.hive.HMSExternalTable;
31+
import org.apache.doris.datasource.hive.HiveExternalMetaCache;
32+
import org.apache.doris.datasource.hive.HiveExternalMetaCache.FileCacheValue;
33+
import org.apache.doris.datasource.hive.HiveExternalMetaCache.HiveFileStatus;
34+
import org.apache.doris.datasource.hive.HivePartition;
35+
import org.apache.doris.planner.PlanNodeId;
36+
import org.apache.doris.planner.ScanContext;
37+
import org.apache.doris.planner.ScanNode;
38+
import org.apache.doris.qe.SessionVariable;
39+
import org.apache.doris.thrift.TFileFormatType;
40+
41+
import org.junit.Assert;
42+
import org.junit.Test;
43+
import org.mockito.MockedStatic;
44+
import org.mockito.Mockito;
45+
46+
import java.lang.reflect.Field;
47+
import java.util.ArrayList;
48+
import java.util.Collections;
49+
import java.util.HashMap;
50+
import java.util.List;
51+
import java.util.Map;
52+
import java.util.concurrent.CompletableFuture;
53+
import java.util.concurrent.ExecutorService;
54+
import java.util.concurrent.Executors;
55+
import java.util.concurrent.TimeUnit;
56+
import java.util.concurrent.atomic.AtomicInteger;
57+
import java.util.stream.Collectors;
58+
59+
public class HiveSplitEstimateTest {
60+
@Test
61+
public void testUniformPartitionsAreIndependentOfBatchSize() throws Exception {
62+
for (int batchSize : new int[] {16, 2, 1}) {
63+
Assert.assertEquals(60, runBatchEstimate(batchSize, false, 10, 10, 10, 10, 10, 10));
64+
}
65+
}
66+
67+
@Test
68+
public void testUnevenAndEmptyPartitionsKeepTheLargestPartitionEstimate() throws Exception {
69+
for (int batchSize : new int[] {16, 2, 1}) {
70+
Assert.assertEquals(100, runBatchEstimate(batchSize, false, 0, 1, 25, 3));
71+
}
72+
}
73+
74+
@Test
75+
public void testSampledPartitionsKeepTheSameEstimate() throws Exception {
76+
for (int batchSize : new int[] {16, 2}) {
77+
Assert.assertEquals(60, runBatchEstimate(batchSize, true, 10, 10, 10, 10, 10, 10));
78+
}
79+
}
80+
81+
@Test
82+
public void testUnpartitionedScanKeepsItsSplitCount() throws Exception {
83+
Assert.assertEquals(25, runBatchEstimate(16, false, 25));
84+
}
85+
86+
private int runBatchEstimate(int batchSize, boolean sample, int... fileCounts) throws Exception {
87+
SessionVariable variables = new SessionVariable();
88+
variables.splitPartitionBatchSize = batchSize;
89+
HMSExternalTable table = Mockito.mock(HMSExternalTable.class);
90+
HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class);
91+
Mockito.when(table.getCatalog()).thenReturn(catalog);
92+
Mockito.when(table.getFileFormatType(variables)).thenReturn(TFileFormatType.FORMAT_PARQUET);
93+
Mockito.when(catalog.bindBrokerName()).thenReturn("");
94+
TupleDescriptor descriptor = new TupleDescriptor(new TupleId(0));
95+
descriptor.setTable(table);
96+
HiveScanNode node = new HiveScanNode(
97+
new PlanNodeId(0), descriptor, false, variables, null, ScanContext.EMPTY);
98+
Field splitter = FileQueryScanNode.class.getDeclaredField("fileSplitter");
99+
splitter.setAccessible(true);
100+
splitter.set(node, new FileSplitter(variables.maxInitialSplitSize,
101+
variables.maxSplitSize, variables.maxInitialSplitNum));
102+
if (sample) {
103+
node.setTableSample(new TableSample(true, 100L, 0L));
104+
}
105+
List<HivePartition> partitions = new ArrayList<>();
106+
Map<HivePartition, FileCacheValue> files = new HashMap<>();
107+
int totalFiles = 0;
108+
for (int partition = 0; partition < fileCounts.length; partition++) {
109+
List<String> values = fileCounts.length == 1
110+
? null : Collections.singletonList(Integer.toString(partition));
111+
String path = "hdfs://warehouse/t/p=" + partition;
112+
HivePartition hivePartition = new HivePartition(null, false, "parquet", path,
113+
values, Collections.emptyMap());
114+
partitions.add(hivePartition);
115+
FileCacheValue cacheValue = new FileCacheValue();
116+
cacheValue.setPartitionValues(values);
117+
for (int file = 0; file < fileCounts[partition]; file++) {
118+
HiveFileStatus status = new HiveFileStatus();
119+
status.setPath(LocationPath.of(path + "/file" + file + ".parquet"));
120+
status.setLength(1024);
121+
cacheValue.getFiles().add(status);
122+
totalFiles++;
123+
}
124+
files.put(hivePartition, cacheValue);
125+
}
126+
Field pruned = HiveScanNode.class.getDeclaredField("prunedPartitions");
127+
pruned.setAccessible(true);
128+
pruned.set(node, partitions);
129+
SplitAssignment assignment = Mockito.mock(SplitAssignment.class);
130+
Mockito.when(assignment.needMoreSplit()).thenReturn(true);
131+
Field assignmentField = ScanNode.class.getDeclaredField("splitAssignment");
132+
assignmentField.setAccessible(true);
133+
assignmentField.set(node, assignment);
134+
CompletableFuture<Void> finished = new CompletableFuture<>();
135+
Mockito.doAnswer(call -> {
136+
finished.complete(null);
137+
return null;
138+
}).when(assignment).finishSchedule();
139+
Mockito.doAnswer(call -> {
140+
finished.completeExceptionally(call.getArgument(0));
141+
return null;
142+
}).when(assignment).setException(Mockito.any());
143+
AtomicInteger delivered = new AtomicInteger();
144+
Mockito.doAnswer(call -> {
145+
delivered.addAndGet(call.<List<?>>getArgument(0).size());
146+
return null;
147+
}).when(assignment).addToQueue(Mockito.anyList());
148+
HiveExternalMetaCache cache = Mockito.mock(HiveExternalMetaCache.class);
149+
Mockito.when(cache.getFilesByPartitions(Mockito.anyList(), Mockito.anyBoolean(), Mockito.anyBoolean(),
150+
Mockito.isNull(), Mockito.same(table))).thenAnswer(call -> call.<List<HivePartition>>getArgument(0)
151+
.stream().map(files::get).collect(Collectors.toList()));
152+
Env env = Mockito.mock(Env.class);
153+
ExternalMetaCacheMgr manager = Mockito.mock(ExternalMetaCacheMgr.class);
154+
ExecutorService executor = Executors.newFixedThreadPool(2);
155+
Mockito.when(env.getExtMetaCacheMgr()).thenReturn(manager);
156+
Mockito.when(manager.hive(Mockito.anyLong())).thenReturn(cache);
157+
Mockito.when(manager.getScheduleExecutor()).thenReturn(executor);
158+
try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
159+
mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
160+
node.startSplit(1);
161+
finished.get(5, TimeUnit.SECONDS);
162+
Assert.assertEquals(totalFiles, delivered.get());
163+
Mockito.verify(assignment, Mockito.never()).setException(Mockito.any());
164+
return node.numApproximateSplits();
165+
} finally {
166+
executor.shutdownNow();
167+
Assert.assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS));
168+
}
169+
}
170+
}

0 commit comments

Comments
 (0)