Skip to content

Commit 0cae86d

Browse files
committed
[fix](fe) Preserve exact scan metadata for admission and explain
### What problem does this PR solve? Issue Number: None Related PR: #69683 Problem Summary: Enabling batch planning for small external scans leaves Iceberg partition counts at zero and split counts approximate when SQL block rules run before execution. It also removes partition and delete-file details from EXPLAIN. Require the existing synchronous planning path for EXPLAIN and enabled applicable partition_num/tablet_num rules, with one shared eligibility gate for Hive and Iceberg. Keep ordinary unrestricted queries on bounded asynchronous split delivery. Rule applicability and the internal-query bypass remain owned by SqlBlockRuleMgr. ### Release note Preserve external-table partition and split SQL limits and complete EXPLAIN scan details when batch split planning is enabled. ### Check List (For Author) - Test: Unit Test - Reproduced the two eligibility regressions before the fix; all 196 related FE unit tests pass after the fix. - Full FE packaging and Checkstyle pass through build.sh --fe -j4 (UI build disabled). - External-service regression remains for remote CI; no BE code changed or BE validation rerun. - Behavior changed: Yes (use exact synchronous scan planning for EXPLAIN and applicable partition/split limits) - Does this need documentation: No
1 parent 994205e commit 0cae86d

7 files changed

Lines changed: 188 additions & 1 deletion

File tree

‎fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,30 @@ private void matchSql(SqlBlockRule rule, String originSql, String sqlHash) throw
267267
}
268268
}
269269

270+
/** Whether this query needs exact partition/split counts before execution can be admitted. */
271+
public boolean hasScanCountLimitations(String user) {
272+
if (ConnectContext.get().getState().isInternal()) {
273+
return false;
274+
}
275+
for (SqlBlockRule rule : nameToSqlBlockRuleMap.values()) {
276+
if (rule.getGlobal() && requiresExactScanCounts(rule)) {
277+
return true;
278+
}
279+
}
280+
for (String ruleName : Env.getCurrentEnv().getAuth().getSqlBlockRules(user)) {
281+
SqlBlockRule rule = nameToSqlBlockRuleMap.get(ruleName);
282+
if (rule != null && requiresExactScanCounts(rule)) {
283+
return true;
284+
}
285+
}
286+
return false;
287+
}
288+
289+
private static boolean requiresExactScanCounts(SqlBlockRule rule) {
290+
// Cardinality and partition-filter checks do not depend on enumerating file splits.
291+
return rule.getEnable() && (rule.getPartitionNum() != 0 || rule.getTabletNum() != 0);
292+
}
293+
270294
/**
271295
* Check number whether legal by user.
272296
**/

‎fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
package org.apache.doris.datasource;
1919

2020
import org.apache.doris.analysis.SlotDescriptor;
21+
import org.apache.doris.analysis.StatementBase;
2122
import org.apache.doris.analysis.TableSample;
2223
import org.apache.doris.analysis.TableScanParams;
2324
import org.apache.doris.analysis.TableSnapshot;
@@ -118,6 +119,7 @@ public abstract class FileQueryScanNode extends FileScanNode {
118119
protected TableScanParams scanParams;
119120
private Optional<MvccSnapshot> relationSnapshot = Optional.empty();
120121
private boolean relationSnapshotInitialized = false;
122+
private Boolean batchModeEnabled;
121123

122124
protected FileSplitter fileSplitter;
123125
protected SummaryProfile summaryProfile;
@@ -149,6 +151,24 @@ public FileQueryScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeNam
149151
? null : statementContext.getExternalScanTaskCache();
150152
}
151153

154+
/** Freeze the planning mode before asynchronous split conversion loses the query thread context. */
155+
protected boolean isBatchModeEnabled() {
156+
if (batchModeEnabled == null) {
157+
batchModeEnabled = sessionVariable.getEnableExternalTableBatchMode();
158+
ConnectContext context = ConnectContext.get();
159+
if (batchModeEnabled && context != null) {
160+
StatementContext statementContext = context.getStatementContext();
161+
StatementBase statement = statementContext == null ? null : statementContext.getParsedStatement();
162+
// EXPLAIN needs the full file/delete details. Scan-count rules run before execution
163+
// and must not compare their limits with the batch path's sample/approximate counts.
164+
batchModeEnabled = (statement == null || !statement.isExplain())
165+
&& !Env.getCurrentEnv().getSqlBlockRuleMgr()
166+
.hasScanCountLimitations(context.getQualifiedUser());
167+
}
168+
}
169+
return batchModeEnabled;
170+
}
171+
152172
/** Whether statement-scoped scan-task reuse is active for this node. */
153173
protected boolean canReuseExternalScanTasks() {
154174
return sessionVariable.enableExternalScanTaskReuse && externalScanTaskCache != null;

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -296,6 +296,9 @@ public void startSplit(int numBackends) {
296296

297297
@Override
298298
public boolean isBatchMode() {
299+
if (!isBatchModeEnabled()) {
300+
return false;
301+
}
299302
if (!partitionInit) {
300303
try {
301304
prunedPartitions = getPartitions();

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3038,7 +3038,7 @@ public boolean isBatchMode() {
30383038
if (prepareTableLevelSnapshotCount()) {
30393039
return false;
30403040
}
3041-
if (!sessionVariable.getEnableExternalTableBatchMode()) {
3041+
if (!isBatchModeEnabled()) {
30423042
isBatchMode = false;
30433043
return false;
30443044
}

‎fe/fe-core/src/test/java/org/apache/doris/blockrule/SqlBlockRuleMgrTest.java‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,18 @@
1717

1818
package org.apache.doris.blockrule;
1919

20+
import org.apache.doris.catalog.Env;
2021
import org.apache.doris.common.AnalysisException;
2122
import org.apache.doris.metric.MetricRepo;
23+
import org.apache.doris.mysql.privilege.Auth;
2224
import org.apache.doris.persist.gson.GsonUtils;
25+
import org.apache.doris.qe.ConnectContext;
2326

2427
import org.junit.Assert;
2528
import org.junit.BeforeClass;
2629
import org.junit.Test;
30+
import org.mockito.MockedStatic;
31+
import org.mockito.Mockito;
2732

2833
import java.util.List;
2934
import java.util.Map;
@@ -35,6 +40,40 @@ public static void setUp() {
3540
MetricRepo.init();
3641
}
3742

43+
@Test
44+
public void testExactScanCountsOnlyForApplicableEnabledCountRules() {
45+
SqlBlockRuleMgr rules = new SqlBlockRuleMgr();
46+
ConnectContext context = new ConnectContext();
47+
context.setThreadLocalInfo();
48+
Env env = Mockito.mock(Env.class);
49+
Auth auth = Mockito.mock(Auth.class);
50+
Mockito.when(env.getAuth()).thenReturn(auth);
51+
Mockito.when(auth.getSqlBlockRules("user")).thenReturn(new String[] {"missing"});
52+
try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
53+
mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
54+
rules.getNameToSqlBlockRuleMap().put("user_limit", new SqlBlockRule(
55+
"user_limit", "NULL", "NULL", 1L, 0L, 0L, false, false, true));
56+
Assert.assertFalse(rules.hasScanCountLimitations("user"));
57+
Mockito.when(auth.getSqlBlockRules("user")).thenReturn(new String[] {"missing", "user_limit"});
58+
Assert.assertTrue(rules.hasScanCountLimitations("user"));
59+
context.getState().setInternal(true);
60+
Assert.assertFalse(rules.hasScanCountLimitations("user"));
61+
context.getState().setInternal(false);
62+
63+
Mockito.when(auth.getSqlBlockRules("user")).thenReturn(new String[0]);
64+
rules.getNameToSqlBlockRuleMap().put("disabled", new SqlBlockRule(
65+
"disabled", "NULL", "NULL", 1L, 1L, 0L, false, true, false));
66+
rules.getNameToSqlBlockRuleMap().put("metadata_only", new SqlBlockRule(
67+
"metadata_only", "SELECT .*", "NULL", 0L, 0L, 1L, true, true, true));
68+
Assert.assertFalse(rules.hasScanCountLimitations("user"));
69+
rules.getNameToSqlBlockRuleMap().put("global_limit", new SqlBlockRule(
70+
"global_limit", "NULL", "NULL", 0L, 1L, 0L, false, true, true));
71+
Assert.assertTrue(rules.hasScanCountLimitations("user"));
72+
} finally {
73+
ConnectContext.remove();
74+
}
75+
}
76+
3877
@Test
3978
public void testToInfoString() {
4079
SqlBlockRuleMgr mgr = new SqlBlockRuleMgr();

‎fe/fe-core/src/test/java/org/apache/doris/datasource/hive/source/HiveScanNodeTest.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
package org.apache.doris.datasource.hive.source;
1919

20+
import org.apache.doris.analysis.StatementBase;
2021
import org.apache.doris.analysis.TableSample;
2122
import org.apache.doris.analysis.TupleDescriptor;
2223
import org.apache.doris.analysis.TupleId;
@@ -60,6 +61,29 @@
6061
public class HiveScanNodeTest {
6162
private static final long MB = 1024L * 1024L;
6263

64+
@Test
65+
public void testExplainUsesExactFilePlanning() throws Exception {
66+
HiveScanNode node = createHiveScanNode();
67+
Field initialized = HiveScanNode.class.getDeclaredField("partitionInit");
68+
initialized.setAccessible(true);
69+
initialized.set(node, true);
70+
Field partitions = HiveScanNode.class.getDeclaredField("prunedPartitions");
71+
partitions.setAccessible(true);
72+
partitions.set(node, Collections.singletonList(Mockito.mock(HivePartition.class)));
73+
ConnectContext context = new ConnectContext();
74+
StatementContext statementContext = new StatementContext();
75+
StatementBase statement = Mockito.mock(StatementBase.class);
76+
Mockito.when(statement.isExplain()).thenReturn(true);
77+
statementContext.setParsedStatement(statement);
78+
context.setStatementContext(statementContext);
79+
context.setThreadLocalInfo();
80+
try {
81+
Assert.assertFalse(node.isBatchMode());
82+
} finally {
83+
ConnectContext.remove();
84+
}
85+
}
86+
6387
@Test
6488
public void testStatementCacheReusesListingOnlyForSamePartitionIdentity() throws Exception {
6589
ConnectContext previousContext = ConnectContext.get();

‎fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,15 @@
2323
import org.apache.doris.analysis.SlotDescriptor;
2424
import org.apache.doris.analysis.SlotId;
2525
import org.apache.doris.analysis.SlotRef;
26+
import org.apache.doris.analysis.StatementBase;
2627
import org.apache.doris.analysis.TableName;
2728
import org.apache.doris.analysis.TableScanParams;
2829
import org.apache.doris.analysis.TableSnapshot;
2930
import org.apache.doris.analysis.TupleDescriptor;
3031
import org.apache.doris.analysis.TupleId;
32+
import org.apache.doris.analysis.UserIdentity;
33+
import org.apache.doris.blockrule.SqlBlockRule;
34+
import org.apache.doris.blockrule.SqlBlockRuleMgr;
3135
import org.apache.doris.catalog.Column;
3236
import org.apache.doris.catalog.DatabaseIf;
3337
import org.apache.doris.catalog.Env;
@@ -65,6 +69,7 @@
6569
import org.apache.doris.datasource.iceberg.cache.ManifestCacheValue;
6670
import org.apache.doris.datasource.iceberg.helper.IcebergWriterHelper;
6771
import org.apache.doris.datasource.mvcc.MvccTableInfo;
72+
import org.apache.doris.mysql.privilege.Auth;
6873
import org.apache.doris.nereids.StatementContext;
6974
import org.apache.doris.planner.PlanNodeId;
7075
import org.apache.doris.planner.ScanContext;
@@ -103,6 +108,7 @@
103108
import org.apache.iceberg.FileFormat;
104109
import org.apache.iceberg.FileMetadata;
105110
import org.apache.iceberg.FileScanTask;
111+
import org.apache.iceberg.ManifestFile;
106112
import org.apache.iceberg.ManifestFiles;
107113
import org.apache.iceberg.MetadataColumns;
108114
import org.apache.iceberg.MetadataTableType;
@@ -123,6 +129,7 @@
123129
import org.apache.iceberg.expressions.Expressions;
124130
import org.apache.iceberg.hadoop.HadoopTables;
125131
import org.apache.iceberg.io.CloseableIterable;
132+
import org.apache.iceberg.io.FileIO;
126133
import org.apache.iceberg.types.Types;
127134
import org.apache.iceberg.util.ScanTaskUtil;
128135
import org.apache.thrift.TDeserializer;
@@ -169,6 +176,76 @@ public class IcebergScanNodeTest {
169176
@Rule
170177
public final TemporaryFolder temporaryFolder = new TemporaryFolder();
171178

179+
@Test
180+
public void testExplainRequiresExactFilePlanning() throws Exception {
181+
assertBatchPlanningAllowed(true, false, 0, 0, true, false);
182+
}
183+
184+
@Test
185+
public void testSqlScanLimitsRequireExactFilePlanning() throws Exception {
186+
assertBatchPlanningAllowed(false, false, 1, 0, true, false);
187+
assertBatchPlanningAllowed(false, false, 0, 1, true, false);
188+
assertBatchPlanningAllowed(false, true, 1, 0, true, false);
189+
assertBatchPlanningAllowed(false, true, 0, 1, true, false);
190+
}
191+
192+
@Test
193+
public void testUnrestrictedQueriesKeepBatchPlanning() throws Exception {
194+
assertBatchPlanningAllowed(false, false, 0, 0, true, true);
195+
assertBatchPlanningAllowed(false, false, 1, 1, false, true);
196+
}
197+
198+
private void assertBatchPlanningAllowed(boolean explain, boolean global, long partitions, long splits,
199+
boolean enabled, boolean expected) throws Exception {
200+
SessionVariable variables = new SessionVariable();
201+
variables.enableExternalTableBatchMode = true;
202+
variables.numFilesInBatchMode = 1;
203+
TestIcebergScanNode node = new TestIcebergScanNode(variables);
204+
Table table = Mockito.mock(Table.class);
205+
FileIO io = Mockito.mock(FileIO.class);
206+
TableScan scan = Mockito.mock(TableScan.class);
207+
Snapshot snapshot = Mockito.mock(Snapshot.class);
208+
ManifestFile manifest = Mockito.mock(ManifestFile.class);
209+
Mockito.when(table.io()).thenReturn(io);
210+
Mockito.when(table.specs()).thenReturn(Collections.singletonMap(0, PartitionSpec.unpartitioned()));
211+
Mockito.when(scan.table()).thenReturn(table);
212+
Mockito.when(scan.schema()).thenReturn(PartitionSpec.unpartitioned().schema());
213+
Mockito.when(scan.snapshot()).thenReturn(snapshot);
214+
Mockito.when(scan.filter()).thenReturn(Expressions.alwaysTrue());
215+
Mockito.when(snapshot.schemaId()).thenReturn(0);
216+
Mockito.when(snapshot.dataManifests(io)).thenReturn(Collections.singletonList(manifest));
217+
Mockito.when(manifest.partitionSpecId()).thenReturn(0);
218+
Mockito.when(manifest.hasAddedFiles()).thenReturn(true);
219+
Mockito.when(manifest.addedFilesCount()).thenReturn(2);
220+
Mockito.when(manifest.existingFilesCount()).thenReturn(0);
221+
node.setTableScan(scan);
222+
setIcebergTable(node, table);
223+
setPreExecutionAuthenticator(node, new ExecutionAuthenticator() {});
224+
225+
ConnectContext context = new ConnectContext();
226+
context.setCurrentUserIdentity(UserIdentity.createAnalyzedUserIdentWithIp("batch_rule_user", "%"));
227+
StatementContext statementContext = new StatementContext();
228+
StatementBase statement = Mockito.mock(StatementBase.class);
229+
Mockito.when(statement.isExplain()).thenReturn(explain);
230+
statementContext.setParsedStatement(statement);
231+
context.setStatementContext(statementContext);
232+
context.setThreadLocalInfo();
233+
Env env = Mockito.mock(Env.class);
234+
Auth auth = Mockito.mock(Auth.class);
235+
SqlBlockRuleMgr rules = new SqlBlockRuleMgr();
236+
rules.getNameToSqlBlockRuleMap().put("scan_limit", new SqlBlockRule(
237+
"scan_limit", "NULL", "NULL", partitions, splits, 0L, false, global, enabled));
238+
Mockito.when(env.getAuth()).thenReturn(auth);
239+
Mockito.when(env.getSqlBlockRuleMgr()).thenReturn(rules);
240+
Mockito.when(auth.getSqlBlockRules("batch_rule_user")).thenReturn(new String[] {"scan_limit"});
241+
try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
242+
mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
243+
Assert.assertEquals(expected, node.isRealBatchMode());
244+
} finally {
245+
ConnectContext.remove();
246+
}
247+
}
248+
172249
@Test
173250
public void testDoesNotUseHiveParquetInt96TimeZone() {
174251
IcebergScanNode node = Mockito.mock(IcebergScanNode.class, Mockito.CALLS_REAL_METHODS);

0 commit comments

Comments
 (0)