Skip to content

Commit a954b95

Browse files
hubgeterYour Name
authored andcommitted
[Enhancement](explain)Display deleteFileNum for FileScanNode when explain verbose (#60308)
### What problem does this PR solve? Problem Summary: This PR enhances the output of EXPLAIN VERBOSE for File Scan nodes by adding the following metrics: `dataFileNum=xxx, deleteFileNum=xxx, deleteSplitNum=xxx` Especially useful for iceberg/paimon/hive acid These metrics provide more visibility into the underlying file and split layout, helping users better tune parameters and control query performance. Details: `dataFileNum` : The number of distinct data files that need to be read. This is not equivalent to the number of splits, since a single data file can be divided into multiple splits. `deleteFileNum` : The number of distinct delete files that need to be read. `deleteSplitNum` : Added because the relationship between data files and delete files is many-to-many: one data file may be associated with multiple delete files one delete file may apply to multiple data files Using deleteSplitNum / dataSplitNum, users can estimate the average number of delete splits that need to be read per data split. Example: ``` mysql> explain verbose select * from iceberg.format_v3.dv_test_1w; +-----------------------------------------------------------------------------------------------------------------------------------------------+ | Explain String(Nereids Planner) | +-----------------------------------------------------------------------------------------------------------------------------------------------+ | PLAN FRAGMENT 0 | | OUTPUT EXPRS: | | id[#0] | | grp[#1] | | value[#2] | | ts[#3] | | PARTITION: RANDOM | | | | HAS_COLO_PLAN_NODE: false | | | | VRESULT SINK | | MYSQL_PROTOCOL | | | | 0:VICEBERG_SCAN_NODE(32) | | table: iceberg.format_v3.dv_test_1w | | inputSplitNum=220, totalFileSize=720774, scanRanges=220 | | partition=0/0 | | backends: | | 1769590309070 | | s3://warehouse/wh/format_v3/dv_test_1w/data/00004-51-fc462f9a-d42a-404d-adfc-c8d2781c8d04-0-00001.parquet start: 4 length: 2672 | | s3://warehouse/wh/format_v3/dv_test_1w/data/00003-50-fc462f9a-d42a-404d-adfc-c8d2781c8d04-0-00001.parquet start: 4 length: 2852 | | s3://warehouse/wh/format_v3/dv_test_1w/data/00000-47-fc462f9a-d42a-404d-adfc-c8d2781c8d04-0-00001.parquet start: 4 length: 2894 | | ... other 216 files ... | | s3://warehouse/wh/format_v3/dv_test_1w/data/00001-48-fc462f9a-d42a-404d-adfc-c8d2781c8d04-0-00001.parquet start: 58397 length: 13894 | | dataFileNum=10, deleteFileNum=1 deleteSplitNum=220 | | cardinality=33334, numNodes=1 | | pushdown agg=NONE | | tuple ids: 0 | | | | Tuples: | | TupleDescriptor{id=0, tbl=dv_test_1w} | | SlotDescriptor{id=0, col=id, colUniqueId=1, type=bigint, nullable=true, isAutoIncrement=false, subColPath=null, virtualColumn=null} | | SlotDescriptor{id=1, col=grp, colUniqueId=2, type=int, nullable=true, isAutoIncrement=false, subColPath=null, virtualColumn=null} | | SlotDescriptor{id=2, col=value, colUniqueId=3, type=int, nullable=true, isAutoIncrement=false, subColPath=null, virtualColumn=null} | | SlotDescriptor{id=3, col=ts, colUniqueId=4, type=datetimev2(6), nullable=true, isAutoIncrement=false, subColPath=null, virtualColumn=null} | | | | | | | | | | ========== STATISTICS ========== | +-----------------------------------------------------------------------------------------------------------------------------------------------+ ```
1 parent 9512b4a commit a954b95

7 files changed

Lines changed: 152 additions & 0 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,8 +55,10 @@
5555

5656
import java.util.Collections;
5757
import java.util.Comparator;
58+
import java.util.HashSet;
5859
import java.util.List;
5960
import java.util.Map;
61+
import java.util.Set;
6062
import java.util.stream.Collectors;
6163

6264
/**
@@ -104,6 +106,17 @@ public long getTotalFileSize() {
104106
return totalFileSize;
105107
}
106108

109+
/**
110+
* Get all delete files for the given file range.
111+
* @param rangeDesc the file range descriptor
112+
* @return list of delete file paths (formatted strings)
113+
*/
114+
protected List<String> getDeleteFiles(TFileRangeDesc rangeDesc) {
115+
// Default implementation: return empty list
116+
// Subclasses should override this method
117+
return Collections.emptyList();
118+
}
119+
107120
@Override
108121
public String getNodeExplainString(String prefix, TExplainLevel detailLevel) {
109122
StringBuilder output = new StringBuilder();
@@ -149,6 +162,21 @@ public int compare(TFileRangeDesc o1, TFileRangeDesc o2) {
149162
return Long.compare(o1.getStartOffset(), o2.getStartOffset());
150163
}
151164
});
165+
166+
// A Data file may be divided into different splits, so a set is used to remove duplicates.
167+
Set<String> dataFilesSet = new HashSet<>();
168+
// A delete file might be used by multiple data files, so use set to remove duplicates.
169+
Set<String> deleteFilesSet = new HashSet<>();
170+
// You can estimate how many delete splits need to be read for a data split
171+
// using deleteSplitNum / dataSplitNum(fileRangeDescs.size()) split.
172+
long deleteSplitNum = 0;
173+
for (TFileRangeDesc fileRangeDesc : fileRangeDescs) {
174+
dataFilesSet.add(fileRangeDesc.getPath());
175+
List<String> deletefiles = getDeleteFiles(fileRangeDesc);
176+
deleteFilesSet.addAll(deletefiles);
177+
deleteSplitNum += deletefiles.size();
178+
}
179+
152180
// 3. if size <= 4, print all. if size > 4, print first 3 and last 1
153181
int size = fileRangeDescs.size();
154182
if (size <= 4) {
@@ -174,6 +202,10 @@ public int compare(TFileRangeDesc o1, TFileRangeDesc o2) {
174202
.append(" length: ").append(file.getSize())
175203
.append("\n");
176204
}
205+
output.append(prefix).append(" ").append("dataFileNum=").append(dataFilesSet.size())
206+
.append(", deleteFileNum=").append(deleteFilesSet.size())
207+
.append(", deleteSplitNum=").append(deleteSplitNum)
208+
.append("\n");
177209
}
178210
}
179211

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -443,6 +443,37 @@ protected void setScanParams(TFileRangeDesc rangeDesc, Split split) {
443443
}
444444
}
445445

446+
@Override
447+
protected List<String> getDeleteFiles(TFileRangeDesc rangeDesc) {
448+
List<String> deleteFiles = new ArrayList<>();
449+
if (rangeDesc == null || !rangeDesc.isSetTableFormatParams()) {
450+
return deleteFiles;
451+
}
452+
TTableFormatFileDesc tableFormatParams = rangeDesc.getTableFormatParams();
453+
if (tableFormatParams == null || !tableFormatParams.isSetTransactionalHiveParams()) {
454+
return deleteFiles;
455+
}
456+
TTransactionalHiveDesc hiveParams = tableFormatParams.getTransactionalHiveParams();
457+
if (hiveParams == null || !hiveParams.isSetDeleteDeltas()) {
458+
return deleteFiles;
459+
}
460+
List<TTransactionalHiveDeleteDeltaDesc> deleteDeltas = hiveParams.getDeleteDeltas();
461+
if (deleteDeltas == null) {
462+
return deleteFiles;
463+
}
464+
// Format: {directory_location}/{file_name}
465+
for (TTransactionalHiveDeleteDeltaDesc deleteDelta : deleteDeltas) {
466+
if (deleteDelta != null && deleteDelta.isSetDirectoryLocation()
467+
&& deleteDelta.isSetFileNames() && deleteDelta.getFileNames() != null) {
468+
String directoryLocation = deleteDelta.getDirectoryLocation();
469+
for (String fileName : deleteDelta.getFileNames()) {
470+
deleteFiles.add(directoryLocation + "/" + fileName);
471+
}
472+
}
473+
}
474+
return deleteFiles;
475+
}
476+
446477
@Override
447478
protected Map<String, String> getLocationProperties() {
448479
return hmsTable.getBackendStorageProperties();

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -269,6 +269,46 @@ private void setIcebergParams(TFileRangeDesc rangeDesc, IcebergSplit icebergSpli
269269
rangeDesc.setTableFormatParams(tableFormatFileDesc);
270270
}
271271

272+
@Override
273+
protected List<String> getDeleteFiles(TFileRangeDesc rangeDesc) {
274+
List<String> deleteFiles = new ArrayList<>();
275+
if (rangeDesc == null || !rangeDesc.isSetTableFormatParams()) {
276+
return deleteFiles;
277+
}
278+
TTableFormatFileDesc tableFormatParams = rangeDesc.getTableFormatParams();
279+
if (tableFormatParams == null || !tableFormatParams.isSetIcebergParams()) {
280+
return deleteFiles;
281+
}
282+
TIcebergFileDesc icebergParams = tableFormatParams.getIcebergParams();
283+
if (icebergParams == null || !icebergParams.isSetDeleteFiles()) {
284+
return deleteFiles;
285+
}
286+
List<TIcebergDeleteFileDesc> icebergDeleteFiles = icebergParams.getDeleteFiles();
287+
if (icebergDeleteFiles == null) {
288+
return deleteFiles;
289+
}
290+
for (TIcebergDeleteFileDesc deleteFile : icebergDeleteFiles) {
291+
if (deleteFile != null && deleteFile.isSetPath()) {
292+
deleteFiles.add(deleteFile.getPath());
293+
}
294+
}
295+
return deleteFiles;
296+
}
297+
298+
private String getDeleteFileContentType(int content) {
299+
// Iceberg file type: 0: data, 1: position delete, 2: equality delete, 3: deletion vector
300+
switch (content) {
301+
case 1:
302+
return "position_delete";
303+
case 2:
304+
return "equality_delete";
305+
case 3:
306+
return "deletion_vector";
307+
default:
308+
return "unknown";
309+
}
310+
}
311+
272312
@Override
273313
public List<Split> getSplits(int numBackends) throws UserException {
274314
try {

‎fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -276,6 +276,28 @@ private void setPaimonParams(TFileRangeDesc rangeDesc, PaimonSplit paimonSplit)
276276
rangeDesc.setTableFormatParams(tableFormatFileDesc);
277277
}
278278

279+
@Override
280+
protected List<String> getDeleteFiles(TFileRangeDesc rangeDesc) {
281+
List<String> deleteFiles = new ArrayList<>();
282+
if (rangeDesc == null || !rangeDesc.isSetTableFormatParams()) {
283+
return deleteFiles;
284+
}
285+
TTableFormatFileDesc tableFormatParams = rangeDesc.getTableFormatParams();
286+
if (tableFormatParams == null || !tableFormatParams.isSetPaimonParams()) {
287+
return deleteFiles;
288+
}
289+
TPaimonFileDesc paimonParams = tableFormatParams.getPaimonParams();
290+
if (paimonParams == null || !paimonParams.isSetDeletionFile()) {
291+
return deleteFiles;
292+
}
293+
TPaimonDeletionFileDesc deletionFile = paimonParams.getDeletionFile();
294+
if (deletionFile != null && deletionFile.isSetPath()) {
295+
// Format: path [offset: offset, length: length]
296+
deleteFiles.add(deletionFile.getPath());
297+
}
298+
return deleteFiles;
299+
}
300+
279301
@Override
280302
public List<Split> getSplits(int numBackends) throws UserException {
281303
boolean forceJniScanner = sessionVariable.isForceJniScanner();

‎regression-test/suites/external_table_p0/hive/test_transactional_hive.groovy‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,13 @@ suite("test_transactional_hive", "p0,external,hive,external_docker,external_dock
142142
qt_count_5 """ select count(*) from orc_acid_major; """ //3
143143
}
144144

145+
def test_explain_verbose = {
146+
explain {
147+
sql ("select count(*) from orc_full_acid")
148+
verbose (true)
149+
contains "deleteFileNum"
150+
}
151+
}
145152

146153
String enabled = context.config.otherConfigs.get("enableHiveTest")
147154
if (enabled == null || !enabled.equalsIgnoreCase("true")) {
@@ -177,6 +184,7 @@ suite("test_transactional_hive", "p0,external,hive,external_docker,external_dock
177184

178185

179186
test_acid_count()
187+
test_explain_verbose()
180188

181189
q01_par_limit()
182190

‎regression-test/suites/external_table_p0/iceberg/test_iceberg_position_delete.groovy‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,16 @@ suite("test_iceberg_position_delete", "p0,external,doris,external_docker,externa
165165
assertTrue(iceberg_position_gen_7.size() == 5632)
166166

167167
// sql """drop catalog ${catalog_name}"""
168+
169+
def test_explain_verbose = {
170+
explain {
171+
sql ("select name from iceberg_position_gen_data where id != 5;")
172+
verbose (true)
173+
contains "deleteFileNum"
174+
}
175+
}
176+
test_explain_verbose()
177+
168178
}
169179
/*
170180

‎regression-test/suites/external_table_p0/paimon/test_paimon_deletion_vector_oss.groovy‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,8 +53,17 @@ suite("test_paimon_deletion_vector_oss", "p0,external,doris,external_docker,exte
5353
qt_6 """select * from deletion_vector_parquet where id > 2 order by id;"""
5454
}
5555

56+
def test_explain_verbose = {
57+
explain {
58+
sql ("select * from deletion_vector_orc;")
59+
verbose (true)
60+
contains "deleteFileNum"
61+
}
62+
}
63+
5664
test_cases("false")
5765
test_cases("true")
66+
test_explain_verbose()
5867

5968
} finally {
6069
sql """set force_jni_scanner=false"""

0 commit comments

Comments
 (0)