Skip to content

Commit d8e75cf

Browse files
[fix](iceberg) Project Iceberg system table scans (#65262)
Iceberg system table scans serialize Iceberg SDK `FileScanTask` objects and the Java metadata scanner materializes rows from the task schema. Doris did not pass the actually required system table columns into the Iceberg SDK scan, so metadata tables such as `$files` and `$data_files` were planned with the full schema. For files metadata tables, the full schema can include virtual `readable_metrics` even when the SQL only requests columns such as `file_size_in_bytes`. This PR applies SDK top-level column projection only for Iceberg system table scans. Normal Iceberg data table scans continue to rely on Doris BE scan range params for column pruning. ### What changed? - Add system-table-only `TableScan.select(...)` projection in `IcebergScanNode` before planning the Iceberg SDK scan. - Skip synthesized/global row id and Iceberg row lineage columns when building the SDK projection list. - Add regression coverage for ORC and Parquet Iceberg tables with `map<boolean, boolean>` columns, verifying projected `$data_files` and `$files` queries.
1 parent 662dc35 commit d8e75cf

8 files changed

Lines changed: 367 additions & 43 deletions

File tree

‎fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableJniScanner.java‎

Lines changed: 18 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -28,15 +28,11 @@
2828
import org.apache.iceberg.FileScanTask;
2929
import org.apache.iceberg.StructLike;
3030
import org.apache.iceberg.io.CloseableIterator;
31-
import org.apache.iceberg.types.Types.NestedField;
32-
import org.apache.iceberg.types.Types.StructType;
3331
import org.apache.iceberg.util.SerializationUtil;
3432
import org.slf4j.Logger;
3533
import org.slf4j.LoggerFactory;
3634

3735
import java.io.IOException;
38-
import java.util.ArrayList;
39-
import java.util.List;
4036
import java.util.Map;
4137
import java.util.TimeZone;
4238
import java.util.stream.Collectors;
@@ -50,7 +46,7 @@ public class IcebergSysTableJniScanner extends JniScanner {
5046
private final ClassLoader classLoader;
5147
private final PreExecutionAuthenticator preExecutionAuthenticator;
5248
private final FileScanTask scanTask;
53-
private final List<SelectedField> fields;
49+
private final int requiredFieldCount;
5450
private final String timezone;
5551
private CloseableIterator<StructLike> reader;
5652

@@ -60,15 +56,22 @@ public IcebergSysTableJniScanner(int batchSize, Map<String, String> params) {
6056
Preconditions.checkArgument(serializedSplitParams != null && !serializedSplitParams.isEmpty(),
6157
"serialized_split should not be empty");
6258
this.scanTask = SerializationUtil.deserializeFromBase64(serializedSplitParams);
63-
String[] requiredFields = params.get("required_fields").split(",");
64-
this.fields = selectSchema(scanTask.schema().asStruct(), requiredFields);
59+
String requiredFieldsParam = params.get("required_fields");
60+
Preconditions.checkArgument(requiredFieldsParam != null && !requiredFieldsParam.isEmpty(),
61+
"required_fields should not be empty");
62+
String[] requiredFields = requiredFieldsParam.split(",");
63+
this.requiredFieldCount = requiredFields.length;
6564
this.timezone = params.getOrDefault("time_zone", TimeZone.getDefault().getID());
6665
Map<String, String> hadoopOptionParams = params.entrySet().stream()
6766
.filter(kv -> kv.getKey().startsWith(HADOOP_OPTION_PREFIX))
6867
.collect(Collectors
6968
.toMap(kv1 -> kv1.getKey().substring(HADOOP_OPTION_PREFIX.length()), kv1 -> kv1.getValue()));
7069
this.preExecutionAuthenticator = PreExecutionAuthenticatorCache.getAuthenticator(hadoopOptionParams);
71-
ColumnType[] requiredTypes = parseRequiredTypes(params.get("required_types").split("#"), requiredFields);
70+
String requiredTypesParam = params.get("required_types");
71+
Preconditions.checkArgument(requiredTypesParam != null && !requiredTypesParam.isEmpty(),
72+
"required_types should not be empty");
73+
String[] requiredTypeStrings = requiredTypesParam.split("#");
74+
ColumnType[] requiredTypes = parseRequiredTypes(requiredTypeStrings, requiredFields);
7275
initTableInfo(requiredTypes, requiredFields, batchSize);
7376
}
7477

@@ -106,9 +109,10 @@ protected int getNext() throws IOException {
106109
break;
107110
}
108111
StructLike row = reader.next();
109-
for (int i = 0; i < fields.size(); i++) {
110-
SelectedField field = fields.get(i);
111-
Object value = row.get(field.sourceIndex, field.field.type().typeId().javaClass());
112+
for (int i = 0; i < requiredFieldCount; i++) {
113+
// FE keeps the fields requested by BE at the start of the Iceberg projection.
114+
// FileScanTask.schema() is not the row schema for every DataTask implementation.
115+
Object value = row.get(i, Object.class);
112116
ColumnValue columnValue = new IcebergSysTableColumnValue(value, timezone);
113117
appendData(i, columnValue);
114118
}
@@ -129,35 +133,10 @@ public void close() throws IOException {
129133
}
130134
}
131135

132-
private static List<SelectedField> selectSchema(StructType schema, String[] requiredFields) {
133-
List<NestedField> schemaFields = schema.fields();
134-
List<SelectedField> selectedFields = new ArrayList<>();
135-
for (String requiredField : requiredFields) {
136-
NestedField field = schema.field(requiredField);
137-
if (field == null) {
138-
throw new IllegalArgumentException("RequiredField " + requiredField + " not found in schema");
139-
}
140-
int sourceIndex = schemaFields.indexOf(field);
141-
if (sourceIndex < 0) {
142-
throw new IllegalArgumentException(
143-
"RequiredField " + requiredField + " not found in source schema fields");
144-
}
145-
selectedFields.add(new SelectedField(sourceIndex, field));
146-
}
147-
return selectedFields;
148-
}
149-
150-
private static final class SelectedField {
151-
private final int sourceIndex;
152-
private final NestedField field;
153-
154-
private SelectedField(int sourceIndex, NestedField field) {
155-
this.sourceIndex = sourceIndex;
156-
this.field = field;
157-
}
158-
}
159-
160136
private static ColumnType[] parseRequiredTypes(String[] typeStrings, String[] requiredFields) {
137+
Preconditions.checkArgument(typeStrings.length == requiredFields.length,
138+
"required_types size %s does not match required_fields size %s",
139+
typeStrings.length, requiredFields.length);
161140
ColumnType[] requiredTypes = new ColumnType[typeStrings.length];
162141
for (int i = 0; i < typeStrings.length; i++) {
163142
String type = typeStrings[i];

‎fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -181,7 +181,7 @@ protected void initSchemaParams() throws UserException {
181181
slotInfo.setSlotId(slot.getId().asInt());
182182
TColumnCategory category = classifyColumn(slot, partitionKeys);
183183
slotInfo.setCategory(category);
184-
slotInfo.setIsFileSlot(category == TColumnCategory.REGULAR || category == TColumnCategory.GENERATED);
184+
slotInfo.setIsFileSlot(isFileSlot(category));
185185
params.addToRequiredSlots(slotInfo);
186186
}
187187
setDefaultValueExprs(getTargetTable(), destSlotDescByName, null, params, false);
@@ -210,7 +210,7 @@ private void updateRequiredSlots() throws UserException {
210210
slotInfo.setSlotId(slot.getId().asInt());
211211
TColumnCategory category = classifyColumn(slot, partitionKeys);
212212
slotInfo.setCategory(category);
213-
slotInfo.setIsFileSlot(category == TColumnCategory.REGULAR || category == TColumnCategory.GENERATED);
213+
slotInfo.setIsFileSlot(isFileSlot(category));
214214
params.addToRequiredSlots(slotInfo);
215215
}
216216
// Update required slots and column_idxs in scanRangeLocations.
@@ -228,6 +228,10 @@ protected TColumnCategory classifyColumn(SlotDescriptor slot, List<String> parti
228228
return TColumnCategory.REGULAR;
229229
}
230230

231+
protected boolean isFileSlot(TColumnCategory category) {
232+
return category == TColumnCategory.REGULAR || category == TColumnCategory.GENERATED;
233+
}
234+
231235
public void setTableSample(TableSample tSample) {
232236
this.tableSample = tSample;
233237
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTable.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,10 @@ public String getSysTableType() {
8080
return sysTableType;
8181
}
8282

83+
public boolean isPositionDeletesTable() {
84+
return MetadataTableType.POSITION_DELETES.name().equalsIgnoreCase(sysTableType);
85+
}
86+
8387
public Table getSysIcebergTable() {
8488
if (sysIcebergTable == null) {
8589
synchronized (this) {

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java‎

Lines changed: 64 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@
9696
import org.apache.iceberg.Table;
9797
import org.apache.iceberg.TableProperties;
9898
import org.apache.iceberg.TableScan;
99+
import org.apache.iceberg.expressions.Binder;
99100
import org.apache.iceberg.expressions.Expression;
100101
import org.apache.iceberg.expressions.Expressions;
101102
import org.apache.iceberg.expressions.InclusiveMetricsEvaluator;
@@ -108,6 +109,7 @@
108109
import org.apache.iceberg.mapping.NameMapping;
109110
import org.apache.iceberg.mapping.NameMappingParser;
110111
import org.apache.iceberg.types.Type;
112+
import org.apache.iceberg.types.TypeUtil;
111113
import org.apache.iceberg.types.Types.NestedField;
112114
import org.apache.iceberg.util.ScanTaskUtil;
113115
import org.apache.iceberg.util.SerializationUtil;
@@ -121,11 +123,13 @@
121123
import java.util.Arrays;
122124
import java.util.Collections;
123125
import java.util.HashMap;
126+
import java.util.HashSet;
124127
import java.util.List;
125128
import java.util.Map;
126129
import java.util.Objects;
127130
import java.util.Optional;
128131
import java.util.OptionalLong;
132+
import java.util.Set;
129133
import java.util.concurrent.CompletableFuture;
130134
import java.util.concurrent.atomic.AtomicReference;
131135

@@ -674,11 +678,70 @@ public TableScan createTableScan() throws UserException {
674678
this.pushdownIcebergPredicates.add(predicate.toString());
675679
}
676680

681+
// Doris reads normal Iceberg table files in BE and applies column pruning through scan range params.
682+
// System tables are different: Iceberg SDK DataTask materializes rows using the projected scan
683+
// schema. Keep Doris file slots in the same order as the JNI reader's required fields.
684+
if (isSystemTable) {
685+
Schema projectedSchema = getSystemTableProjectedSchema(expressions, scan.isCaseSensitive());
686+
Preconditions.checkState(!projectedSchema.columns().isEmpty(),
687+
"Iceberg system table scan must materialize at least one file slot");
688+
scan = scan.project(projectedSchema);
689+
}
690+
677691
icebergTableScan = scan.planWith(source.getCatalog().getThreadPoolWithPreAuth());
678692

679693
return icebergTableScan;
680694
}
681695

696+
@VisibleForTesting
697+
Schema getSystemTableProjectedSchema(List<Expression> expressions, boolean caseSensitive)
698+
throws UserException {
699+
List<NestedField> projectedFields = new ArrayList<>();
700+
Set<Integer> projectedFieldIds = new HashSet<>();
701+
List<String> partitionKeys = getPathPartitionKeys();
702+
for (SlotDescriptor slot : desc.getSlots()) {
703+
Column column = slot.getColumn();
704+
String columnName = column.getName();
705+
if (!isFileSlot(classifyColumn(slot, partitionKeys))) {
706+
continue;
707+
}
708+
709+
NestedField field = caseSensitive
710+
? icebergTable.schema().findField(columnName)
711+
: icebergTable.schema().caseInsensitiveFindField(columnName);
712+
if (field == null) {
713+
throw new UserException("Column " + columnName + " not found in Iceberg system table schema");
714+
}
715+
if (projectedFieldIds.add(field.fieldId())) {
716+
projectedFields.add(field);
717+
}
718+
}
719+
720+
Set<Integer> filterFieldIds = Binder.boundReferences(
721+
icebergTable.schema().asStruct(), expressions, caseSensitive);
722+
for (Integer fieldId : filterFieldIds) {
723+
NestedField field = getTopLevelSystemTableField(fieldId);
724+
if (field == null) {
725+
throw new UserException(
726+
"Column with field id " + fieldId + " not found in Iceberg system table schema");
727+
}
728+
if (!projectedFieldIds.contains(field.fieldId())) {
729+
throw new UserException("Iceberg system table filter column " + field.name()
730+
+ " is not materialized by the planner");
731+
}
732+
}
733+
return new Schema(projectedFields);
734+
}
735+
736+
private NestedField getTopLevelSystemTableField(int fieldId) {
737+
for (NestedField field : icebergTable.schema().columns()) {
738+
if (field.fieldId() == fieldId || TypeUtil.getProjectedIds(field.type()).contains(fieldId)) {
739+
return field;
740+
}
741+
}
742+
return null;
743+
}
744+
682745
private CloseableIterable<FileScanTask> planFileScanTask(TableScan scan) {
683746
if (!IcebergUtils.isManifestCacheEnabled(source.getCatalog())) {
684747
return splitFiles(scan);
@@ -1199,7 +1262,7 @@ private List<Split> doGetSystemTableSplits() throws UserException {
11991262
private boolean isPositionDeletesSystemTable() {
12001263
TableIf targetTable = source.getTargetTable();
12011264
return targetTable instanceof IcebergSysExternalTable
1202-
&& "position_deletes".equalsIgnoreCase(((IcebergSysExternalTable) targetTable).getSysTableType());
1265+
&& ((IcebergSysExternalTable) targetTable).isPositionDeletesTable();
12031266
}
12041267

12051268
private List<Split> doGetPositionDeletesSystemTableSplits() throws UserException {

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -233,8 +233,13 @@ public List<Slot> computeAsteriskOutput() {
233233
@Override
234234
public boolean supportPruneNestedColumn() {
235235
ExternalTable table = getTable();
236-
if (table instanceof IcebergExternalTable || table instanceof IcebergSysExternalTable) {
236+
if (table instanceof IcebergExternalTable) {
237237
return true;
238+
} else if (table instanceof IcebergSysExternalTable) {
239+
// Position deletes use the native reader, which supports nested column pruning. Other
240+
// Iceberg system tables are materialized as StructLike rows by the SDK and consumed by
241+
// ordinal in the JNI reader, so their nested struct layout must remain unchanged.
242+
return ((IcebergSysExternalTable) table).isPositionDeletesTable();
238243
} else if (table instanceof HMSExternalTable) {
239244
HMSExternalTable hmsTable = (HMSExternalTable) table;
240245
if (hmsTable.getDlaType() == HMSExternalTable.DLAType.HUDI) {

‎fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,12 @@
1717

1818
package org.apache.doris.datasource.iceberg.source;
1919

20+
import org.apache.doris.analysis.SlotDescriptor;
21+
import org.apache.doris.analysis.SlotId;
2022
import org.apache.doris.analysis.TupleDescriptor;
2123
import org.apache.doris.analysis.TupleId;
24+
import org.apache.doris.catalog.Column;
25+
import org.apache.doris.catalog.Type;
2226
import org.apache.doris.common.UserException;
2327
import org.apache.doris.common.util.LocationPath;
2428
import org.apache.doris.datasource.TableFormatType;
@@ -38,6 +42,8 @@
3842
import org.apache.iceberg.Snapshot;
3943
import org.apache.iceberg.Table;
4044
import org.apache.iceberg.TableScan;
45+
import org.apache.iceberg.expressions.Expression;
46+
import org.apache.iceberg.expressions.Expressions;
4147
import org.apache.iceberg.types.Types;
4248
import org.apache.iceberg.util.ScanTaskUtil;
4349
import org.junit.Assert;
@@ -88,6 +94,64 @@ public boolean isBatchMode() {
8894
protected boolean getEnableMappingVarbinary() {
8995
return enableMappingVarbinary;
9096
}
97+
98+
@Override
99+
public List<String> getPathPartitionKeys() {
100+
return Collections.emptyList();
101+
}
102+
103+
void addSlot(int slotId, Column column) {
104+
SlotDescriptor slot = new SlotDescriptor(new SlotId(slotId), desc.getId());
105+
slot.setColumn(column);
106+
desc.addSlot(slot);
107+
}
108+
}
109+
110+
@Test
111+
public void testSystemTableProjectionMatchesFileSlotOrder() throws Exception {
112+
Schema systemTableSchema = new Schema(
113+
Types.NestedField.required(1, "file_path", Types.StringType.get()),
114+
Types.NestedField.required(2, "record_count", Types.LongType.get()),
115+
Types.NestedField.optional(3, "readable_metrics", Types.StructType.of(
116+
Types.NestedField.optional(4, "id", Types.StructType.of(
117+
Types.NestedField.optional(5, "lower_bound", Types.IntegerType.get()))))));
118+
Table systemTable = Mockito.mock(Table.class);
119+
Mockito.when(systemTable.schema()).thenReturn(systemTableSchema);
120+
121+
TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable());
122+
setIcebergTable(node, systemTable);
123+
node.addSlot(1, new Column("RECORD_COUNT", Type.BIGINT));
124+
node.addSlot(2, new Column(Column.GLOBAL_ROWID_COL + "system_table", Type.BIGINT));
125+
node.addSlot(3, new Column("FILE_PATH", Type.STRING));
126+
127+
List<Expression> filters = Collections.singletonList(Expressions.greaterThan("record_count", 0L));
128+
Schema projectedSchema = node.getSystemTableProjectedSchema(filters, false);
129+
130+
Assert.assertEquals(2, projectedSchema.columns().size());
131+
Assert.assertEquals("record_count", projectedSchema.columns().get(0).name());
132+
Assert.assertEquals("file_path", projectedSchema.columns().get(1).name());
133+
Assert.assertNull(projectedSchema.findField("readable_metrics"));
134+
}
135+
136+
@Test
137+
public void testSystemTableProjectionRejectsUnmaterializedFilterColumn() throws Exception {
138+
Schema systemTableSchema = new Schema(
139+
Types.NestedField.required(1, "file_path", Types.StringType.get()),
140+
Types.NestedField.required(2, "record_count", Types.LongType.get()));
141+
Table systemTable = Mockito.mock(Table.class);
142+
Mockito.when(systemTable.schema()).thenReturn(systemTableSchema);
143+
144+
TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable());
145+
setIcebergTable(node, systemTable);
146+
node.addSlot(1, new Column("record_count", Type.BIGINT));
147+
148+
try {
149+
node.getSystemTableProjectedSchema(
150+
Collections.singletonList(Expressions.equal("file_path", "data.parquet")), true);
151+
Assert.fail("Filter columns must be materialized by the planner");
152+
} catch (UserException e) {
153+
Assert.assertTrue(e.getMessage().contains("filter column file_path is not materialized"));
154+
}
91155
}
92156

93157
@Test
@@ -142,6 +206,12 @@ public void testInitialDefaultMetadataUsesSystemTableSchemaWithoutTableScan() th
142206
Mockito.verify(node, Mockito.never()).createTableScan();
143207
}
144208

209+
private static void setIcebergTable(IcebergScanNode node, Table table) throws Exception {
210+
Field icebergTableField = IcebergScanNode.class.getDeclaredField("icebergTable");
211+
icebergTableField.setAccessible(true);
212+
icebergTableField.set(node, table);
213+
}
214+
145215
@Test
146216
public void testDetermineTargetFileSplitSizeHonorsMaxFileSplitNum() throws Exception {
147217
SessionVariable sv = new SessionVariable();

0 commit comments

Comments
 (0)