Skip to content

Commit dfe6bac

Browse files
committed
[fix](connector) Harden metadata column compatibility
### What problem does this PR solve? Issue Number: close apache#66816 Related PR: apache#67207 Problem Summary: Iceberg and Paimon row-location metadata columns must remain correct across native FileScannerV2 scans, streaming split planning, schema evolution, and externally-created tables. The previous paths could silently replace or misclassify physical columns with metadata-like names, or route metadata projections through schema-evolution and legacy readers that cannot materialize file and row positions. This change rejects unsafe physical-name collisions at connector metadata assembly, keeps synthetic Paimon handles out of physical projections and schema evolution, validates Iceberg metadata columns on streaming and eager paths, fences scans to the current backend semantics, and makes BE slot classification depend on the explicit synthesized category. Targeted connector and BE tests cover these paths. ### Release note None ### Check List (For Author) - Test: Iceberg connector tests (253 passed), Paimon connector tests (118 passed), BE FileScannerV2 classification tests (2 passed), format check, build hygiene check - Behavior changed: Yes (correctly rejects unsafe external metadata-name collisions and prevents unsupported reader fallback) - Does this need documentation: No
1 parent 68e9df4 commit dfe6bac

10 files changed

Lines changed: 199 additions & 47 deletions

File tree

‎be/src/exec/scan/file_scanner_v2.cpp‎

Lines changed: 6 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,6 @@
7575
#include "runtime/runtime_state.h"
7676
#include "service/backend_options.h"
7777
#include "storage/id_manager.h"
78-
#include "util/string_util.h"
7978

8079
namespace doris {
8180
namespace {
@@ -196,27 +195,14 @@ bool is_wal_format(TFileFormatType::type format_type) {
196195
}
197196

198197
bool is_partition_slot(const TFileScanSlotInfo& slot_info, const std::string& column_name) {
199-
if (column_name.starts_with(BeConsts::GLOBAL_ROWID_COL) ||
200-
iequal(column_name, BeConsts::ICEBERG_ROWID_COL) ||
201-
iequal(column_name, BeConsts::ICEBERG_FILE_PATH_COL) ||
202-
iequal(column_name, BeConsts::ICEBERG_ROW_POSITION_COL) ||
203-
iequal(column_name, BeConsts::PAIMON_FILE_PATH_COL) ||
204-
iequal(column_name, BeConsts::PAIMON_ROW_POSITION_COL)) {
205-
return false;
198+
if (slot_info.__isset.category) {
199+
return slot_info.category == TColumnCategory::PARTITION_KEY;
206200
}
207-
return slot_info.__isset.category ? slot_info.category == TColumnCategory::PARTITION_KEY
208-
: !slot_info.is_file_slot;
201+
return !slot_info.is_file_slot && !column_name.starts_with(BeConsts::GLOBAL_ROWID_COL) &&
202+
column_name != BeConsts::ICEBERG_ROWID_COL;
209203
}
210204

211205
bool is_data_file_slot(const TFileScanSlotInfo& slot_info, const std::string& column_name) {
212-
if (column_name.starts_with(BeConsts::GLOBAL_ROWID_COL) ||
213-
iequal(column_name, BeConsts::ICEBERG_ROWID_COL) ||
214-
iequal(column_name, BeConsts::ICEBERG_FILE_PATH_COL) ||
215-
iequal(column_name, BeConsts::ICEBERG_ROW_POSITION_COL) ||
216-
iequal(column_name, BeConsts::PAIMON_FILE_PATH_COL) ||
217-
iequal(column_name, BeConsts::PAIMON_ROW_POSITION_COL)) {
218-
return false;
219-
}
220206
// CSV and other non-self-describing formats need FE slot descriptors for only the columns that
221207
// are physically read from the file. Partition/default/virtual columns stay in TableReader's
222208
// mapping layer and are materialized after the file-local block is read. New FE provides an
@@ -225,7 +211,8 @@ bool is_data_file_slot(const TFileScanSlotInfo& slot_info, const std::string& co
225211
return slot_info.category == TColumnCategory::REGULAR ||
226212
slot_info.category == TColumnCategory::GENERATED;
227213
}
228-
return slot_info.is_file_slot;
214+
return slot_info.is_file_slot && !column_name.starts_with(BeConsts::GLOBAL_ROWID_COL) &&
215+
column_name != BeConsts::ICEBERG_ROWID_COL;
229216
}
230217

231218
Status rewrite_slot_refs_to_global_index(

‎be/test/exec/scan/file_scanner_v2_test.cpp‎

Lines changed: 33 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -875,8 +875,8 @@ TEST(FileScannerV2Test, OrcScannerResidualFilterRetainsNextBatchContext) {
875875
}
876876

877877
// Scenario: partition slots are identified from the explicit FE category when present, otherwise
878-
// from the legacy is_file_slot flag. Scanner-generated rowid columns must never be treated as
879-
// partition columns even if FE marks them as non-file slots.
878+
// from the legacy is_file_slot flag. Metadata columns must be classified by the explicit SYNTHESIZED
879+
// category so ordinary physical or partition columns with the same spelling remain readable.
880880
TEST(FileScannerV2Test, PartitionSlotClassificationMatrix) {
881881
TFileScanSlotInfo legacy_partition;
882882
legacy_partition.__set_is_file_slot(false);
@@ -900,14 +900,23 @@ TEST(FileScannerV2Test, PartitionSlotClassificationMatrix) {
900900
FileScannerV2::TEST_is_partition_slot(legacy_partition, BeConsts::GLOBAL_ROWID_COL));
901901
EXPECT_FALSE(
902902
FileScannerV2::TEST_is_partition_slot(legacy_partition, BeConsts::ICEBERG_ROWID_COL));
903-
EXPECT_FALSE(FileScannerV2::TEST_is_partition_slot(legacy_partition,
904-
BeConsts::ICEBERG_FILE_PATH_COL));
905-
EXPECT_FALSE(FileScannerV2::TEST_is_partition_slot(legacy_partition,
906-
BeConsts::ICEBERG_ROW_POSITION_COL));
907-
EXPECT_FALSE(FileScannerV2::TEST_is_partition_slot(legacy_partition, "_FILE"));
908-
EXPECT_FALSE(FileScannerV2::TEST_is_partition_slot(legacy_partition, "_POS"));
909-
EXPECT_FALSE(FileScannerV2::TEST_is_partition_slot(legacy_partition, "__PAIMON_FILE_PATH"));
910-
EXPECT_FALSE(FileScannerV2::TEST_is_partition_slot(legacy_partition, "__PAIMON_ROW_INDEX"));
903+
904+
TFileScanSlotInfo synthesized;
905+
synthesized.__set_is_file_slot(true);
906+
synthesized.__set_category(TColumnCategory::SYNTHESIZED);
907+
EXPECT_FALSE(
908+
FileScannerV2::TEST_is_partition_slot(synthesized, BeConsts::ICEBERG_FILE_PATH_COL));
909+
EXPECT_FALSE(
910+
FileScannerV2::TEST_is_partition_slot(synthesized, BeConsts::ICEBERG_ROW_POSITION_COL));
911+
EXPECT_FALSE(
912+
FileScannerV2::TEST_is_partition_slot(synthesized, BeConsts::PAIMON_FILE_PATH_COL));
913+
EXPECT_FALSE(
914+
FileScannerV2::TEST_is_partition_slot(synthesized, BeConsts::PAIMON_ROW_POSITION_COL));
915+
916+
// Names alone must not hide a normal partition column. This is the collision case for a
917+
// non-Iceberg connector whose physical/partition field happens to use a metadata spelling.
918+
EXPECT_TRUE(FileScannerV2::TEST_is_partition_slot(legacy_partition, "_FILE"));
919+
EXPECT_TRUE(FileScannerV2::TEST_is_partition_slot(legacy_partition, "__PAIMON_FILE_PATH"));
911920
}
912921

913922
// Scenario: data-file slots are the complement of partition/default/synthesized columns for
@@ -944,14 +953,22 @@ TEST(FileScannerV2Test, DataFileSlotClassificationMatrix) {
944953

945954
EXPECT_FALSE(FileScannerV2::TEST_is_data_file_slot(legacy_file, BeConsts::GLOBAL_ROWID_COL));
946955
EXPECT_FALSE(FileScannerV2::TEST_is_data_file_slot(legacy_file, BeConsts::ICEBERG_ROWID_COL));
956+
957+
TFileScanSlotInfo synthesized;
958+
synthesized.__set_is_file_slot(true);
959+
synthesized.__set_category(TColumnCategory::SYNTHESIZED);
960+
EXPECT_FALSE(
961+
FileScannerV2::TEST_is_data_file_slot(synthesized, BeConsts::ICEBERG_FILE_PATH_COL));
947962
EXPECT_FALSE(
948-
FileScannerV2::TEST_is_data_file_slot(legacy_file, BeConsts::ICEBERG_FILE_PATH_COL));
963+
FileScannerV2::TEST_is_data_file_slot(synthesized, BeConsts::ICEBERG_ROW_POSITION_COL));
949964
EXPECT_FALSE(
950-
FileScannerV2::TEST_is_data_file_slot(legacy_file, BeConsts::ICEBERG_ROW_POSITION_COL));
951-
EXPECT_FALSE(FileScannerV2::TEST_is_data_file_slot(legacy_file, "_FILE"));
952-
EXPECT_FALSE(FileScannerV2::TEST_is_data_file_slot(legacy_file, "_POS"));
953-
EXPECT_FALSE(FileScannerV2::TEST_is_data_file_slot(legacy_file, "__PAIMON_FILE_PATH"));
954-
EXPECT_FALSE(FileScannerV2::TEST_is_data_file_slot(legacy_file, "__PAIMON_ROW_INDEX"));
965+
FileScannerV2::TEST_is_data_file_slot(synthesized, BeConsts::PAIMON_FILE_PATH_COL));
966+
EXPECT_FALSE(
967+
FileScannerV2::TEST_is_data_file_slot(synthesized, BeConsts::PAIMON_ROW_POSITION_COL));
968+
969+
// A legacy physical slot remains a data-file slot regardless of its spelling.
970+
EXPECT_TRUE(FileScannerV2::TEST_is_data_file_slot(legacy_file, "_FILE"));
971+
EXPECT_TRUE(FileScannerV2::TEST_is_data_file_slot(legacy_file, "__PAIMON_FILE_PATH"));
955972
}
956973

957974
// Scenario: table conjuncts are cloned into global-index space before they are handed to

‎fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadata.java‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -502,6 +502,7 @@ private ConnectorTableSchema buildTableSchema(String tableName, Table table, Sch
502502
// SELECT * / DESCRIBE unless explicitly requested. They are synthesized by the native BE
503503
// reader and are not part of the Iceberg schema or physical file projection.
504504
if (appendDataFileMetadataColumns) {
505+
rejectReservedMetadataColumns(schema);
505506
columns.add(new ConnectorColumn(ICEBERG_FILE_PATH_COL, ConnectorType.of("STRING"),
506507
"Iceberg data file path", false, null, false).invisible());
507508
columns.add(new ConnectorColumn(ICEBERG_ROW_POSITION_COL, ConnectorType.of("BIGINT"),
@@ -757,6 +758,9 @@ public boolean supportsColumnHandleSnapshotPin(ConnectorSession session) {
757758
private static Map<String, ConnectorColumnHandle> buildColumnHandles(
758759
Schema schema, boolean appendDataFileMetadataColumns) {
759760
List<Types.NestedField> fields = schema.columns();
761+
if (appendDataFileMetadataColumns) {
762+
rejectReservedMetadataColumns(schema);
763+
}
760764
Map<String, ConnectorColumnHandle> handles = new LinkedHashMap<>(
761765
fields.size() + (appendDataFileMetadataColumns ? 2 : 0));
762766
for (Types.NestedField field : fields) {
@@ -1073,11 +1077,7 @@ private void rejectReservedRowLineageColumns(ConnectorCreateTableRequest request
10731077
int formatVersion = IcebergSchemaBuilder.getEffectiveFormatVersion(request.getProperties(), properties);
10741078
for (ConnectorColumn column : request.getColumns()) {
10751079
String name = column.getName();
1076-
if (ICEBERG_FILE_PATH_COL.equalsIgnoreCase(name)
1077-
|| ICEBERG_ROW_POSITION_COL.equalsIgnoreCase(name)) {
1078-
throw new DorisConnectorException("Cannot create Iceberg table with reserved metadata column: "
1079-
+ name);
1080-
}
1080+
rejectReservedMetadataColumn(name);
10811081
if (formatVersion >= ICEBERG_ROW_LINEAGE_MIN_VERSION
10821082
&& (ICEBERG_ROW_ID_COL.equalsIgnoreCase(name)
10831083
|| ICEBERG_LAST_UPDATED_SEQUENCE_NUMBER_COL.equalsIgnoreCase(name))) {
@@ -1087,6 +1087,19 @@ private void rejectReservedRowLineageColumns(ConnectorCreateTableRequest request
10871087
}
10881088
}
10891089

1090+
private static void rejectReservedMetadataColumn(String name) {
1091+
if (ICEBERG_FILE_PATH_COL.equalsIgnoreCase(name)
1092+
|| ICEBERG_ROW_POSITION_COL.equalsIgnoreCase(name)) {
1093+
throw new DorisConnectorException("Cannot create Iceberg table with reserved metadata column: " + name);
1094+
}
1095+
}
1096+
1097+
private static void rejectReservedMetadataColumns(Schema schema) {
1098+
for (Types.NestedField field : schema.columns()) {
1099+
rejectReservedMetadataColumn(field.name());
1100+
}
1101+
}
1102+
10901103
/**
10911104
* Rejects a {@code DISTRIBUTE BY} clause: iceberg has no hash/random distribution, buckets are expressed via
10921105
* {@code bucket(num, column)} inside {@code PARTITIONED BY}. Moved off fe-core {@code CreateTableInfo.validate}
@@ -1212,6 +1225,7 @@ public void renameTable(ConnectorSession session, ConnectorTableHandle handle, S
12121225
public void addColumn(ConnectorSession session, ConnectorTableHandle handle,
12131226
ConnectorColumn column, ConnectorColumnPosition position) {
12141227
IcebergTableHandle iceHandle = (IcebergTableHandle) handle;
1228+
rejectReservedMetadataColumn(column.getName());
12151229
IcebergColumnChange change = toAddColumnChange(column);
12161230
try {
12171231
context.executeAuthenticated(() -> {
@@ -1231,6 +1245,7 @@ public void addColumns(ConnectorSession session, ConnectorTableHandle handle, Li
12311245
IcebergTableHandle iceHandle = (IcebergTableHandle) handle;
12321246
List<IcebergColumnChange> changes = new ArrayList<>(columns.size());
12331247
for (ConnectorColumn column : columns) {
1248+
rejectReservedMetadataColumn(column.getName());
12341249
changes.add(toAddColumnChange(column));
12351250
}
12361251
try {
@@ -1266,6 +1281,7 @@ public void dropColumn(ConnectorSession session, ConnectorTableHandle handle, St
12661281
public void renameColumn(ConnectorSession session, ConnectorTableHandle handle, String oldName,
12671282
String newName) {
12681283
IcebergTableHandle iceHandle = (IcebergTableHandle) handle;
1284+
rejectReservedMetadataColumn(newName);
12691285
try {
12701286
context.executeAuthenticated(() -> {
12711287
catalogOps.renameColumn(iceHandle.getDbName(), iceHandle.getTableName(), oldName, newName);

‎fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2316,6 +2316,9 @@ private static boolean requiresCurrentScanSemantics(
23162316
Table table, TableScan scan, Schema scanSchema, List<ConnectorColumnHandle> columns,
23172317
boolean hasApplicableEqualityDeletes,
23182318
Optional<Map<Integer, List<String>>> nameMapping) {
2319+
if (requiresMetadataColumns(columns)) {
2320+
return true;
2321+
}
23192322
if (hasApplicableEqualityDeletes) {
23202323
return true;
23212324
}

‎fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataDdlTest.java‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -323,6 +323,35 @@ public void testCreateTableRejectsReservedFileMetadataColumnsAtAllVersions() {
323323
}
324324
}
325325

326+
@Test
327+
public void testSchemaEvolutionRejectsReservedFileMetadataColumns() {
328+
ConnectorColumn reserved = new ConnectorColumn("_file", ConnectorType.of("STRING"), "", true, null, false);
329+
330+
RecordingIcebergCatalogOps addOps = new RecordingIcebergCatalogOps();
331+
RecordingConnectorContext addContext = new RecordingConnectorContext();
332+
Assertions.assertThrows(DorisConnectorException.class,
333+
() -> metadata(addOps, addContext, IcebergCatalogProperties.TYPE_REST)
334+
.addColumn(null, new IcebergTableHandle("db1", "t1"), reserved, null));
335+
Assertions.assertTrue(addOps.log.isEmpty());
336+
Assertions.assertEquals(0, addContext.authCount);
337+
338+
RecordingIcebergCatalogOps batchOps = new RecordingIcebergCatalogOps();
339+
RecordingConnectorContext batchContext = new RecordingConnectorContext();
340+
Assertions.assertThrows(DorisConnectorException.class,
341+
() -> metadata(batchOps, batchContext, IcebergCatalogProperties.TYPE_REST)
342+
.addColumns(null, new IcebergTableHandle("db1", "t1"), Collections.singletonList(reserved)));
343+
Assertions.assertTrue(batchOps.log.isEmpty());
344+
Assertions.assertEquals(0, batchContext.authCount);
345+
346+
RecordingIcebergCatalogOps renameOps = new RecordingIcebergCatalogOps();
347+
RecordingConnectorContext renameContext = new RecordingConnectorContext();
348+
Assertions.assertThrows(DorisConnectorException.class,
349+
() -> metadata(renameOps, renameContext, IcebergCatalogProperties.TYPE_REST)
350+
.renameColumn(null, new IcebergTableHandle("db1", "t1"), "id", "_POS"));
351+
Assertions.assertTrue(renameOps.log.isEmpty());
352+
Assertions.assertEquals(0, renameContext.authCount);
353+
}
354+
326355
@Test
327356
public void testCreateTableRejectsReservedColumnViaCatalogTableDefaultV3() {
328357
// FULL effective-format-version precedence: a catalog-level table-default.format-version=3 with NO

‎fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataTest.java‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -698,6 +698,37 @@ public void getTableSchemaParsesColumnsFromLoadedTable() {
698698
Assertions.assertEquals("ICEBERG", schema.getTableFormatType());
699699
}
700700

701+
@Test
702+
public void externalPhysicalMetadataColumnFailsBeforeSyntheticSchemaAssembly() {
703+
// An externally-created Iceberg table may legally contain a physical _file or _pos field. The
704+
// connector cannot expose that table together with Doris's synthetic metadata columns because the
705+
// schema/handle maps are name-keyed. Reject the collision before appending synthetic fields instead of
706+
// silently replacing the physical field handle with a metadata handle.
707+
for (String reserved : new String[] {"_file", "_pos", "_FILE", "_POS"}) {
708+
Schema physicalSchema = new Schema(
709+
Types.NestedField.optional(1, reserved, Types.StringType.get()),
710+
Types.NestedField.optional(2, "id", Types.IntegerType.get()));
711+
712+
RecordingIcebergCatalogOps schemaOps = new RecordingIcebergCatalogOps();
713+
schemaOps.table = new FakeIcebergTable(
714+
"t1", physicalSchema, PartitionSpec.unpartitioned(),
715+
"s3://bucket/db1/t1", Collections.emptyMap());
716+
DorisConnectorException schemaException = Assertions.assertThrows(DorisConnectorException.class,
717+
() -> metadataWith(schemaOps).getTableSchema(null, new IcebergTableHandle("db1", "t1")));
718+
Assertions.assertTrue(schemaException.getMessage().contains("reserved metadata column"),
719+
schemaException.getMessage());
720+
721+
RecordingIcebergCatalogOps handleOps = new RecordingIcebergCatalogOps();
722+
handleOps.table = new FakeIcebergTable(
723+
"t1", physicalSchema, PartitionSpec.unpartitioned(),
724+
"s3://bucket/db1/t1", Collections.emptyMap());
725+
DorisConnectorException handleException = Assertions.assertThrows(DorisConnectorException.class,
726+
() -> metadataWith(handleOps).getColumnHandles(null, new IcebergTableHandle("db1", "t1")));
727+
Assertions.assertTrue(handleException.getMessage().contains("reserved metadata column"),
728+
handleException.getMessage());
729+
}
730+
}
731+
701732
@Test
702733
public void getTableSchemaCarriesFieldDocAsComment() {
703734
RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps();

‎fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -936,6 +936,21 @@ public void metadataColumnsRejectDisabledFileScannerV2DuringPlanning() {
936936
ex.getMessage());
937937
}
938938

939+
@Test
940+
public void metadataColumnsRequireCurrentBackendSemantics() {
941+
IcebergScanPlanProvider provider = providerOver(
942+
createTable("metadata_semantics", SCHEMA, PartitionSpec.unpartitioned()));
943+
List<ConnectorColumnHandle> columns = Arrays.asList(
944+
new IcebergColumnHandle("id", 1),
945+
new IcebergColumnHandle("_pos", -1));
946+
947+
Map<String, String> props = provider.getScanNodeProperties(
948+
null, new IcebergTableHandle("db1", "metadata_semantics"), columns, Optional.empty());
949+
950+
Assertions.assertEquals("Current Iceberg scan semantics",
951+
props.get(ScanNodePropertyKeys.REQUIRED_CURRENT_BACKEND_SEMANTICS));
952+
}
953+
939954
@Test
940955
public void metadataColumnsRejectForceJniInStreamingPath() throws IOException {
941956
IcebergScanPlanProvider provider = providerOver(

‎fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonColumnHandle.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,10 @@ public int getFieldIndex() {
4545
return fieldIndex;
4646
}
4747

48+
public boolean isMetadataColumn() {
49+
return fieldIndex < 0;
50+
}
51+
4852
@Override
4953
public boolean equals(Object o) {
5054
if (this == o) {

0 commit comments

Comments
 (0)