Skip to content

Commit 2cf14a4

Browse files
committed
[fix](be) Preserve legacy metadata-named physical columns
### What problem does this PR solve? Issue Number: close apache#66816 Related PR: apache#67207 Problem Summary: FileScannerV2 globally classified the new Iceberg and Paimon metadata names as virtual columns in its legacy compatibility path. During an old-FE/new-BE rolling upgrade, unrelated physical columns with the same names could therefore be removed from file projection. Current FE versions send explicit synthesized-column categories for these metadata columns, so keep the name-based legacy fallback only for the pre-existing row-id columns and let metadata-like physical names from an old FE follow is_file_slot. ### Release note None ### Check List (For Author) - Test: Unit Test - Compiled the FileScannerV2 unit-test source and the production unity target; git diff --check passed - Behavior changed: No - Does this need documentation: No
1 parent 255ecad commit 2cf14a4

2 files changed

Lines changed: 12 additions & 16 deletions

File tree

‎be/src/exec/scan/file_scanner_v2.cpp‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -197,11 +197,7 @@ bool is_wal_format(TFileFormatType::type format_type) {
197197

198198
bool is_legacy_virtual_slot(const std::string& column_name) {
199199
return column_name.starts_with(BeConsts::GLOBAL_ROWID_COL) ||
200-
iequal(column_name, BeConsts::ICEBERG_ROWID_COL) ||
201-
iequal(column_name, BeConsts::ICEBERG_FILE_PATH_COL) ||
202-
iequal(column_name, BeConsts::ICEBERG_ROW_POSITION_COL) ||
203-
iequal(column_name, BeConsts::PAIMON_FILE_PATH_COL) ||
204-
iequal(column_name, BeConsts::PAIMON_ROW_POSITION_COL);
200+
iequal(column_name, BeConsts::ICEBERG_ROWID_COL);
205201
}
206202

207203
bool is_partition_slot(const TFileScanSlotInfo& slot_info, const std::string& column_name) {

‎be/test/exec/scan/file_scanner_v2_test.cpp‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -875,9 +875,9 @@ TEST(FileScannerV2Test, OrcScannerResidualFilterRetainsNextBatchContext) {
875875
}
876876

877877
// Scenario: partition slots are identified from the explicit FE category when present, otherwise
878-
// from the legacy is_file_slot flag. The legacy fallback recognizes virtual metadata columns by name,
879-
// case-insensitively, so they never reach a physical file reader. Explicit categories retain precedence
880-
// for physical fields that share a metadata spelling.
878+
// from the legacy is_file_slot flag. Only pre-existing row-id columns use the name-based legacy
879+
// fallback. New connector metadata columns require an explicit category so an old FE can still scan
880+
// unrelated physical fields that share a metadata spelling.
881881
TEST(FileScannerV2Test, PartitionSlotClassificationMatrix) {
882882
TFileScanSlotInfo legacy_partition;
883883
legacy_partition.__set_is_file_slot(false);
@@ -915,10 +915,10 @@ TEST(FileScannerV2Test, PartitionSlotClassificationMatrix) {
915915
EXPECT_FALSE(
916916
FileScannerV2::TEST_is_partition_slot(synthesized, BeConsts::PAIMON_ROW_POSITION_COL));
917917

918-
EXPECT_FALSE(FileScannerV2::TEST_is_partition_slot(legacy_partition, "_FILE"));
919-
EXPECT_FALSE(FileScannerV2::TEST_is_partition_slot(legacy_partition, "_POS"));
920-
EXPECT_FALSE(FileScannerV2::TEST_is_partition_slot(legacy_partition, "__PAIMON_FILE_PATH"));
921-
EXPECT_FALSE(FileScannerV2::TEST_is_partition_slot(legacy_partition, "__PAIMON_ROW_INDEX"));
918+
EXPECT_TRUE(FileScannerV2::TEST_is_partition_slot(legacy_partition, "_FILE"));
919+
EXPECT_TRUE(FileScannerV2::TEST_is_partition_slot(legacy_partition, "_POS"));
920+
EXPECT_TRUE(FileScannerV2::TEST_is_partition_slot(legacy_partition, "__PAIMON_FILE_PATH"));
921+
EXPECT_TRUE(FileScannerV2::TEST_is_partition_slot(legacy_partition, "__PAIMON_ROW_INDEX"));
922922
}
923923

924924
// Scenario: data-file slots are the complement of partition/default/synthesized columns for
@@ -969,10 +969,10 @@ TEST(FileScannerV2Test, DataFileSlotClassificationMatrix) {
969969
EXPECT_FALSE(
970970
FileScannerV2::TEST_is_data_file_slot(synthesized, BeConsts::PAIMON_ROW_POSITION_COL));
971971

972-
EXPECT_FALSE(FileScannerV2::TEST_is_data_file_slot(legacy_file, "_FILE"));
973-
EXPECT_FALSE(FileScannerV2::TEST_is_data_file_slot(legacy_file, "_POS"));
974-
EXPECT_FALSE(FileScannerV2::TEST_is_data_file_slot(legacy_file, "__PAIMON_FILE_PATH"));
975-
EXPECT_FALSE(FileScannerV2::TEST_is_data_file_slot(legacy_file, "__PAIMON_ROW_INDEX"));
972+
EXPECT_TRUE(FileScannerV2::TEST_is_data_file_slot(legacy_file, "_FILE"));
973+
EXPECT_TRUE(FileScannerV2::TEST_is_data_file_slot(legacy_file, "_POS"));
974+
EXPECT_TRUE(FileScannerV2::TEST_is_data_file_slot(legacy_file, "__PAIMON_FILE_PATH"));
975+
EXPECT_TRUE(FileScannerV2::TEST_is_data_file_slot(legacy_file, "__PAIMON_ROW_INDEX"));
976976
}
977977

978978
// Scenario: table conjuncts are cloned into global-index space before they are handed to

0 commit comments

Comments
 (0)