Skip to content

Commit 1af387c

Browse files
committed
[fix](be) Classify legacy metadata slots case-insensitively
### What problem does this PR solve? Issue Number: close apache#66816 Related PR: apache#67207 Problem Summary: FileScannerV2's compatibility fallback for plans without TColumnCategory classified metadata slots using case-sensitive names. Uppercase Iceberg or Paimon metadata projections could therefore be sent to physical file readers. Centralize legacy virtual-slot recognition with case-insensitive comparisons while preserving explicit category precedence, and cover Iceberg and Paimon file path/row-position variants. ### Release note None ### Check List (For Author) - Test: Unit Test - BE FileScannerV2 classification tests: 2 passed. - build hygiene check, clang-format check, clang-tidy check. - Behavior changed: Yes (legacy slot fallback now recognizes metadata names case-insensitively). - Does this need documentation: No
1 parent 6afc891 commit 1af387c

2 files changed

Lines changed: 25 additions & 13 deletions

File tree

‎be/src/exec/scan/file_scanner_v2.cpp‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@
7575
#include "runtime/runtime_state.h"
7676
#include "service/backend_options.h"
7777
#include "storage/id_manager.h"
78+
#include "util/string_util.h"
7879

7980
namespace doris {
8081
namespace {
@@ -194,12 +195,20 @@ bool is_wal_format(TFileFormatType::type format_type) {
194195
return format_type == TFileFormatType::FORMAT_WAL;
195196
}
196197

198+
bool is_legacy_virtual_slot(const std::string& column_name) {
199+
return column_name.starts_with(BeConsts::GLOBAL_ROWID_COL) ||
200+
iequal(column_name, BeConsts::ICEBERG_ROWID_COL) ||
201+
iequal(column_name, BeConsts::ICEBERG_FILE_PATH_COL) ||
202+
iequal(column_name, BeConsts::ICEBERG_ROW_POSITION_COL) ||
203+
iequal(column_name, BeConsts::PAIMON_FILE_PATH_COL) ||
204+
iequal(column_name, BeConsts::PAIMON_ROW_POSITION_COL);
205+
}
206+
197207
bool is_partition_slot(const TFileScanSlotInfo& slot_info, const std::string& column_name) {
198208
if (slot_info.__isset.category) {
199209
return slot_info.category == TColumnCategory::PARTITION_KEY;
200210
}
201-
return !slot_info.is_file_slot && !column_name.starts_with(BeConsts::GLOBAL_ROWID_COL) &&
202-
column_name != BeConsts::ICEBERG_ROWID_COL;
211+
return !slot_info.is_file_slot && !is_legacy_virtual_slot(column_name);
203212
}
204213

205214
bool is_data_file_slot(const TFileScanSlotInfo& slot_info, const std::string& column_name) {
@@ -211,8 +220,7 @@ bool is_data_file_slot(const TFileScanSlotInfo& slot_info, const std::string& co
211220
return slot_info.category == TColumnCategory::REGULAR ||
212221
slot_info.category == TColumnCategory::GENERATED;
213222
}
214-
return slot_info.is_file_slot && !column_name.starts_with(BeConsts::GLOBAL_ROWID_COL) &&
215-
column_name != BeConsts::ICEBERG_ROWID_COL;
223+
return slot_info.is_file_slot && !is_legacy_virtual_slot(column_name);
216224
}
217225

218226
Status rewrite_slot_refs_to_global_index(

‎be/test/exec/scan/file_scanner_v2_test.cpp‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -875,8 +875,9 @@ TEST(FileScannerV2Test, OrcScannerResidualFilterRetainsNextBatchContext) {
875875
}
876876

877877
// Scenario: partition slots are identified from the explicit FE category when present, otherwise
878-
// from the legacy is_file_slot flag. Metadata columns must be classified by the explicit SYNTHESIZED
879-
// category so ordinary physical or partition columns with the same spelling remain readable.
878+
// from the legacy is_file_slot flag. The legacy fallback recognizes virtual metadata columns by name,
879+
// case-insensitively, so they never reach a physical file reader. Explicit categories retain precedence
880+
// for physical fields that share a metadata spelling.
880881
TEST(FileScannerV2Test, PartitionSlotClassificationMatrix) {
881882
TFileScanSlotInfo legacy_partition;
882883
legacy_partition.__set_is_file_slot(false);
@@ -895,6 +896,7 @@ TEST(FileScannerV2Test, PartitionSlotClassificationMatrix) {
895896
categorized_regular.__set_is_file_slot(false);
896897
categorized_regular.__set_category(TColumnCategory::REGULAR);
897898
EXPECT_FALSE(FileScannerV2::TEST_is_partition_slot(categorized_regular, "regular_col"));
899+
EXPECT_FALSE(FileScannerV2::TEST_is_partition_slot(categorized_regular, "_FILE"));
898900

899901
EXPECT_FALSE(
900902
FileScannerV2::TEST_is_partition_slot(legacy_partition, BeConsts::GLOBAL_ROWID_COL));
@@ -913,10 +915,10 @@ TEST(FileScannerV2Test, PartitionSlotClassificationMatrix) {
913915
EXPECT_FALSE(
914916
FileScannerV2::TEST_is_partition_slot(synthesized, BeConsts::PAIMON_ROW_POSITION_COL));
915917

916-
// Names alone must not hide a normal partition column. This is the collision case for a
917-
// non-Iceberg connector whose physical/partition field happens to use a metadata spelling.
918-
EXPECT_TRUE(FileScannerV2::TEST_is_partition_slot(legacy_partition, "_FILE"));
919-
EXPECT_TRUE(FileScannerV2::TEST_is_partition_slot(legacy_partition, "__PAIMON_FILE_PATH"));
918+
EXPECT_FALSE(FileScannerV2::TEST_is_partition_slot(legacy_partition, "_FILE"));
919+
EXPECT_FALSE(FileScannerV2::TEST_is_partition_slot(legacy_partition, "_POS"));
920+
EXPECT_FALSE(FileScannerV2::TEST_is_partition_slot(legacy_partition, "__PAIMON_FILE_PATH"));
921+
EXPECT_FALSE(FileScannerV2::TEST_is_partition_slot(legacy_partition, "__PAIMON_ROW_INDEX"));
920922
}
921923

922924
// Scenario: data-file slots are the complement of partition/default/synthesized columns for
@@ -935,6 +937,7 @@ TEST(FileScannerV2Test, DataFileSlotClassificationMatrix) {
935937
categorized_regular.__set_is_file_slot(false);
936938
categorized_regular.__set_category(TColumnCategory::REGULAR);
937939
EXPECT_TRUE(FileScannerV2::TEST_is_data_file_slot(categorized_regular, "regular_col"));
940+
EXPECT_TRUE(FileScannerV2::TEST_is_data_file_slot(categorized_regular, "_POS"));
938941

939942
TFileScanSlotInfo categorized_generated;
940943
categorized_generated.__set_is_file_slot(false);
@@ -966,9 +969,10 @@ TEST(FileScannerV2Test, DataFileSlotClassificationMatrix) {
966969
EXPECT_FALSE(
967970
FileScannerV2::TEST_is_data_file_slot(synthesized, BeConsts::PAIMON_ROW_POSITION_COL));
968971

969-
// A legacy physical slot remains a data-file slot regardless of its spelling.
970-
EXPECT_TRUE(FileScannerV2::TEST_is_data_file_slot(legacy_file, "_FILE"));
971-
EXPECT_TRUE(FileScannerV2::TEST_is_data_file_slot(legacy_file, "__PAIMON_FILE_PATH"));
972+
EXPECT_FALSE(FileScannerV2::TEST_is_data_file_slot(legacy_file, "_FILE"));
973+
EXPECT_FALSE(FileScannerV2::TEST_is_data_file_slot(legacy_file, "_POS"));
974+
EXPECT_FALSE(FileScannerV2::TEST_is_data_file_slot(legacy_file, "__PAIMON_FILE_PATH"));
975+
EXPECT_FALSE(FileScannerV2::TEST_is_data_file_slot(legacy_file, "__PAIMON_ROW_INDEX"));
972976
}
973977

974978
// Scenario: table conjuncts are cloned into global-index space before they are handed to

0 commit comments

Comments
 (0)