Skip to content

Commit 2bffb2d

Browse files
committed
[fix](be) Reject lossy file predicate literal conversion
### What problem does this PR solve? Issue Number: N/A Related PR: N/A Problem Summary: FileScannerV2 localized predicate literals to the file column type whenever conversion succeeded, even when the conversion changed the literal value. For example, localizing a DOUBLE predicate value of 1.5 to an INT file column changed the boundary to 1 and could make file-level filtering silently drop matching rows. Require an exact round trip through the file type before localizing a literal; otherwise, preserve table semantics by casting the file slot. Exact conversions such as BIGINT 1 to INT 1 remain localized. ### Release note Fix FileScannerV2 predicate localization to avoid missing rows after lossy literal conversion. ### Check List (For Author) - Test: Unit Test - ColumnMapperCastTest targeted literal localization tests - Behavior changed: Yes, lossy literal conversions now fall back to casting the file slot. - Does this need documentation: No
1 parent e05c331 commit 2bffb2d

2 files changed

Lines changed: 95 additions & 0 deletions

File tree

‎be/src/format_v2/column_mapper.cpp‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -553,6 +553,21 @@ static VExprSPtr rewrite_literal_to_file_type(const VExprSPtr& literal_expr,
553553
if (file_field.get_type() != remove_nullable(rewrite_info.file_type)->get_primitive_type()) {
554554
return nullptr;
555555
}
556+
Field round_trip_field;
557+
try {
558+
convert_field_to_type(file_field, *original_literal->data_type(), &round_trip_field,
559+
rewrite_info.file_type.get());
560+
} catch (const Exception&) {
561+
return nullptr;
562+
}
563+
// Only localize a literal when converting it to the file type is lossless. For example,
564+
// BIGINT 1 -> INT 1 -> BIGINT 1 succeeds. However, for a table predicate `value < 1.5`
565+
// with DOUBLE table type and INT file type, rewriting 1.5 to INT 1 would make a file value
566+
// of 1 fail `value < 1` and silently drop a matching row. Its round trip
567+
// DOUBLE 1.5 -> INT 1 -> DOUBLE 1.0 fails here and keeps the table-typed predicate instead.
568+
if (round_trip_field != original_field) {
569+
return nullptr;
570+
}
556571
auto literal = std::make_shared<SplitLocalFileLiteral>(
557572
rewrite_info.file_type, file_field, original_literal->data_type(), original_field);
558573
rewrite_context->add_created_expr(literal);

‎be/test/format_v2/column_mapper_test.cpp‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3486,6 +3486,44 @@ TEST_F(ColumnMapperCastTest, ColumnMapperCastsLiteralForLiteralSlotPredicateType
34863486
file_request.conjuncts[0]->close();
34873487
}
34883488

3489+
// Scenario: a fractional table literal cannot be localized to an integral file type without
3490+
// changing the predicate boundary, so the mapper must cast the file slot instead.
3491+
TEST_F(ColumnMapperCastTest, ColumnMapperRejectsLossyBinaryLiteralConversion) {
3492+
TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME});
3493+
auto table_column = name_col("value", f64());
3494+
std::vector<ColumnDefinition> projected_columns {table_column};
3495+
3496+
auto file_field = name_col("value", i32(), 0);
3497+
std::vector<ColumnDefinition> file_schema {file_field};
3498+
3499+
auto status = mapper.create_mapping(projected_columns, {}, file_schema);
3500+
ASSERT_TRUE(status.ok()) << status;
3501+
3502+
auto predicate = binary_predicate(
3503+
TExprOpcode::LT, VSlotRef::create_shared(0, 0, -1, table_column.type, "value"),
3504+
VLiteral::create_shared(table_column.type, Field::create_field<TYPE_DOUBLE>(1.5)));
3505+
TableFilter table_filter;
3506+
table_filter.conjunct = VExprContext::create_shared(predicate);
3507+
table_filter.global_indices = {GlobalIndex(0)};
3508+
3509+
FileScanRequest file_request;
3510+
ASSERT_TRUE(mapper.create_scan_request({table_filter}, projected_columns, &file_request, &state)
3511+
.ok());
3512+
ASSERT_EQ(file_request.conjuncts.size(), 1);
3513+
const auto& localized_expr = file_request.conjuncts[0]->root();
3514+
ASSERT_EQ(localized_expr->get_num_children(), 2);
3515+
const auto& localized_slot_cast = localized_expr->children()[0];
3516+
ASSERT_NE(dynamic_cast<const Cast*>(localized_slot_cast.get()), nullptr);
3517+
EXPECT_TRUE(localized_slot_cast->data_type()->equals(*table_column.type));
3518+
ASSERT_EQ(localized_slot_cast->get_num_children(), 1);
3519+
const auto* localized_slot =
3520+
assert_cast<const VSlotRef*>(localized_slot_cast->children()[0].get());
3521+
EXPECT_EQ(localized_slot->column_id(), 0);
3522+
EXPECT_TRUE(localized_slot->data_type()->equals(*file_field.type));
3523+
EXPECT_TRUE(localized_expr->children()[1]->is_literal());
3524+
EXPECT_TRUE(localized_expr->children()[1]->data_type()->equals(*table_column.type));
3525+
}
3526+
34893527
// Scenario: IN predicate literals are all rewritten to file type when every literal conversion is safe.
34903528
TEST_F(ColumnMapperCastTest, ColumnMapperCastsInPredicateLiteralsForTypeMismatch) {
34913529
TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME});
@@ -3524,6 +3562,48 @@ TEST_F(ColumnMapperCastTest, ColumnMapperCastsInPredicateLiteralsForTypeMismatch
35243562
EXPECT_TRUE(localized_expr->children()[2]->data_type()->equals(*file_field.type));
35253563
}
35263564

3565+
// Scenario: one lossy IN literal prevents the entire predicate from being localized to file type.
3566+
TEST_F(ColumnMapperCastTest, ColumnMapperRejectsLossyInPredicateLiteralConversion) {
3567+
TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME});
3568+
auto table_column = name_col("value", f64());
3569+
std::vector<ColumnDefinition> projected_columns {table_column};
3570+
3571+
auto file_field = name_col("value", i32(), 0);
3572+
std::vector<ColumnDefinition> file_schema {file_field};
3573+
3574+
auto status = mapper.create_mapping(projected_columns, {}, file_schema);
3575+
ASSERT_TRUE(status.ok()) << status;
3576+
3577+
auto predicate = create_in_predicate();
3578+
predicate->add_child(VSlotRef::create_shared(0, 0, -1, table_column.type, "value"));
3579+
predicate->add_child(
3580+
VLiteral::create_shared(table_column.type, Field::create_field<TYPE_DOUBLE>(1.0)));
3581+
predicate->add_child(
3582+
VLiteral::create_shared(table_column.type, Field::create_field<TYPE_DOUBLE>(1.5)));
3583+
TableFilter table_filter;
3584+
table_filter.conjunct = VExprContext::create_shared(predicate);
3585+
table_filter.global_indices = {GlobalIndex(0)};
3586+
3587+
FileScanRequest file_request;
3588+
ASSERT_TRUE(mapper.create_scan_request({table_filter}, projected_columns, &file_request, &state)
3589+
.ok());
3590+
ASSERT_EQ(file_request.conjuncts.size(), 1);
3591+
const auto& localized_expr = file_request.conjuncts[0]->root();
3592+
ASSERT_EQ(localized_expr->get_num_children(), 3);
3593+
const auto& localized_slot_cast = localized_expr->children()[0];
3594+
ASSERT_NE(dynamic_cast<const Cast*>(localized_slot_cast.get()), nullptr);
3595+
EXPECT_TRUE(localized_slot_cast->data_type()->equals(*table_column.type));
3596+
ASSERT_EQ(localized_slot_cast->get_num_children(), 1);
3597+
const auto* localized_slot =
3598+
assert_cast<const VSlotRef*>(localized_slot_cast->children()[0].get());
3599+
EXPECT_EQ(localized_slot->column_id(), 0);
3600+
EXPECT_TRUE(localized_slot->data_type()->equals(*file_field.type));
3601+
EXPECT_TRUE(localized_expr->children()[1]->is_literal());
3602+
EXPECT_TRUE(localized_expr->children()[1]->data_type()->equals(*table_column.type));
3603+
EXPECT_TRUE(localized_expr->children()[2]->is_literal());
3604+
EXPECT_TRUE(localized_expr->children()[2]->data_type()->equals(*table_column.type));
3605+
}
3606+
35273607
// Scenario: IN predicate falls back to casting the file slot when any literal cannot be converted safely.
35283608
TEST_F(ColumnMapperCastTest, ColumnMapperFallsBackToSlotCastWhenInPredicateLiteralRewriteFails) {
35293609
TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME});

0 commit comments

Comments
 (0)