Skip to content

Commit 32fd2fe

Browse files
committed
[fix](external) Preserve UUID bytes for binary table columns
1 parent e18aff1 commit 32fd2fe

10 files changed

Lines changed: 430 additions & 19 deletions

File tree

‎be/src/exprs/function/cast/function_cast.cpp‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
#include <utility>
1919

20+
#include "core/column/column_varbinary.h"
2021
#include "core/data_type/data_type_agg_state.h"
2122
#include "core/data_type/data_type_decimal.h"
2223
#include "core/data_type/data_type_number.h" // IWYU pragma: keep
@@ -26,6 +27,7 @@
2627
#include "core/data_type/data_type_variant.h"
2728
#include "core/data_type/data_type_variant_v2.h"
2829
#include "core/data_type/primitive_type.h"
30+
#include "core/value/uuid_value.h"
2931
#include "exprs/function/cast/cast_to_array.h"
3032
#include "exprs/function/cast/cast_to_jsonb.h"
3133
#include "exprs/function/cast/cast_to_map.h"
@@ -65,7 +67,26 @@ WrapperType create_varbinary_wrapper(const DataTypePtr& from_type_untyped) {
6567
return cast_from_string_to_generic;
6668
}
6769

68-
return CastWrapper::create_unsupport_wrapper("Cast to Varbinary only support from String type");
70+
if (from_type_untyped->get_primitive_type() == TYPE_UUID) {
71+
return [](FunctionContext*, Block& block, const ColumnNumbers& arguments, uint32_t result,
72+
size_t rows, const NullMap::value_type*) {
73+
const auto& source =
74+
assert_cast<const ColumnUUID&>(*block.get_by_position(arguments[0]).column);
75+
auto binary = ColumnVarbinary::create();
76+
binary->reserve(rows);
77+
for (size_t i = 0; i < rows; ++i) {
78+
// File Scanner V2 decodes UUIDs before mapping Hive BINARY columns. Recreate the
79+
// file's big-endian bytes, not the host integer layout or canonical UUID text.
80+
const auto bytes = UUIDValue::to_big_endian(source.get_data()[i]);
81+
binary->insert_data(reinterpret_cast<const char*>(bytes.data()), bytes.size());
82+
}
83+
block.get_by_position(result).column = std::move(binary);
84+
return Status::OK();
85+
};
86+
}
87+
88+
return CastWrapper::create_unsupport_wrapper(
89+
"Cast to Varbinary only support from String or UUID type");
6990
}
7091

7192
WrapperType prepare_unpack_dictionaries(FunctionContext* context, const DataTypePtr& from_type,

‎be/src/format/orc/vorc_reader.cpp‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2415,6 +2415,12 @@ Status OrcReader::_orc_column_to_doris_column(
24152415
MutableColumnPtr data_column;
24162416
if (orc_column_type != nullptr) {
24172417
auto src_type = convert_to_doris_type(orc_column_type);
2418+
if (src_type->get_primitive_type() == TYPE_UUID &&
2419+
data_type->get_primitive_type() == TYPE_VARBINARY) {
2420+
// Hive BINARY keeps the file bytes even when the ORC carrier has a UUID annotation.
2421+
// Decode that carrier directly, including recursively visited complex-type children.
2422+
src_type = DataTypeFactory::instance().create_data_type(TYPE_VARBINARY, true);
2423+
}
24182424
bool is_dict_filter_col = false;
24192425
for (const std::pair<std::string, int>& dict_col : _dict_filter_cols) {
24202426
if (col_name == dict_col.first) {

‎be/src/format/parquet/parquet_column_convert.cpp‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -261,9 +261,10 @@ std::unique_ptr<PhysicalToLogicalConverter> PhysicalToLogicalConverter::get_conv
261261
} else if (is_parquet_native_type(src_logical_primitive)) {
262262
bool is_string_logical_type = is_string_type(src_logical_primitive);
263263
if (is_string_logical_type && src_physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY) {
264-
if (parquet_schema.logicalType.__isset.UUID && !preserve_binary_uuid) {
265-
// Ordinary Parquet scans expose canonical text, as in File Scanner V2.
266-
// Iceberg retains its raw 16-byte STRING carrier for equality deletes/defaults.
264+
if (parquet_schema.logicalType.__isset.UUID && !preserve_binary_uuid &&
265+
dst_logical_type->get_primitive_type() != TYPE_VARBINARY) {
266+
// Inferred STRING fields expose canonical text. Explicit BINARY targets and
267+
// Iceberg's equality-delete carrier must retain the original 16 bytes.
267268
if (parquet_schema.type_length != UUIDValue::BINARY_LENGTH) {
268269
physical_converter = std::make_unique<UnsupportedConverter>(src_physical_type,
269270
src_logical_type);
Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#include "core/block/block.h"
19+
#include "core/column/column_array.h"
20+
#include "core/column/column_nullable.h"
21+
#include "core/column/column_varbinary.h"
22+
#include "core/column/column_vector.h"
23+
#include "core/data_type/data_type_array.h"
24+
#include "core/data_type/data_type_nullable.h"
25+
#include "core/data_type/data_type_uuid.h"
26+
#include "core/data_type/data_type_varbinary.h"
27+
#include "core/value/uuid_value.h"
28+
#include "exprs/function/simple_function_factory.h"
29+
#include "exprs/function_context.h"
30+
#include "gtest/gtest.h"
31+
#include "runtime/runtime_state.h"
32+
33+
namespace doris {
34+
35+
TEST(UuidBinaryCastTest, PreservesFileBytesAndNestedNulls) {
36+
for (bool strict : {false, true}) {
37+
RuntimeState state;
38+
for (bool nested : {false, true}) {
39+
const std::vector<std::string> bytes = {std::string(16, '\0'), std::string(16, '\xff'),
40+
std::string("\x80", 1) + std::string(15, '\0'),
41+
"abcDeFGhijkLmnOp"};
42+
auto values = ColumnUUID::create();
43+
auto nulls = ColumnUInt8::create();
44+
for (const auto& value : bytes) {
45+
values->insert_value(
46+
UUIDValue::from_big_endian(reinterpret_cast<const uint8_t*>(value.data())));
47+
nulls->insert_value(0);
48+
}
49+
values->insert_default();
50+
nulls->insert_value(1);
51+
ColumnPtr source = ColumnNullable::create(std::move(values), std::move(nulls));
52+
DataTypePtr source_type = make_nullable(std::make_shared<DataTypeUUID>());
53+
DataTypePtr target_type = make_nullable(std::make_shared<DataTypeVarbinary>());
54+
if (nested) {
55+
auto offsets = ColumnArray::ColumnOffsets::create();
56+
offsets->insert_value(3);
57+
offsets->insert_value(3);
58+
offsets->insert_value(5);
59+
source = ColumnArray::create(source, std::move(offsets));
60+
source_type = std::make_shared<DataTypeArray>(source_type);
61+
target_type = std::make_shared<DataTypeArray>(target_type);
62+
}
63+
auto context = FunctionContext::create_context(&state, {}, {});
64+
context->set_enable_strict_mode(strict);
65+
Block block;
66+
block.insert({source, source_type, "uuid"});
67+
auto function = SimpleFunctionFactory::instance().get_function(
68+
"CAST", {{source, source_type, "uuid"}}, target_type);
69+
ASSERT_NE(function, nullptr);
70+
const int result_id = block.columns();
71+
block.insert({nullptr, target_type, "binary"});
72+
auto status = function->execute(context.get(), block, {0}, result_id, source->size());
73+
ASSERT_TRUE(status.ok()) << status;
74+
ColumnPtr result = block.get_by_position(result_id).column;
75+
if (nested) {
76+
const auto& array = assert_cast<const ColumnArray&>(*result);
77+
EXPECT_EQ(array.get_offsets(),
78+
assert_cast<const ColumnArray&>(*source).get_offsets());
79+
result = array.get_data_ptr();
80+
}
81+
const auto& nullable = assert_cast<const ColumnNullable&>(*result);
82+
EXPECT_EQ(nullable.size(), 5);
83+
for (size_t i = 0; i < bytes.size(); ++i) {
84+
EXPECT_FALSE(nullable.is_null_at(i));
85+
EXPECT_EQ(nullable.get_nested_column().get_data_at(i).to_string(), bytes[i]);
86+
}
87+
EXPECT_TRUE(nullable.is_null_at(4));
88+
}
89+
}
90+
}
91+
92+
} // namespace doris

‎be/test/format/parquet/parquet_column_convert_test.cpp‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -63,11 +63,14 @@ TEST(ParquetColumnConvertTest, UuidTextAndBinaryCarriersPreserveNullsAcrossBatch
6363
"00000000-0000-0000-0000-000000000000", "7f7f7f7f-7f7f-7f7f-7f7f-7f7f7f7f7f7f",
6464
"80000000-0000-0000-0000-000000000000", "ffffffff-ffff-ffff-ffff-ffffffffffff",
6565
"61626344-6546-4768-696a-6b4c6d6e4f70", "00000000-0000-0000-0000-000000000000"};
66-
for (const bool preserve_binary : {false, true}) {
66+
for (int carrier = 0; carrier < 3; ++carrier) {
67+
const bool preserve_binary = carrier == 1;
68+
const auto dst_type = DataTypeFactory::instance().create_data_type(
69+
carrier == 2 ? TYPE_VARBINARY : TYPE_STRING, true);
6770
auto converter = PhysicalToLogicalConverter::get_converter(
68-
&field, field.data_type, field.data_type, nullptr, false, preserve_binary);
71+
&field, field.data_type, dst_type, nullptr, false, preserve_binary);
6972
ASSERT_TRUE(converter->support());
70-
ColumnPtr dst = field.data_type->create_column();
73+
ColumnPtr dst = dst_type->create_column();
7174
for (size_t batch = 0; batch < 2; ++batch) {
7275
auto values = ColumnFixedLengthObject::create(16);
7376
auto nulls = ColumnUInt8::create();
@@ -76,14 +79,14 @@ TEST(ParquetColumnConvertTest, UuidTextAndBinaryCarriersPreserveNullsAcrossBatch
7679
nulls->insert_value(i == 5);
7780
}
7881
ColumnPtr src = ColumnNullable::create(std::move(values), std::move(nulls));
79-
ASSERT_TRUE(converter->convert(src, field.data_type, field.data_type, dst, false).ok());
82+
ASSERT_TRUE(converter->convert(src, field.data_type, dst_type, dst, false).ok());
8083
EXPECT_EQ(dst->size(), (batch + 1) * 3);
8184
}
8285
const auto& nullable = assert_cast<const ColumnNullable&>(*dst);
8386
for (size_t i = 0; i < raw_values.size(); ++i) {
8487
EXPECT_EQ(nullable.get_null_map_data()[i], i == 5);
8588
EXPECT_EQ(nullable.get_nested_column().get_data_at(i).to_string(),
86-
preserve_binary ? raw_values[i] : text_values[i]);
89+
carrier != 0 ? raw_values[i] : text_values[i]);
8790
}
8891
}
8992
}
Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#include <gtest/gtest.h>
19+
20+
#include <cstdlib>
21+
22+
#include "core/column/column_array.h"
23+
#include "core/column/column_struct.h"
24+
#include "core/data_type/data_type_array.h"
25+
#include "core/data_type/data_type_struct.h"
26+
#include "core/data_type/data_type_varbinary.h"
27+
#include "format/parquet/vparquet_reader.h"
28+
#include "io/fs/file_meta_cache.h"
29+
#include "io/fs/local_file_system.h"
30+
#include "util/timezone_utils.h"
31+
32+
namespace doris {
33+
34+
TEST(ParquetUuidBinaryTest, LegacyReaderPreservesNestedBytesAcrossBatches) {
35+
// Timestamp-semantics negotiation can force V2 even with its session flag disabled.
36+
// Exercise the legacy reader directly to keep its nested BINARY mapping covered.
37+
const char* source_root = std::getenv("ROOT");
38+
ASSERT_NE(source_root, nullptr);
39+
const std::vector<std::string> bytes = {std::string(16, '\0'),
40+
std::string(16, '\0'),
41+
std::string(15, '\0') + std::string(1, '\1'),
42+
std::string(1, '\x7f') + std::string(15, '\xff'),
43+
std::string(1, '\x80') + std::string(15, '\0'),
44+
std::string(16, '\xff')};
45+
auto check_value = [&](const IColumn& column, size_t row, size_t id) {
46+
EXPECT_EQ(column.is_null_at(row), id % 6 == 0);
47+
if (id % 6 != 0) {
48+
EXPECT_EQ(assert_cast<const ColumnNullable&>(column)
49+
.get_nested_column()
50+
.get_data_at(row)
51+
.to_string(),
52+
bytes[id % 6]);
53+
}
54+
};
55+
const auto binary = make_nullable(std::make_shared<DataTypeVarbinary>());
56+
const DataTypes types = {
57+
make_nullable(std::make_shared<DataTypeInt32>()), binary,
58+
make_nullable(std::make_shared<DataTypeArray>(binary)),
59+
make_nullable(std::make_shared<DataTypeStruct>(DataTypes {binary}, Strings {"k"}))};
60+
const std::vector<std::string> names = {"id", "u", "a", "s"};
61+
std::unordered_map<std::string, uint32_t> indexes = {{"id", 0}, {"u", 1}, {"a", 2}, {"s", 3}};
62+
cctz::time_zone zone;
63+
ASSERT_TRUE(TimezoneUtils::find_cctz_time_zone("UTC", zone));
64+
FileMetaCache cache(1024);
65+
for (const std::string encoding : {"plain", "dictionary"}) {
66+
io::FileReaderSPtr file;
67+
ASSERT_TRUE(
68+
io::global_local_filesystem()
69+
->open_file(
70+
std::string(source_root) +
71+
"/regression-test/data/datatype_p0/uuid/test_data/uuid_" +
72+
encoding + ".parquet",
73+
&file)
74+
.ok());
75+
TFileScanRangeParams params;
76+
params.__set_enable_mapping_varbinary(true);
77+
params.__set_file_type(TFileType::FILE_LOCAL);
78+
TFileRangeDesc range;
79+
range.__set_start_offset(0);
80+
range.__set_size(file->size());
81+
range.__set_file_size(file->size());
82+
range.__set_path(file->path().native());
83+
ParquetReader reader(nullptr, params, range, 37, &zone, nullptr, nullptr, &cache);
84+
phmap::flat_hash_map<int, std::vector<std::shared_ptr<ColumnPredicate>>> predicates;
85+
auto status = reader.init_reader(names, &indexes, {}, predicates, nullptr, nullptr, nullptr,
86+
nullptr, nullptr);
87+
ASSERT_TRUE(status.ok()) << status;
88+
std::unordered_map<std::string, std::tuple<std::string, const SlotDescriptor*>> partitions;
89+
std::unordered_map<std::string, VExprContextSPtr> missing;
90+
ASSERT_TRUE(reader.set_fill_columns(partitions, missing).ok());
91+
bool eof = false;
92+
size_t total = 0;
93+
while (!eof) {
94+
Block block;
95+
for (size_t i = 0; i < names.size(); ++i) {
96+
block.insert({types[i]->create_column(), types[i], names[i]});
97+
}
98+
size_t rows = 0;
99+
status = reader.get_next_block(&block, &rows, &eof);
100+
ASSERT_TRUE(status.ok()) << status;
101+
for (size_t row = 0; row < rows; ++row) {
102+
const size_t id = total + row;
103+
check_value(*block.get_by_position(1).column, row, id);
104+
const auto& array_nullable =
105+
assert_cast<const ColumnNullable&>(*block.get_by_position(2).column);
106+
EXPECT_EQ(array_nullable.is_null_at(row), id % 7 == 0);
107+
const auto& array =
108+
assert_cast<const ColumnArray&>(array_nullable.get_nested_column());
109+
const auto begin = array.get_offsets()[static_cast<ssize_t>(row) - 1];
110+
const auto end = array.get_offsets()[row];
111+
if (id % 7 > 1) {
112+
ASSERT_EQ(end - begin, 2);
113+
check_value(array.get_data(), begin, id);
114+
EXPECT_TRUE(array.get_data().is_null_at(begin + 1));
115+
} else {
116+
EXPECT_EQ(end, begin);
117+
}
118+
const auto& structure =
119+
assert_cast<const ColumnNullable&>(*block.get_by_position(3).column);
120+
EXPECT_EQ(structure.is_null_at(row), id % 5 == 0);
121+
if (id % 5 != 0) {
122+
const auto& fields =
123+
assert_cast<const ColumnStruct&>(structure.get_nested_column());
124+
check_value(fields.get_column(0), row, id);
125+
}
126+
}
127+
total += rows;
128+
}
129+
EXPECT_EQ(total, 4097);
130+
}
131+
}
132+
133+
} // namespace doris

0 commit comments

Comments
 (0)