Skip to content

Commit 0c8b6cb

Browse files
committed
[fix](datalake) Reconcile UUID support after branch-4.2 rebase
1 parent 64f3cfa commit 0c8b6cb

10 files changed

Lines changed: 31 additions & 64 deletions

‎be/src/core/data_type_serde/data_type_varbinary_serde.cpp‎

Lines changed: 0 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -375,42 +375,6 @@ Status DataTypeVarbinarySerDe::write_column_to_orc(const std::string& timezone,
375375
return Status::OK();
376376
}
377377

378-
Status DataTypeVarbinarySerDe::read_column_from_orc(IColumn& column,
379-
const OrcDecodedColumnView& view) const {
380-
DORIS_CHECK(view.file_type->getKind() == orc::BINARY);
381-
const auto* batch = dynamic_cast<const orc::StringVectorBatch*>(view.batch);
382-
DORIS_CHECK(batch != nullptr);
383-
const auto* encoded_batch = dynamic_cast<const orc::EncodedStringVectorBatch*>(batch);
384-
const bool is_encoded = encoded_batch != nullptr && encoded_batch->isEncoded;
385-
if (is_encoded) {
386-
DORIS_CHECK(encoded_batch->dictionary != nullptr);
387-
}
388-
auto& binary_column = assert_cast<ColumnVarbinary&>(column);
389-
const auto rows = orc_serde_utils::orc_decode_row_count(view.rows, view.selected_rows);
390-
binary_column.reserve(binary_column.size() + rows);
391-
for (size_t row = 0; row < rows; ++row) {
392-
const auto source_row = orc_serde_utils::orc_source_row_at(row, view.selected_rows);
393-
if (orc_serde_utils::orc_row_is_null(*batch, source_row)) {
394-
binary_column.insert_default();
395-
continue;
396-
}
397-
char* data = nullptr;
398-
int64_t length = 0;
399-
if (is_encoded) {
400-
encoded_batch->dictionary->getValueByIndex(encoded_batch->index[source_row], data,
401-
length);
402-
} else {
403-
data = batch->data[source_row];
404-
length = batch->length[source_row];
405-
}
406-
if (length < 0) {
407-
return Status::Corruption("Invalid negative ORC binary length {}", length);
408-
}
409-
binary_column.insert_data(length == 0 ? "" : data, cast_set<size_t>(length));
410-
}
411-
return Status::OK();
412-
}
413-
414378
Status DataTypeVarbinarySerDe::serialize_one_cell_to_json(const IColumn& column, int64_t row_num,
415379
BufferWritable& bw,
416380
FormatOptions& options) const {

‎be/src/core/data_type_serde/data_type_varbinary_serde.h‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -114,8 +114,6 @@ class DataTypeVarbinarySerDe : public DataTypeSerDe {
114114
int64_t start, int64_t end, Arena& arena,
115115
const FormatOptions& options) const override;
116116

117-
Status read_column_from_orc(IColumn& column, const OrcDecodedColumnView& view) const override;
118-
119117
void to_string(const IColumn& column, size_t row_num, BufferWritable& bw,
120118
const FormatOptions& options) const override;
121119

‎regression-test/data/datatype_p0/uuid/test_uuid_orc_varbinary_mapping.out‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
-- This file is automatically generated. You should know what you did if you want to edit this
22
-- !schema_false_false --
33
a array<uuid> Yes false \N NONE
4-
b text Yes false \N NONE
4+
b varbinary(65533) Yes false \N NONE
55
id int Yes false \N NONE
66
m map<uuid,uuid> Yes false \N NONE
77
s struct<k:uuid> Yes false \N NONE
@@ -41,7 +41,7 @@ u varbinary(65533) Yes false \N NONE
4141

4242
-- !schema_true_false --
4343
a array<uuid> Yes false \N NONE
44-
b text Yes false \N NONE
44+
b varbinary(65533) Yes false \N NONE
4545
id int Yes false \N NONE
4646
m map<uuid,uuid> Yes false \N NONE
4747
s struct<k:uuid> Yes false \N NONE

‎regression-test/data/datatype_p0/uuid/test_uuid_parquet_native.out‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,9 @@ s struct<k:varbinary(16)> Yes false \N NONE
2121

2222
-- !external_schema --
2323
id int Yes false \N NONE
24-
u text Yes false \N NONE
25-
a array<text> Yes false \N NONE
26-
s struct<k:text> Yes false \N NONE
24+
u uuid Yes false \N NONE
25+
a array<uuid> Yes false \N NONE
26+
s struct<k:uuid> Yes false \N NONE
2727

2828
-- !external_values --
2929
0 \N \N \N \N \N
@@ -131,9 +131,9 @@ s struct<k:varbinary(16)> Yes false \N NONE
131131

132132
-- !external_schema --
133133
id int Yes false \N NONE
134-
u text Yes false \N NONE
135-
a array<text> Yes false \N NONE
136-
s struct<k:text> Yes false \N NONE
134+
u uuid Yes false \N NONE
135+
a array<uuid> Yes false \N NONE
136+
s struct<k:uuid> Yes false \N NONE
137137

138138
-- !external_values --
139139
0 \N \N \N \N \N
@@ -241,9 +241,9 @@ s struct<k:varbinary(16)> Yes false \N NONE
241241

242242
-- !external_schema --
243243
id int Yes false \N NONE
244-
u text Yes false \N NONE
245-
a array<text> Yes false \N NONE
246-
s struct<k:text> Yes false \N NONE
244+
u uuid Yes false \N NONE
245+
a array<uuid> Yes false \N NONE
246+
s struct<k:uuid> Yes false \N NONE
247247

248248
-- !external_values --
249249
0 \N \N \N \N \N
@@ -351,9 +351,9 @@ s struct<k:varbinary(16)> Yes false \N NONE
351351

352352
-- !external_schema --
353353
id int Yes false \N NONE
354-
u text Yes false \N NONE
355-
a array<text> Yes false \N NONE
356-
s struct<k:text> Yes false \N NONE
354+
u uuid Yes false \N NONE
355+
a array<uuid> Yes false \N NONE
356+
s struct<k:uuid> Yes false \N NONE
357357

358358
-- !external_values --
359359
0 \N \N \N \N \N

‎regression-test/data/datatype_p0/uuid/test_uuid_parquet_roundtrip.out‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@
1010
3 \N \N \N
1111

1212
-- !inferred_types --
13-
a array<text> Yes false \N NONE
13+
a array<uuid> Yes false \N NONE
1414
id int Yes false \N NONE
15-
s struct<k:text> Yes false \N NONE
16-
u text Yes false \N NONE
15+
s struct<k:uuid> Yes false \N NONE
16+
u uuid Yes false \N NONE
1717

‎regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group6.out‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -808,15 +808,15 @@ true
808808

809809
-- !test_98 --
810810
\N \N \N
811-
61626344-6546-4768-696a-6b4c6d6e4f70 682.56 1212
812-
61626344-6546-4768-696a-6b4c6d6e4f70 682.56 1212
813-
61626344-6546-4768-696a-6b4c6d6e4f70 682.56 1212
814-
61626344-6546-4768-696a-6b4c6d6e4f70 682.56 1212
811+
61626344-6546-4768-696a-6b4c6d6e4f70 682.56 0x313231320000000000000000
812+
61626344-6546-4768-696a-6b4c6d6e4f70 682.56 0x313231320000000000000000
813+
61626344-6546-4768-696a-6b4c6d6e4f70 682.56 0x313231320000000000000000
814+
61626344-6546-4768-696a-6b4c6d6e4f70 682.56 0x313231320000000000000000
815815

816816
-- !test_98_desc --
817817
decimal_flba decimal(5,2) Yes false \N NONE
818-
interval text Yes false \N NONE
819-
uuid text Yes false \N NONE
818+
interval varbinary(65533) Yes false \N NONE
819+
uuid uuid Yes false \N NONE
820820

821821
-- !test_100 --
822822
1317017856 1 18752152 809291 1089176 19951117 3-MEDIUM 0 40 4801000 16034243 9 4368910 72015 3 19951228 RAIL Customer#018752152 q4gN2btSpiKXdN,6 ALGERIA 1 ALGERIA AFRICA 10-753-996-8708 MACHINERY Supplier#001089176 ROidEL1L6yeFsJqnUjD EGYPT 5 EGYPT MIDDLE EAST 14-807-108-7869 blanched gainsboro MFGR#4 MFGR#43 MFGR#433 brown MEDIUM BRUSHED STEEL 42 MED BAG

‎regression-test/suites/datatype_p0/uuid/test_uuid_orc_varbinary_mapping.groovy‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ suite("test_uuid_orc_varbinary_mapping") {
5656
String source = """local("file_path"="${outputDirectory.fileName}/orc_*",
5757
"format"="orc", "backend_id"="${backend.BackendId}",
5858
"enable_mapping_varbinary"="${binaryMapping}")"""
59+
// Ordinary BINARY stays VARBINARY even when the UUID binary override is off.
5960
"order_qt_schema_${scannerV2}_${binaryMapping}" "DESC FUNCTION ${source}"
6061
String uuidValue = binaryMapping ? "CAST(HEX(u) AS UUID)" : "u"
6162
String nestedValue = binaryMapping ? "CAST(HEX(value) AS UUID)" : "value"

‎regression-test/suites/datatype_p0/uuid/test_uuid_parquet_native.groovy‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ suite("test_uuid_parquet_native", "p0,external") {
5656
qt_binary_schema "DESC FUNCTION ${binarySource}"
5757
qt_binary_values """SELECT id,HEX(u),CAST(HEX(u) AS UUID) FROM ${binarySource}
5858
WHERE id < 12 ORDER BY id"""
59+
// Schema inference retains UUID annotations, including nested fields.
5960
qt_external_schema "DESC FUNCTION ${source}"
6061
qt_external_values "SELECT id,u,a,s,LENGTH(u),CAST(u AS UUID) FROM ${source} WHERE id < 12 ORDER BY id"
6162
qt_external_group "SELECT u,COUNT(*),SUM(id) FROM ${source} GROUP BY u ORDER BY u"

‎regression-test/suites/datatype_p0/uuid/test_uuid_parquet_roundtrip.groovy‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ suite("test_uuid_parquet_roundtrip") {
4848
SELECT * FROM local("file_path"="${path}", "format"="parquet",
4949
"backend_id"="${backend.BackendId}")"""
5050
qt_roundtrip "SELECT * FROM uuid_file_reload_parquet ORDER BY id"
51+
// The inferred schema must preserve the UUID annotations written by OUTFILE.
5152
order_qt_inferred_types """DESC FUNCTION local(
5253
"file_path"="${path}", "format"="parquet", "backend_id"="${backend.BackendId}")"""
5354
} finally {

‎regression-test/suites/external_table_p0/tvf/test_hdfs_binary_mapping.groovy‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,15 +27,17 @@ suite("test_hdfs_binary_mapping", "p0,external,hive,tvf,external_docker") {
2727
sql "SET enable_file_scanner_v2=true"
2828
for (def file : [["orc", "test_hdfs_orc/group6/orc_test_binary_column.orc"],
2929
["parquet", "test_hdfs_parquet/group6/fixed_len_byte_array.parquet"]]) {
30+
// UUID annotations retain a separate explicit override; compare the ordinary binary leaf.
31+
String projection = file[0] == "parquet" ? "decimal_flba, `interval`" : "*"
3032
def expectedSchema = null
3133
def expectedRows = null
3234
for (String option : ["", ', "enable_mapping_varbinary"="false"',
3335
', "enable_mapping_varbinary"="true"']) {
3436
String tvf = """HDFS("uri"="hdfs://${host}:${port}/user/doris/tvf_data/${file[1]}",
3537
"hadoop.username"="doris", "format"="${file[0]}" ${option})"""
36-
def schema = sql("DESC FUNCTION ${tvf}")
38+
def schema = sql("DESC FUNCTION ${tvf}").findAll { it[0] != "uuid" }
3739
assertTrue(schema.any { it[1].toString().toLowerCase().startsWith("varbinary") })
38-
def rows = sql("SELECT * FROM ${tvf} LIMIT 5")
40+
def rows = sql("SELECT ${projection} FROM ${tvf} LIMIT 5")
3941
// The legacy option must change neither the inferred type nor the bytes returned.
4042
if (expectedSchema == null) {
4143
expectedSchema = schema
@@ -49,7 +51,7 @@ suite("test_hdfs_binary_mapping", "p0,external,hive,tvf,external_docker") {
4951
test {
5052
sql """CREATE TABLE ${target} DUPLICATE KEY(mapping_key)
5153
DISTRIBUTED BY HASH(mapping_key) BUCKETS 1
52-
PROPERTIES("replication_num"="1") AS SELECT 1 AS mapping_key, * FROM ${tvf} LIMIT 5"""
54+
PROPERTIES("replication_num"="1") AS SELECT 1 AS mapping_key, ${projection} FROM ${tvf} LIMIT 5"""
5355
exception "doris do not support varbinary create table"
5456
}
5557
}

0 commit comments

Comments
 (0)