Skip to content

Commit b35abbd

Browse files
committed
[fix](type) Preserve VARBINARY length in serialized blocks
1 parent cd673b8 commit b35abbd

5 files changed

Lines changed: 92 additions & 1 deletion

File tree

‎be/src/core/data_type/data_type_factory.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -298,7 +298,7 @@ DataTypePtr DataTypeFactory::create_data_type(const PColumnMeta& pcolumn) {
298298
nested = std::make_shared<DataTypeHLL>();
299299
break;
300300
case PGenericType::VARBINARY:
301-
nested = std::make_shared<DataTypeVarbinary>();
301+
nested = std::make_shared<DataTypeVarbinary>(pcolumn.varbinary_length());
302302
break;
303303
case PGenericType::LIST:
304304
DCHECK(pcolumn.children_size() == 1);

‎be/src/core/data_type/data_type_varbinary.cpp‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
#include "core/data_type/data_type_varbinary.h"
1919

20+
#include <gen_cpp/data.pb.h>
2021
#include <glog/logging.h>
2122
#include <lz4/lz4.h>
2223
#include <streamvbyte.h>
@@ -54,6 +55,12 @@ bool DataTypeVarbinary::equals(const IDataType& rhs) const {
5455
return typeid(rhs) == typeid(*this);
5556
}
5657

58+
void DataTypeVarbinary::to_pb_column_meta(PColumnMeta* col_meta) const {
59+
IDataType::to_pb_column_meta(col_meta);
60+
// Iceberg FIXED/UUID writers need the declared width after a Block crosses an exchange.
61+
col_meta->set_varbinary_length(_len);
62+
}
63+
5764
// binary: const flag| row num | real saved num | size array | data array
5865
// bool size_t size_t
5966
// <size array>: data1 size | data2 size | ... sizeof(size_t) * real_need_copy_num

‎be/src/core/data_type/data_type_varbinary.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,8 @@ class DataTypeVarbinary : public IDataType {
8181
return std::make_shared<DataTypeVarbinarySerDe>(nesting_level);
8282
};
8383

84+
void to_pb_column_meta(PColumnMeta* col_meta) const override;
85+
8486
void to_protobuf(PTypeDesc* ptype, PTypeNode* node, PScalarType* scalar_type) const override {
8587
scalar_type->set_len(_len);
8688
}

‎be/test/core/data_type/data_type_varbinary_test.cpp‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,13 +30,18 @@
3030
#include "agent/be_exec_version_manager.h"
3131
#include "common/exception.h"
3232
#include "core/assert_cast.h"
33+
#include "core/block/block.h"
3334
#include "core/column/column_const.h"
3435
#include "core/column/column_string.h"
3536
#include "core/column/column_varbinary.h"
3637
#include "core/data_type/common_data_type_serder_test.h"
3738
#include "core/data_type/common_data_type_test.h"
39+
#include "core/data_type/data_type_array.h"
3840
#include "core/data_type/data_type_factory.hpp"
41+
#include "core/data_type/data_type_map.h"
42+
#include "core/data_type/data_type_nullable.h"
3943
#include "core/data_type/data_type_string.h"
44+
#include "core/data_type/data_type_struct.h"
4045
#include "core/data_type_serde/data_type_serde.h"
4146
#include "core/field.h"
4247
#include "core/string_buffer.hpp"
@@ -287,6 +292,81 @@ TEST_F(DataTypeVarbinaryTest, ToProtobufDefaultLen) {
287292
EXPECT_EQ(scalar.len(), -1);
288293
}
289294

295+
TEST_F(DataTypeVarbinaryTest, BlockRoundTripPreservesDeclaredLength) {
296+
for (int length : {-1, 16, 32}) {
297+
for (bool nullable : {false, true}) {
298+
SCOPED_TRACE(::testing::Message() << "length=" << length << ", nullable=" << nullable);
299+
DataTypePtr type = std::make_shared<DataTypeVarbinary>(length);
300+
if (nullable) {
301+
type = make_nullable(type);
302+
}
303+
auto column = type->create_column();
304+
const std::string value = make_bytes(16);
305+
column->insert(Field::create_field<TYPE_VARBINARY>(StringView(value)));
306+
if (nullable) {
307+
column->insert_default();
308+
}
309+
Block block {{std::move(column), type, "binary_value"}};
310+
PBlock serialized;
311+
size_t uncompressed_bytes = 0;
312+
size_t compressed_bytes = 0;
313+
int64_t compress_time = 0;
314+
ASSERT_TRUE(block.serialize(BeExecVersionManager::get_newest_version(), &serialized,
315+
&uncompressed_bytes, &compressed_bytes, &compress_time,
316+
segment_v2::CompressionTypePB::SNAPPY)
317+
.ok());
318+
std::string wire;
319+
ASSERT_TRUE(serialized.SerializeToString(&wire));
320+
PBlock received;
321+
ASSERT_TRUE(received.ParseFromString(wire));
322+
Block restored;
323+
int64_t decompress_time = 0;
324+
ASSERT_TRUE(restored.deserialize(received, &uncompressed_bytes, &decompress_time).ok());
325+
const auto& entry = restored.get_by_position(0);
326+
ASSERT_EQ(entry.type->is_nullable(), nullable);
327+
EXPECT_EQ(assert_cast<const DataTypeVarbinary&>(*remove_nullable(entry.type)).len(),
328+
length);
329+
ASSERT_EQ(restored.rows(), block.rows());
330+
for (size_t row = 0; row < block.rows(); ++row) {
331+
EXPECT_EQ((*entry.column)[row], (*block.get_by_position(0).column)[row]);
332+
}
333+
}
334+
}
335+
}
336+
337+
TEST_F(DataTypeVarbinaryTest, NestedColumnMetadataPreservesDeclaredLength) {
338+
auto binary16 = std::make_shared<DataTypeVarbinary>(16);
339+
auto binary32 = std::make_shared<DataTypeVarbinary>(32);
340+
DataTypeStruct type({std::make_shared<DataTypeArray>(make_nullable(binary16)),
341+
std::make_shared<DataTypeMap>(binary16, make_nullable(binary32))},
342+
{"array_value", "map_value"});
343+
PColumnMeta metadata;
344+
type.to_pb_column_meta(&metadata);
345+
std::string wire;
346+
ASSERT_TRUE(metadata.SerializeToString(&wire));
347+
PColumnMeta received;
348+
ASSERT_TRUE(received.ParseFromString(wire));
349+
auto restored = DataTypeFactory::instance().create_data_type(received);
350+
const auto& fields = assert_cast<const DataTypeStruct&>(*restored).get_elements();
351+
const auto& array = assert_cast<const DataTypeArray&>(*fields[0]);
352+
ASSERT_TRUE(array.get_nested_type()->is_nullable());
353+
EXPECT_EQ(
354+
assert_cast<const DataTypeVarbinary&>(*remove_nullable(array.get_nested_type())).len(),
355+
16);
356+
const auto& map = assert_cast<const DataTypeMap&>(*fields[1]);
357+
EXPECT_EQ(assert_cast<const DataTypeVarbinary&>(*map.get_key_type()).len(), 16);
358+
ASSERT_TRUE(map.get_value_type()->is_nullable());
359+
EXPECT_EQ(assert_cast<const DataTypeVarbinary&>(*remove_nullable(map.get_value_type())).len(),
360+
32);
361+
}
362+
363+
TEST_F(DataTypeVarbinaryTest, LegacyColumnMetadataUsesUnspecifiedLength) {
364+
PColumnMeta metadata;
365+
metadata.set_type(PGenericType::VARBINARY);
366+
auto restored = DataTypeFactory::instance().create_data_type(metadata);
367+
EXPECT_EQ(assert_cast<const DataTypeVarbinary&>(*restored).len(), -1);
368+
}
369+
290370
TEST_F(DataTypeVarbinaryTest, ProtobufPreservesDeclaredLength) {
291371
PTypeDesc type;
292372
auto* node = type.add_types();

‎gensrc/proto/data.proto‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,8 @@ message PColumnMeta {
7070
optional int32 variant_max_subcolumns_count = 10 [default = 0];
7171
optional bool variant_enable_doc_mode = 11 [default = false];
7272
optional bool variant_is_v2 = 12 [default = false];
73+
// Older blocks omit the declared VARBINARY length and retain the unspecified sentinel.
74+
optional int32 varbinary_length = 13 [default = -1];
7375
}
7476

7577
message PBlock {

0 commit comments

Comments
 (0)