Skip to content

Commit 5f01461

Browse files
committed
update
1 parent 7554ffb commit 5f01461

4 files changed

Lines changed: 193 additions & 18 deletions

File tree

‎be/src/core/data_type_serde/data_type_string_serde.cpp‎

Lines changed: 43 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -261,23 +261,50 @@ Status DataTypeStringSerDeBase<ColumnType>::read_column_from_arrow(
261261
arrow_array->type_id() == arrow::Type::BINARY) {
262262
const auto* concrete_array = dynamic_cast<const arrow::BinaryArray*>(arrow_array);
263263
std::shared_ptr<arrow::Buffer> buffer = concrete_array->value_data();
264-
const uint8_t* offsets_data = concrete_array->value_offsets()->data();
265-
const size_t offset_size = sizeof(int32_t);
266264

267-
for (auto offset_i = start; offset_i < end; ++offset_i) {
268-
if (!concrete_array->IsNull(offset_i)) {
269-
int32_t start_offset = 0;
270-
int32_t end_offset = 0;
271-
memcpy(&start_offset, offsets_data + offset_i * offset_size, offset_size);
272-
memcpy(&end_offset, offsets_data + (offset_i + 1) * offset_size, offset_size);
273-
274-
int32_t length = end_offset - start_offset;
275-
const auto* raw_data = buffer->data() + start_offset;
276-
277-
assert_cast<ColumnType&>(column).insert_data(
278-
reinterpret_cast<const char*>(raw_data), length);
279-
} else {
280-
assert_cast<ColumnType&>(column).insert_default();
265+
const int64_t offsets_buf_size = concrete_array->value_offsets()->size();
266+
const int64_t expected_int64_size = (arrow_array->length() + 1) * sizeof(int64_t);
267+
268+
// Detect schema/data mismatch: sender may upgrade utf8 to large_utf8 (int64 offsets)
269+
// without updating the schema. For N rows, int32 offsets need (N+1)*4 bytes while
270+
// int64 offsets need (N+1)*8 bytes. These sizes never collide for N >= 0.
271+
const bool offsets_are_int64 = (offsets_buf_size == expected_int64_size);
272+
273+
if (offsets_are_int64) {
274+
// Reconstruct as LargeBinaryArray to read int64 offsets correctly.
275+
auto large_type = (arrow_array->type_id() == arrow::Type::STRING)
276+
? arrow::large_utf8()
277+
: arrow::large_binary();
278+
auto new_data = arrow::ArrayData::Make(large_type, arrow_array->length(),
279+
arrow_array->data()->buffers,
280+
arrow_array->null_count(),
281+
concrete_array->offset());
282+
auto large_array = std::make_shared<arrow::LargeBinaryArray>(new_data);
283+
std::shared_ptr<arrow::Buffer> large_buffer = large_array->value_data();
284+
285+
for (auto offset_i = start; offset_i < end; ++offset_i) {
286+
if (!large_array->IsNull(offset_i)) {
287+
const auto* raw_data =
288+
large_buffer->data() + large_array->value_offset(offset_i);
289+
assert_cast<ColumnType&>(column).insert_data(
290+
reinterpret_cast<const char*>(raw_data),
291+
large_array->value_length(offset_i));
292+
} else {
293+
assert_cast<ColumnType&>(column).insert_default();
294+
}
295+
}
296+
} else {
297+
for (auto offset_i = start; offset_i < end; ++offset_i) {
298+
if (!concrete_array->IsNull(offset_i)) {
299+
const int32_t start_offset = concrete_array->value_offset(offset_i);
300+
const int32_t end_offset = concrete_array->value_offset(offset_i + 1);
301+
const int32_t length = end_offset - start_offset;
302+
const auto* raw_data = buffer->data() + start_offset;
303+
assert_cast<ColumnType&>(column).insert_data(
304+
reinterpret_cast<const char*>(raw_data), length);
305+
} else {
306+
assert_cast<ColumnType&>(column).insert_default();
307+
}
281308
}
282309
}
283310
} else if (arrow_array->type_id() == arrow::Type::FIXED_SIZE_BINARY) {

‎be/src/format/arrow/arrow_block_convertor.cpp‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,9 @@ Status FromBlockToRecordBatchConverter::convert(std::shared_ptr<arrow::RecordBat
7171
}
7272

7373
_arrays.resize(num_fields);
74+
// Track whether any field type was changed (e.g. utf8 -> large_utf8)
75+
bool schema_changed = false;
76+
std::vector<std::shared_ptr<arrow::Field>> fields(num_fields);
7477

7578
for (int idx = 0; idx < num_fields; ++idx) {
7679
_cur_field_idx = idx;
@@ -82,7 +85,9 @@ Status FromBlockToRecordBatchConverter::convert(std::shared_ptr<arrow::RecordBat
8285
auto arrow_type = _schema->field(idx)->type();
8386
if (arrow_type->name() == "utf8" && column->byte_size() >= MAX_ARROW_UTF8) {
8487
arrow_type = arrow::large_utf8();
88+
schema_changed = true;
8589
}
90+
fields[idx] = _schema->field(idx)->WithType(arrow_type);
8691
std::unique_ptr<arrow::ArrayBuilder> builder;
8792
auto arrow_st = arrow::MakeBuilder(_pool, arrow_type, &builder);
8893
if (!arrow_st.ok()) {
@@ -103,7 +108,8 @@ Status FromBlockToRecordBatchConverter::convert(std::shared_ptr<arrow::RecordBat
103108
return to_doris_status(arrow_st);
104109
}
105110
}
106-
*out = arrow::RecordBatch::Make(_schema, actual_rows, std::move(_arrays));
111+
auto schema = schema_changed ? arrow::schema(std::move(fields)) : _schema;
112+
*out = arrow::RecordBatch::Make(std::move(schema), _block.rows(), std::move(_arrays));
107113
return Status::OK();
108114
}
109115

‎be/src/format/arrow/arrow_row_batch.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ class Schema;
3939

4040
namespace doris {
4141

42-
constexpr size_t MAX_ARROW_UTF8 = (1ULL << 21); // 2G
42+
constexpr size_t MAX_ARROW_UTF8 = (1ULL << 31); // 2G
4343

4444
class RowDescriptor;
4545

‎be/test/core/data_type_serde/data_type_serde_string_test.cpp‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
#include <streamvbyte.h>
2525

2626
#include <cstddef>
27+
#include <iomanip>
2728
#include <iostream>
2829
#include <limits>
2930
#include <type_traits>
@@ -315,4 +316,145 @@ TEST_F(DataTypeStringSerDeTest, ArrowMemNotAlignedNestedArr) {
315316
EXPECT_TRUE(st.ok());
316317
}
317318

319+
// Simulate the sender-side bug: the sender upgrades a utf8 builder to large_utf8 (for columns
320+
// exceeding 2MB), producing int64 offsets in the buffer, but forgets to update the schema,
321+
// so the schema still says utf8. This test demonstrates that naive int32 reading of such a
322+
// buffer gives wrong (negative) lengths at every other row, and that our receiver-side fix
323+
// correctly recovers the data.
324+
TEST_F(DataTypeStringSerDeTest, ArrowInt64OffsetsInUtf8Schema_ShowBug) {
325+
// Build a few strings that are easy to verify.
326+
std::vector<std::string> strings = {"apple", "banana", "cherry", "doris_db",
327+
"arrow_data", "flight", "segment", "row_seven",
328+
"eighth_row", "last_row"};
329+
const int64_t row_count = static_cast<int64_t>(strings.size());
330+
331+
// Step 1: Build a LargeStringArray — this uses int64 offsets in its buffer.
332+
arrow::LargeStringBuilder builder;
333+
for (const auto& s : strings) {
334+
ASSERT_TRUE(builder.Append(s).ok());
335+
}
336+
std::shared_ptr<arrow::Array> large_array;
337+
ASSERT_TRUE(builder.Finish(&large_array).ok());
338+
ASSERT_EQ(large_array->type_id(), arrow::Type::LARGE_STRING);
339+
340+
// Step 2: Simulate the buggy sender — wrap with utf8 type but keep the int64 offsets buffer.
341+
// This is exactly what block_convertor.cpp did before the sender-side fix.
342+
auto buggy_data =
343+
arrow::ArrayData::Make(arrow::utf8(), // schema says utf8 (int32)
344+
large_array->length(), // but the buffer has int64 offsets!
345+
large_array->data()->buffers, large_array->null_count(), 0);
346+
auto buggy_array = std::make_shared<arrow::StringArray>(buggy_data);
347+
348+
// Step 3: Verify the offsets buffer is int64-sized (the mismatch we're testing).
349+
const int64_t expected_int64_buf = (row_count + 1) * static_cast<int64_t>(sizeof(int64_t));
350+
const int64_t expected_int32_buf = (row_count + 1) * static_cast<int64_t>(sizeof(int32_t));
351+
// Print the buffer contents so the int32/int64 mismatch is visually clear.
352+
std::cout << "\n--- Offsets buffer viewed as int64 vs int32 (the bug) ---\n";
353+
std::cout << std::left << std::setw(6) << "Row" << std::setw(16) << "int64 offset"
354+
<< std::setw(20) << "int32[2r] (low)" << std::setw(20) << "int32[2r+1] (high)"
355+
<< "naive length = int32[2r+1-1] - int32[2r-1] (*)\n";
356+
const int64_t* as_int64 =
357+
reinterpret_cast<const int64_t*>(buggy_array->value_offsets()->data());
358+
const int32_t* as_int32 =
359+
reinterpret_cast<const int32_t*>(buggy_array->value_offsets()->data());
360+
for (int i = 0; i <= static_cast<int>(strings.size()); ++i) {
361+
std::cout << std::setw(6) << i << std::setw(16) << as_int64[i] << std::setw(20)
362+
<< as_int32[2 * i] // lower 4 bytes
363+
<< std::setw(20) << as_int32[2 * i + 1] // upper 4 bytes (always 0 here)
364+
<< "\n";
365+
}
366+
// Show the naive int32 length computation for each row.
367+
// The receiver calls value_offset(r) = as_int32[r], so:
368+
// naive_length(r) = as_int32[r+1] - as_int32[r]
369+
// This reads ACROSS the int64 boundary: as_int32[2r+1] (high half of offset[r])
370+
// minus as_int32[2r] (low half of offset[r]), which gives 0 - low_half = negative.
371+
std::cout << "\n--- Naive int32 length per row (what the buggy receiver computes) ---\n";
372+
for (int r = 0; r < static_cast<int>(strings.size()); ++r) {
373+
int32_t naive_start = as_int32[r];
374+
int32_t naive_end = as_int32[r + 1];
375+
int32_t naive_len = naive_end - naive_start;
376+
std::cout << "Row " << r << " (\"" << strings[r] << "\", real len=" << strings[r].size()
377+
<< "): naive start=" << naive_start << " end=" << naive_end
378+
<< " length=" << naive_len
379+
<< (naive_len < 0 ? " ← CRASH (negative length passed to memcpy)" : "") << "\n";
380+
}
381+
std::cout << "---\n\n";
382+
383+
EXPECT_EQ(buggy_array->value_offsets()->size(), expected_int64_buf)
384+
<< "Offsets buffer is int64-sized, but schema says utf8 (int32)";
385+
EXPECT_NE(buggy_array->value_offsets()->size(), expected_int32_buf)
386+
<< "Should NOT equal int32 expected size";
387+
388+
// Step 4: Show the bug — naive int32 reading of the int64 buffer gives wrong lengths.
389+
//
390+
// The 10-row LargeStringArray has 11 int64 offsets in its buffer (row N's data spans
391+
// [offset[N], offset[N+1])). With strings "apple"(5), "banana"(6), "cherry"(6), ...,
392+
// the cumulative byte positions are: 0, 5, 11, 17, 25, 35, 41, 48, 57, 67, 75.
393+
//
394+
// These int64 values are laid out in memory (little-endian) as 8 bytes each.
395+
// For small values (< 2^32), the upper 4 bytes are always 0:
396+
//
397+
// Byte offset Value Binary (LE, 8 bytes) As two int32 words
398+
// 0 0 00 00 00 00 | 00 00 00 00 → int32[0]=0, int32[1]=0
399+
// 8 5 05 00 00 00 | 00 00 00 00 → int32[2]=5, int32[3]=0
400+
// 16 11 0B 00 00 00 | 00 00 00 00 → int32[4]=11, int32[5]=0
401+
// 24 17 11 00 00 00 | 00 00 00 00 → int32[6]=17, int32[7]=0
402+
// ...
403+
//
404+
// The buggy receiver treats this buffer as int32 and calls value_offset(r) which returns
405+
// the r-th int32 word (not the r-th int64 offset).
406+
//
407+
// For row 2 ("cherry", expected length 6):
408+
// start_offset = value_offset(2) = int32[2] = 5 ← lower half of int64(5)
409+
// end_offset = value_offset(3) = int32[3] = 0 ← upper half of int64(5), always 0
410+
// length = end_offset - start_offset = 0 - 5 = -5 ← NEGATIVE → crash
411+
//
412+
// The same pattern repeats for every even row (0, 2, 4, ...) where the upper-half word
413+
// of the preceding int64 offset (always 0) is used as end_offset.
414+
const int32_t* naive_int32_offsets =
415+
reinterpret_cast<const int32_t*>(buggy_array->value_offsets()->data());
416+
// int32[2] = lower 4 bytes of int64(5) = 5 → becomes start_offset of row 2
417+
// int32[3] = upper 4 bytes of int64(5) = 0 → becomes end_offset of row 2 (corrupted)
418+
// naive length for row 2 = 0 - 5 = -5
419+
const int32_t naive_len_row2 = naive_int32_offsets[3] - naive_int32_offsets[2];
420+
EXPECT_LT(naive_len_row2, 0) << "Naive int32 reading of int64 offsets gives negative length "
421+
"(demonstrating the crash bug): got "
422+
<< naive_len_row2;
423+
}
424+
425+
TEST_F(DataTypeStringSerDeTest, ArrowInt64OffsetsInUtf8Schema_ReceiverFix) {
426+
std::vector<std::string> strings = {"apple", "banana", "cherry", "doris_db",
427+
"arrow_data", "flight", "segment", "row_seven",
428+
"eighth_row", "last_row"};
429+
const int64_t row_count = static_cast<int64_t>(strings.size());
430+
431+
// Build a LargeStringArray (int64 offsets), then rewrap with utf8 type — simulates bug.
432+
arrow::LargeStringBuilder builder;
433+
for (const auto& s : strings) {
434+
ASSERT_TRUE(builder.Append(s).ok());
435+
}
436+
std::shared_ptr<arrow::Array> large_array;
437+
ASSERT_TRUE(builder.Finish(&large_array).ok());
438+
439+
auto buggy_data =
440+
arrow::ArrayData::Make(arrow::utf8(), large_array->length(),
441+
large_array->data()->buffers, large_array->null_count(), 0);
442+
auto buggy_array = std::make_shared<arrow::StringArray>(buggy_data);
443+
444+
// Our receiver-side fix: read_column_from_arrow detects int64 offsets by comparing
445+
// offsets buffer size against (N+1)*8 vs (N+1)*4, and falls through to LargeBinaryArray.
446+
auto result_col = ColumnString::create();
447+
cctz::time_zone tz;
448+
auto st = serde_str->read_column_from_arrow(*result_col, buggy_array.get(), 0, row_count, tz);
449+
ASSERT_TRUE(st.ok()) << "Receiver fix should handle int64 offsets in utf8 array: "
450+
<< st.to_string();
451+
452+
// Verify every row is correctly recovered.
453+
ASSERT_EQ(static_cast<int64_t>(result_col->size()), row_count);
454+
for (int64_t i = 0; i < row_count; i++) {
455+
EXPECT_EQ(result_col->get_data_at(i).to_string(), strings[i])
456+
<< "Data mismatch at row " << i;
457+
}
458+
}
459+
318460
} // namespace doris

0 commit comments

Comments
 (0)