|
24 | 24 | #include <streamvbyte.h> |
25 | 25 |
|
26 | 26 | #include <cstddef> |
| 27 | +#include <iomanip> |
27 | 28 | #include <iostream> |
28 | 29 | #include <limits> |
29 | 30 | #include <type_traits> |
@@ -315,4 +316,145 @@ TEST_F(DataTypeStringSerDeTest, ArrowMemNotAlignedNestedArr) { |
315 | 316 | EXPECT_TRUE(st.ok()); |
316 | 317 | } |
317 | 318 |
|
| 319 | +// Simulate the sender-side bug: the sender upgrades a utf8 builder to large_utf8 (for columns |
| 320 | +// exceeding 2MB), producing int64 offsets in the buffer, but forgets to update the schema, |
| 321 | +// so the schema still says utf8. This test demonstrates that naive int32 reading of such a |
| 322 | +// buffer gives wrong (negative) lengths at every other row, and that our receiver-side fix |
| 323 | +// correctly recovers the data. |
| 324 | +TEST_F(DataTypeStringSerDeTest, ArrowInt64OffsetsInUtf8Schema_ShowBug) { |
| 325 | + // Build a few strings that are easy to verify. |
| 326 | + std::vector<std::string> strings = {"apple", "banana", "cherry", "doris_db", |
| 327 | + "arrow_data", "flight", "segment", "row_seven", |
| 328 | + "eighth_row", "last_row"}; |
| 329 | + const int64_t row_count = static_cast<int64_t>(strings.size()); |
| 330 | + |
| 331 | + // Step 1: Build a LargeStringArray — this uses int64 offsets in its buffer. |
| 332 | + arrow::LargeStringBuilder builder; |
| 333 | + for (const auto& s : strings) { |
| 334 | + ASSERT_TRUE(builder.Append(s).ok()); |
| 335 | + } |
| 336 | + std::shared_ptr<arrow::Array> large_array; |
| 337 | + ASSERT_TRUE(builder.Finish(&large_array).ok()); |
| 338 | + ASSERT_EQ(large_array->type_id(), arrow::Type::LARGE_STRING); |
| 339 | + |
| 340 | + // Step 2: Simulate the buggy sender — wrap with utf8 type but keep the int64 offsets buffer. |
| 341 | + // This is exactly what block_convertor.cpp did before the sender-side fix. |
| 342 | + auto buggy_data = |
| 343 | + arrow::ArrayData::Make(arrow::utf8(), // schema says utf8 (int32) |
| 344 | + large_array->length(), // but the buffer has int64 offsets! |
| 345 | + large_array->data()->buffers, large_array->null_count(), 0); |
| 346 | + auto buggy_array = std::make_shared<arrow::StringArray>(buggy_data); |
| 347 | + |
| 348 | + // Step 3: Verify the offsets buffer is int64-sized (the mismatch we're testing). |
| 349 | + const int64_t expected_int64_buf = (row_count + 1) * static_cast<int64_t>(sizeof(int64_t)); |
| 350 | + const int64_t expected_int32_buf = (row_count + 1) * static_cast<int64_t>(sizeof(int32_t)); |
| 351 | + // Print the buffer contents so the int32/int64 mismatch is visually clear. |
| 352 | + std::cout << "\n--- Offsets buffer viewed as int64 vs int32 (the bug) ---\n"; |
| 353 | + std::cout << std::left << std::setw(6) << "Row" << std::setw(16) << "int64 offset" |
| 354 | + << std::setw(20) << "int32[2r] (low)" << std::setw(20) << "int32[2r+1] (high)" |
| 355 | + << "naive length = int32[2r+1-1] - int32[2r-1] (*)\n"; |
| 356 | + const int64_t* as_int64 = |
| 357 | + reinterpret_cast<const int64_t*>(buggy_array->value_offsets()->data()); |
| 358 | + const int32_t* as_int32 = |
| 359 | + reinterpret_cast<const int32_t*>(buggy_array->value_offsets()->data()); |
| 360 | + for (int i = 0; i <= static_cast<int>(strings.size()); ++i) { |
| 361 | + std::cout << std::setw(6) << i << std::setw(16) << as_int64[i] << std::setw(20) |
| 362 | + << as_int32[2 * i] // lower 4 bytes |
| 363 | + << std::setw(20) << as_int32[2 * i + 1] // upper 4 bytes (always 0 here) |
| 364 | + << "\n"; |
| 365 | + } |
| 366 | + // Show the naive int32 length computation for each row. |
| 367 | + // The receiver calls value_offset(r) = as_int32[r], so: |
| 368 | + // naive_length(r) = as_int32[r+1] - as_int32[r] |
| 369 | + // This reads ACROSS the int64 boundary: as_int32[2r+1] (high half of offset[r]) |
| 370 | + // minus as_int32[2r] (low half of offset[r]), which gives 0 - low_half = negative. |
| 371 | + std::cout << "\n--- Naive int32 length per row (what the buggy receiver computes) ---\n"; |
| 372 | + for (int r = 0; r < static_cast<int>(strings.size()); ++r) { |
| 373 | + int32_t naive_start = as_int32[r]; |
| 374 | + int32_t naive_end = as_int32[r + 1]; |
| 375 | + int32_t naive_len = naive_end - naive_start; |
| 376 | + std::cout << "Row " << r << " (\"" << strings[r] << "\", real len=" << strings[r].size() |
| 377 | + << "): naive start=" << naive_start << " end=" << naive_end |
| 378 | + << " length=" << naive_len |
| 379 | + << (naive_len < 0 ? " ← CRASH (negative length passed to memcpy)" : "") << "\n"; |
| 380 | + } |
| 381 | + std::cout << "---\n\n"; |
| 382 | + |
| 383 | + EXPECT_EQ(buggy_array->value_offsets()->size(), expected_int64_buf) |
| 384 | + << "Offsets buffer is int64-sized, but schema says utf8 (int32)"; |
| 385 | + EXPECT_NE(buggy_array->value_offsets()->size(), expected_int32_buf) |
| 386 | + << "Should NOT equal int32 expected size"; |
| 387 | + |
| 388 | + // Step 4: Show the bug — naive int32 reading of the int64 buffer gives wrong lengths. |
| 389 | + // |
| 390 | + // The 10-row LargeStringArray has 11 int64 offsets in its buffer (row N's data spans |
| 391 | + // [offset[N], offset[N+1])). With strings "apple"(5), "banana"(6), "cherry"(6), ..., |
| 392 | + // the cumulative byte positions are: 0, 5, 11, 17, 25, 35, 41, 48, 57, 67, 75. |
| 393 | + // |
| 394 | + // These int64 values are laid out in memory (little-endian) as 8 bytes each. |
| 395 | + // For small values (< 2^32), the upper 4 bytes are always 0: |
| 396 | + // |
| 397 | + // Byte offset Value Binary (LE, 8 bytes) As two int32 words |
| 398 | + // 0 0 00 00 00 00 | 00 00 00 00 → int32[0]=0, int32[1]=0 |
| 399 | + // 8 5 05 00 00 00 | 00 00 00 00 → int32[2]=5, int32[3]=0 |
| 400 | + // 16 11 0B 00 00 00 | 00 00 00 00 → int32[4]=11, int32[5]=0 |
| 401 | + // 24 17 11 00 00 00 | 00 00 00 00 → int32[6]=17, int32[7]=0 |
| 402 | + // ... |
| 403 | + // |
| 404 | + // The buggy receiver treats this buffer as int32 and calls value_offset(r) which returns |
| 405 | + // the r-th int32 word (not the r-th int64 offset). |
| 406 | + // |
| 407 | + // For row 2 ("cherry", expected length 6): |
| 408 | + // start_offset = value_offset(2) = int32[2] = 5 ← lower half of int64(5) |
| 409 | + // end_offset = value_offset(3) = int32[3] = 0 ← upper half of int64(5), always 0 |
| 410 | + // length = end_offset - start_offset = 0 - 5 = -5 ← NEGATIVE → crash |
| 411 | + // |
| 412 | + // The same pattern repeats for every even row (0, 2, 4, ...) where the upper-half word |
| 413 | + // of the preceding int64 offset (always 0) is used as end_offset. |
| 414 | + const int32_t* naive_int32_offsets = |
| 415 | + reinterpret_cast<const int32_t*>(buggy_array->value_offsets()->data()); |
| 416 | + // int32[2] = lower 4 bytes of int64(5) = 5 → becomes start_offset of row 2 |
| 417 | + // int32[3] = upper 4 bytes of int64(5) = 0 → becomes end_offset of row 2 (corrupted) |
| 418 | + // naive length for row 2 = 0 - 5 = -5 |
| 419 | + const int32_t naive_len_row2 = naive_int32_offsets[3] - naive_int32_offsets[2]; |
| 420 | + EXPECT_LT(naive_len_row2, 0) << "Naive int32 reading of int64 offsets gives negative length " |
| 421 | + "(demonstrating the crash bug): got " |
| 422 | + << naive_len_row2; |
| 423 | +} |
| 424 | + |
| 425 | +TEST_F(DataTypeStringSerDeTest, ArrowInt64OffsetsInUtf8Schema_ReceiverFix) { |
| 426 | + std::vector<std::string> strings = {"apple", "banana", "cherry", "doris_db", |
| 427 | + "arrow_data", "flight", "segment", "row_seven", |
| 428 | + "eighth_row", "last_row"}; |
| 429 | + const int64_t row_count = static_cast<int64_t>(strings.size()); |
| 430 | + |
| 431 | + // Build a LargeStringArray (int64 offsets), then rewrap with utf8 type — simulates bug. |
| 432 | + arrow::LargeStringBuilder builder; |
| 433 | + for (const auto& s : strings) { |
| 434 | + ASSERT_TRUE(builder.Append(s).ok()); |
| 435 | + } |
| 436 | + std::shared_ptr<arrow::Array> large_array; |
| 437 | + ASSERT_TRUE(builder.Finish(&large_array).ok()); |
| 438 | + |
| 439 | + auto buggy_data = |
| 440 | + arrow::ArrayData::Make(arrow::utf8(), large_array->length(), |
| 441 | + large_array->data()->buffers, large_array->null_count(), 0); |
| 442 | + auto buggy_array = std::make_shared<arrow::StringArray>(buggy_data); |
| 443 | + |
| 444 | + // Our receiver-side fix: read_column_from_arrow detects int64 offsets by comparing |
| 445 | + // offsets buffer size against (N+1)*8 vs (N+1)*4, and falls through to LargeBinaryArray. |
| 446 | + auto result_col = ColumnString::create(); |
| 447 | + cctz::time_zone tz; |
| 448 | + auto st = serde_str->read_column_from_arrow(*result_col, buggy_array.get(), 0, row_count, tz); |
| 449 | + ASSERT_TRUE(st.ok()) << "Receiver fix should handle int64 offsets in utf8 array: " |
| 450 | + << st.to_string(); |
| 451 | + |
| 452 | + // Verify every row is correctly recovered. |
| 453 | + ASSERT_EQ(static_cast<int64_t>(result_col->size()), row_count); |
| 454 | + for (int64_t i = 0; i < row_count; i++) { |
| 455 | + EXPECT_EQ(result_col->get_data_at(i).to_string(), strings[i]) |
| 456 | + << "Data mismatch at row " << i; |
| 457 | + } |
| 458 | +} |
| 459 | + |
318 | 460 | } // namespace doris |
0 commit comments