|
24 | 24 |
|
25 | 25 | #include <cstddef> |
26 | 26 | #include <iostream> |
| 27 | +#include <limits> |
27 | 28 | #include <type_traits> |
| 29 | +#include <vector> |
28 | 30 |
|
29 | 31 | #include "core/assert_cast.h" |
30 | 32 | #include "core/column/column.h" |
@@ -274,4 +276,87 @@ TEST_F(DataTypeTimeStampTzSerDeTest, binary_roundtrip) { |
274 | 276 | test_func(*serde_tz_3, column_tz_3, 3); |
275 | 277 | test_func(*serde_tz_6, column_tz_6, 6); |
276 | 278 | } |
| 279 | + |
| 280 | +// Roundtrip through Arrow. Same failure shape as binary_roundtrip above: read_column_from_arrow |
| 281 | +// was inherited from DataTypeNumberSerDe, whose fixed-width path memcpy'd Arrow's int64 EPOCH |
| 282 | +// values over this column's PACKED values. Both are 8 bytes, so nothing failed -- the read |
| 283 | +// succeeded with every row wrong. Comparing the value that comes back against the one that went |
| 284 | +// out is what makes deleting the override fail here instead of in a user's result set. |
| 285 | +TEST_F(DataTypeTimeStampTzSerDeTest, ArrowRoundTrip) { |
| 286 | + auto test_func = [&](const DataTypeTimeStampTzSerDe& serde, arrow::TimeUnit::type unit, |
| 287 | + uint16_t year, uint8_t month, uint8_t day, uint8_t hour, uint8_t minute, |
| 288 | + uint8_t second, uint32_t microsecond, int64_t expected_arrow_value) { |
| 289 | + auto source_column = ColumnTimeStampTz::create(); |
| 290 | + TimestampTzValue source_value; |
| 291 | + source_value.unchecked_set_time(year, month, day, hour, minute, second, microsecond); |
| 292 | + source_column->insert_value(source_value); |
| 293 | + |
| 294 | + arrow::TimestampBuilder builder(arrow::timestamp(unit), arrow::default_memory_pool()); |
| 295 | + ASSERT_TRUE(serde.write_column_to_arrow(*source_column, nullptr, &builder, 0, |
| 296 | + source_column->size(), cctz::utc_time_zone()) |
| 297 | + .ok()); |
| 298 | + std::shared_ptr<arrow::Array> array; |
| 299 | + ASSERT_TRUE(builder.Finish(&array).ok()); |
| 300 | + // Pinned so the roundtrip cannot pass by both halves sharing one wrong encoding: what |
| 301 | + // crosses the wire is an epoch count in the unit, not the column's packed representation. |
| 302 | + ASSERT_EQ(expected_arrow_value, |
| 303 | + assert_cast<const arrow::TimestampArray*>(array.get())->Value(0)); |
| 304 | + |
| 305 | + auto dest_column = ColumnTimeStampTz::create(); |
| 306 | + ASSERT_TRUE(serde.read_column_from_arrow(*dest_column, array.get(), 0, array->length(), |
| 307 | + cctz::utc_time_zone()) |
| 308 | + .ok()); |
| 309 | + ASSERT_EQ(1, dest_column->size()); |
| 310 | + EXPECT_EQ(source_value, dest_column->get_element(0)); |
| 311 | + }; |
| 312 | + |
| 313 | + // Before the epoch, so a division that truncates toward zero instead of flooring lands one |
| 314 | + // unit late rather than agreeing by accident. |
| 315 | + test_func(*serde_tz_6, arrow::TimeUnit::MICRO, 1969, 12, 31, 23, 59, 59, 123456, -876544); |
| 316 | + test_func(*serde_tz_3, arrow::TimeUnit::MILLI, 1969, 12, 31, 23, 59, 59, 123000, -877); |
| 317 | + test_func(*serde_tz_0, arrow::TimeUnit::SECOND, 1969, 12, 31, 23, 59, 59, 0, -1); |
| 318 | + test_func(*serde_tz_6, arrow::TimeUnit::MICRO, 2023, 1, 2, 3, 4, 5, 123456, 1672628645123456); |
| 319 | +} |
| 320 | + |
| 321 | +// An Arrow type this serde cannot decode must fail the scan, not produce values. The inherited |
| 322 | +// reader accepted anything 8 bytes wide, which is how the corruption above stayed invisible. |
| 323 | +TEST_F(DataTypeTimeStampTzSerDeTest, ReadArrowRejectsNonTimestampType) { |
| 324 | + arrow::Int64Builder builder; |
| 325 | + ASSERT_TRUE(builder.Append(1672628645123456).ok()); |
| 326 | + std::shared_ptr<arrow::Array> array; |
| 327 | + ASSERT_TRUE(builder.Finish(&array).ok()); |
| 328 | + |
| 329 | + auto dest_column = ColumnTimeStampTz::create(); |
| 330 | + auto st = serde_tz_6->read_column_from_arrow(*dest_column, array.get(), 0, array->length(), |
| 331 | + cctz::utc_time_zone()); |
| 332 | + EXPECT_FALSE(st.ok()); |
| 333 | + EXPECT_EQ(0, dest_column->size()); |
| 334 | +} |
| 335 | + |
| 336 | +// A null slot's value buffer holds whatever the source left there. Converting it would range-check |
| 337 | +// garbage and fail a batch whose rows are all well-formed, so nulls take a default value instead. |
| 338 | +TEST_F(DataTypeTimeStampTzSerDeTest, ReadArrowIgnoresValuesUnderNulls) { |
| 339 | + TimestampTzValue source_value; |
| 340 | + source_value.unchecked_set_time(2023, 1, 2, 3, 4, 5, 123456); |
| 341 | + |
| 342 | + std::vector<int64_t> values = {1672628645123456, std::numeric_limits<int64_t>::max()}; |
| 343 | + // Bit 0 set, bit 1 clear: row 0 valid, row 1 null. |
| 344 | + std::vector<uint8_t> validity = {0b01}; |
| 345 | + auto data = arrow::ArrayData::Make(arrow::timestamp(arrow::TimeUnit::MICRO), values.size(), |
| 346 | + {arrow::Buffer::Wrap(validity.data(), validity.size()), |
| 347 | + arrow::Buffer::Wrap(values.data(), values.size())}, |
| 348 | + /*null_count=*/1); |
| 349 | + std::shared_ptr<arrow::Array> array = arrow::MakeArray(data); |
| 350 | + ASSERT_TRUE(array->IsNull(1)); |
| 351 | + |
| 352 | + auto dest_column = ColumnTimeStampTz::create(); |
| 353 | + ASSERT_TRUE(serde_tz_6 |
| 354 | + ->read_column_from_arrow(*dest_column, array.get(), 0, array->length(), |
| 355 | + cctz::utc_time_zone()) |
| 356 | + .ok()); |
| 357 | + // One value per row, nulls included: the caller took the validity bitmap and expects the |
| 358 | + // nested column to have grown by the full range. |
| 359 | + ASSERT_EQ(2, dest_column->size()); |
| 360 | + EXPECT_EQ(source_value, dest_column->get_element(0)); |
| 361 | +} |
277 | 362 | } // namespace doris |
0 commit comments