|
28 | 28 | #include <span> |
29 | 29 | #include <string> |
30 | 30 | #include <string_view> |
| 31 | +#include <utility> |
31 | 32 | #include <vector> |
32 | 33 |
|
33 | 34 | #include "core/arena.h" |
@@ -216,6 +217,49 @@ std::unique_ptr<arrow::StructBuilder> binary_variant_arrow_builder() { |
216 | 217 | std::make_shared<arrow::BinaryBuilder>(arrow::default_memory_pool())}); |
217 | 218 | } |
218 | 219 |
|
| 220 | +// The struct<metadata, value> storage that Iceberg writes into Parquet. |
| 221 | +std::unique_ptr<arrow::StructBuilder> parquet_variant_arrow_builder() { |
| 222 | + return std::make_unique<arrow::StructBuilder>( |
| 223 | + arrow::struct_({arrow::field("metadata", arrow::binary(), false), |
| 224 | + arrow::field("value", arrow::binary(), false)}), |
| 225 | + arrow::default_memory_pool(), |
| 226 | + std::vector<std::shared_ptr<arrow::ArrayBuilder>> { |
| 227 | + std::make_shared<arrow::BinaryBuilder>(arrow::default_memory_pool()), |
| 228 | + std::make_shared<arrow::BinaryBuilder>(arrow::default_memory_pool())}); |
| 229 | +} |
| 230 | + |
| 231 | +// The Variant rows that the Iceberg and the Paimon writer produce for `column`, as they write them. |
| 232 | +std::vector<std::vector<std::pair<std::string, std::string>>> table_variant_rows( |
| 233 | + const IColumn& column) { |
| 234 | + DataTypeVariantV2SerDe serde; |
| 235 | + auto iceberg = parquet_variant_arrow_builder(); |
| 236 | + Status status = |
| 237 | + serde.write_column_to_iceberg_arrow(nullptr, column, nullptr, nullptr, iceberg.get(), 0, |
| 238 | + column.size(), cctz::utc_time_zone()); |
| 239 | + EXPECT_TRUE(status.ok()) << status; |
| 240 | + auto paimon = binary_variant_arrow_builder(); |
| 241 | + status = serde.write_column_to_paimon_arrow(nullptr, column, nullptr, nullptr, paimon.get(), 0, |
| 242 | + column.size(), cctz::utc_time_zone()); |
| 243 | + EXPECT_TRUE(status.ok()) << status; |
| 244 | + |
| 245 | + std::vector<std::vector<std::pair<std::string, std::string>>> writers; |
| 246 | + const std::array<std::pair<arrow::StructBuilder*, int>, 2> builders { |
| 247 | + {{iceberg.get(), 0}, {paimon.get(), 1}}}; |
| 248 | + for (const auto& [builder, metadata_field] : builders) { |
| 249 | + std::shared_ptr<arrow::Array> output; |
| 250 | + EXPECT_TRUE(builder->Finish(&output).ok()); |
| 251 | + const auto& array = assert_cast<const arrow::StructArray&>(*output); |
| 252 | + const auto& metadata = assert_cast<const arrow::BinaryArray&>(*array.field(metadata_field)); |
| 253 | + const auto& values = |
| 254 | + assert_cast<const arrow::BinaryArray&>(*array.field(1 - metadata_field)); |
| 255 | + auto& rows = writers.emplace_back(); |
| 256 | + for (int64_t row = 0; row < array.length(); ++row) { |
| 257 | + rows.emplace_back(std::string(metadata.GetView(row)), std::string(values.GetView(row))); |
| 258 | + } |
| 259 | + } |
| 260 | + return writers; |
| 261 | +} |
| 262 | + |
219 | 263 | void expect_binary_variant_bytes(const DataTypeVariantV2SerDe& serde, const IColumn& column, |
220 | 264 | const ColumnVariantV2& encoded, |
221 | 265 | const NullMap* null_map = nullptr) { |
@@ -509,6 +553,71 @@ TEST(DataTypeVariantV2SerdeOutputTest, PaimonKeepsItsOwnBinaryLayout) { |
509 | 553 | EXPECT_EQ(binary.GetView(0), std::string_view(expected.data, expected.size)); |
510 | 554 | } |
511 | 555 |
|
| 556 | +TEST(DataTypeVariantV2SerdeOutputTest, TableWritersGiveRowsTheirOwnKeysInsteadOfALargeDictionary) { |
| 557 | + // parse_json() encodes a batch with one dictionary of all its keys: 256 rows with a distinct |
| 558 | + // 200-byte key share a 52 KiB dictionary. Copied into every row it would cost 13 MiB for 2 KiB of |
| 559 | + // values, and 4064 such rows overflow an Arrow binary array, so the Iceberg and the Paimon writer |
| 560 | + // give each row a dictionary of its own key; the row still decodes to its key and value. Row 0 |
| 561 | + // holds a 64 KiB string, which the shared dictionary does not exceed by more than 1 KiB, so it |
| 562 | + // keeps that dictionary. MUTATION: keeping every shared dictionary, or giving row 0 its own key |
| 563 | + // -> red. |
| 564 | + constexpr size_t ROWS = 256; |
| 565 | + JsonStringToVariantEncoder encoder( |
| 566 | + {.max_json_key_length = 1024, .throw_on_invalid_json = true}); |
| 567 | + std::vector<std::string> keys; |
| 568 | + for (size_t row = 0; row < ROWS; ++row) { |
| 569 | + keys.push_back("k" + std::to_string(row) + std::string(200, 'x')); |
| 570 | + const std::string value = |
| 571 | + row == 0 ? "\"" + std::string(64 * 1024, 'v') + "\"" : std::to_string(row); |
| 572 | + const std::string document = "{\"" + keys.back() + "\":" + value + "}"; |
| 573 | + encoder.add_json({document.data(), document.size()}); |
| 574 | + } |
| 575 | + VariantBatchBuilder batch = encoder.finish_batch(); |
| 576 | + auto column = ColumnVariantV2::create(); |
| 577 | + column->insert_encoded_batch(batch); |
| 578 | + const VariantRef shared = column->get_value_ref(1); |
| 579 | + ASSERT_EQ(shared.metadata.dict_size(), ROWS); |
| 580 | + |
| 581 | + for (const auto& rows : table_variant_rows(*column)) { |
| 582 | + ASSERT_EQ(rows.size(), ROWS); |
| 583 | + EXPECT_EQ(rows[0].first, std::string_view(shared.metadata.data, shared.metadata.size)); |
| 584 | + for (size_t row = 0; row < ROWS; ++row) { |
| 585 | + const VariantRef value { |
| 586 | + .metadata = {.data = rows[row].first.data(), .size = rows[row].first.size()}, |
| 587 | + .value = {rows[row].second.data(), rows[row].second.size()}}; |
| 588 | + if (row != 0) { |
| 589 | + EXPECT_EQ(value.metadata.dict_size(), 1U) << row; |
| 590 | + } |
| 591 | + ASSERT_EQ(value.basic_type(), VariantBasicType::OBJECT) << row; |
| 592 | + ASSERT_EQ(value.num_elements(), 1U) << row; |
| 593 | + uint32_t field_id = 0; |
| 594 | + const VariantRef child = value.object_view().value_at(0, &field_id); |
| 595 | + EXPECT_EQ(value.metadata.key_at(field_id).to_string(), keys[row]); |
| 596 | + if (row == 0) { |
| 597 | + EXPECT_EQ(child.get_string().size, size_t {64 * 1024}); |
| 598 | + } else { |
| 599 | + EXPECT_EQ(child.get_int(), static_cast<int64_t>(row)); |
| 600 | + } |
| 601 | + } |
| 602 | + } |
| 603 | +} |
| 604 | + |
| 605 | +TEST(DataTypeVariantV2SerdeOutputTest, TableWritersKeepASmallSharedDictionary) { |
| 606 | + // A dictionary that exceeds a row's value by at most 1 KiB is written as the batch shares it: |
| 607 | + // Parquet dictionary-encodes the metadata the rows then have in common, and the row is not |
| 608 | + // re-encoded. MUTATION: giving every row its own keys, as Flight does -> red. |
| 609 | + auto column = encoded_json({R"({"a":1})", R"({"b":[true,"x"]})", R"({"a":{"b":2}})"}); |
| 610 | + for (const auto& rows : table_variant_rows(*column)) { |
| 611 | + ASSERT_EQ(rows.size(), column->size()); |
| 612 | + for (size_t row = 0; row < column->size(); ++row) { |
| 613 | + const VariantRef expected = column->get_value_ref(row); |
| 614 | + EXPECT_EQ(rows[row].first, |
| 615 | + std::string_view(expected.metadata.data, expected.metadata.size)); |
| 616 | + EXPECT_EQ(rows[row].second, std::string_view(expected.value.data, expected.value.size)); |
| 617 | + } |
| 618 | + } |
| 619 | +} |
| 620 | + |
512 | 621 | TEST(DataTypeVariantV2SerdeOutputTest, BinaryStructRejectsUnsupportedPrimitive) { |
513 | 622 | DataTypeVariantV2SerDe serde; |
514 | 623 | VariantBatchBuilder builder(VariantBatchBuilder::ReserveHint {.rows = 1}); |
|
0 commit comments