|
25 | 25 | #include <unordered_set> |
26 | 26 |
|
27 | 27 | #include <flatbuffers/flatbuffers.h> |
| 28 | +#include <gmock/gmock-matchers.h> |
28 | 29 | #include <gtest/gtest.h> |
29 | 30 |
|
30 | 31 | #include "arrow/array.h" |
|
52 | 53 | #include "arrow/util/checked_cast.h" |
53 | 54 | #include "arrow/util/io_util.h" |
54 | 55 | #include "arrow/util/key_value_metadata.h" |
| 56 | +#include "arrow/util/ubsan.h" |
55 | 57 |
|
56 | 58 | #include "generated/Message_generated.h" // IWYU pragma: keep |
57 | 59 |
|
@@ -735,6 +737,69 @@ TEST_F(TestWriteRecordBatch, WriteWithCompression) { |
735 | 737 | } |
736 | 738 | } |
737 | 739 |
|
| 740 | +TEST_F(TestWriteRecordBatch, WriteWithCompressionAndMinSavings) { |
| 741 | + // A small batch that's known to be compressible |
| 742 | + auto batch = RecordBatchFromJSON(schema({field("n", int64())}), R"([ |
| 743 | + {"n":0},{"n":1},{"n":2},{"n":3},{"n":4}, |
| 744 | + {"n":5},{"n":6},{"n":7},{"n":8},{"n":9}])"); |
| 745 | + |
| 746 | + auto prefixed_size = [](const Buffer& buffer) -> int64_t { |
| 747 | + if (buffer.size() < int64_t(sizeof(int64_t))) return 0; |
| 748 | + return bit_util::FromLittleEndian(util::SafeLoadAs<int64_t>(buffer.data())); |
| 749 | + }; |
| 750 | + auto content_size = [](const Buffer& buffer) -> int64_t { |
| 751 | + return buffer.size() - sizeof(int64_t); |
| 752 | + }; |
| 753 | + |
| 754 | + for (auto codec : {Compression::LZ4_FRAME, Compression::ZSTD}) { |
| 755 | + if (!util::Codec::IsAvailable(codec)) { |
| 756 | + continue; |
| 757 | + } |
| 758 | + |
| 759 | + auto write_options = IpcWriteOptions::Defaults(); |
| 760 | + ASSERT_OK_AND_ASSIGN(write_options.codec, util::Codec::Create(codec)); |
| 761 | + auto read_options = IpcReadOptions::Defaults(); |
| 762 | + |
| 763 | + IpcPayload payload; |
| 764 | + ASSERT_OK(GetRecordBatchPayload(*batch, write_options, &payload)); |
| 765 | + ASSERT_EQ(payload.body_buffers.size(), 2); |
| 766 | + // Compute the savings when body buffers are compressed unconditionally. We also |
| 767 | + // validate that our test batch is indeed compressible. |
| 768 | + const int64_t uncompressed_size = prefixed_size(*payload.body_buffers[1]); |
| 769 | + const int64_t compressed_size = content_size(*payload.body_buffers[1]); |
| 770 | + ASSERT_LT(compressed_size, uncompressed_size); |
| 771 | + ASSERT_GT(compressed_size, 0); |
| 772 | + const double expected_savings = |
| 773 | + 1.0 - static_cast<double>(compressed_size) / uncompressed_size; |
| 774 | + |
| 775 | + // Using the known savings % as the minimum should yield the same outcome. |
| 776 | + write_options.min_space_savings = expected_savings; |
| 777 | + payload = IpcPayload(); |
| 778 | + ASSERT_OK(GetRecordBatchPayload(*batch, write_options, &payload)); |
| 779 | + ASSERT_EQ(payload.body_buffers.size(), 2); |
| 780 | + ASSERT_EQ(prefixed_size(*payload.body_buffers[1]), uncompressed_size); |
| 781 | + ASSERT_EQ(content_size(*payload.body_buffers[1]), compressed_size); |
| 782 | + CheckRoundtrip(*batch, write_options, read_options); |
| 783 | + |
| 784 | + // Slightly bump the threshold. The body buffer should now be prefixed with -1 and its |
| 785 | + // content left uncompressed. |
| 786 | + write_options.min_space_savings = std::nextafter(expected_savings, 1.0); |
| 787 | + payload = IpcPayload(); |
| 788 | + ASSERT_OK(GetRecordBatchPayload(*batch, write_options, &payload)); |
| 789 | + ASSERT_EQ(payload.body_buffers.size(), 2); |
| 790 | + ASSERT_EQ(prefixed_size(*payload.body_buffers[1]), -1); |
| 791 | + ASSERT_EQ(content_size(*payload.body_buffers[1]), uncompressed_size); |
| 792 | + CheckRoundtrip(*batch, write_options, read_options); |
| 793 | + |
| 794 | + for (double out_of_range : {std::nextafter(1.0, 2.0), std::nextafter(0.0, -1.0)}) { |
| 795 | + write_options.min_space_savings = out_of_range; |
| 796 | + EXPECT_RAISES_WITH_MESSAGE_THAT( |
| 797 | + Invalid, ::testing::StartsWith("Invalid: min_space_savings not in range [0,1]"), |
| 798 | + SerializeRecordBatch(*batch, write_options)); |
| 799 | + } |
| 800 | + } |
| 801 | +} |
| 802 | + |
738 | 803 | TEST_F(TestWriteRecordBatch, SliceTruncatesBinaryOffsets) { |
739 | 804 | // ARROW-6046 |
740 | 805 | std::shared_ptr<Array> array; |
|
0 commit comments