Skip to content

Commit 2ec0215

Browse files
benibuszeroshadelidavidmpitrou
authored
GH-15102: [C++] Could not decompress arrow stream sent from Java arrow SDK (#15194)
* Closes: #15102 Lead-authored-by: benibus <[email protected]> Co-authored-by: Ben Harkins <[email protected]> Co-authored-by: Matt Topol <[email protected]> Co-authored-by: David Li <[email protected]> Co-authored-by: Antoine Pitrou <[email protected]> Signed-off-by: David Li <[email protected]>
1 parent f32c27b commit 2ec0215

6 files changed

Lines changed: 129 additions & 7 deletions

File tree

‎cpp/src/arrow/ipc/options.h‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
#pragma once
1919

2020
#include <cstdint>
21+
#include <optional>
2122
#include <vector>
2223

2324
#include "arrow/io/caching.h"
@@ -67,6 +68,21 @@ struct ARROW_EXPORT IpcWriteOptions {
6768
/// May only be UNCOMPRESSED, LZ4_FRAME and ZSTD.
6869
std::shared_ptr<util::Codec> codec;
6970

71+
/// \brief Minimum space savings percentage required for compression to be applied
72+
///
73+
/// Space savings is calculated as (1.0 - compressed_size / uncompressed_size).
74+
///
75+
/// For example, if min_space_savings = 0.1, a 100-byte body buffer won't undergo
76+
/// compression if its expected compressed size exceeds 90 bytes. If this option is
77+
/// unset, compression will be used indiscriminately. If no codec was supplied, this
78+
/// option is ignored.
79+
///
80+
/// Values outside of the range [0,1] are handled as errors.
81+
///
82+
/// Note that enabling this option may result in unreadable data for Arrow C++ versions
83+
/// prior to 12.0.0.
84+
std::optional<double> min_space_savings;
85+
7086
/// \brief Use global CPU thread pool to parallelize any computational tasks
7187
/// like compression
7288
bool use_threads = true;

‎cpp/src/arrow/ipc/read_write_test.cc‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
#include <unordered_set>
2626

2727
#include <flatbuffers/flatbuffers.h>
28+
#include <gmock/gmock-matchers.h>
2829
#include <gtest/gtest.h>
2930

3031
#include "arrow/array.h"
@@ -52,6 +53,7 @@
5253
#include "arrow/util/checked_cast.h"
5354
#include "arrow/util/io_util.h"
5455
#include "arrow/util/key_value_metadata.h"
56+
#include "arrow/util/ubsan.h"
5557

5658
#include "generated/Message_generated.h" // IWYU pragma: keep
5759

@@ -735,6 +737,69 @@ TEST_F(TestWriteRecordBatch, WriteWithCompression) {
735737
}
736738
}
737739

740+
TEST_F(TestWriteRecordBatch, WriteWithCompressionAndMinSavings) {
741+
// A small batch that's known to be compressible
742+
auto batch = RecordBatchFromJSON(schema({field("n", int64())}), R"([
743+
{"n":0},{"n":1},{"n":2},{"n":3},{"n":4},
744+
{"n":5},{"n":6},{"n":7},{"n":8},{"n":9}])");
745+
746+
auto prefixed_size = [](const Buffer& buffer) -> int64_t {
747+
if (buffer.size() < int64_t(sizeof(int64_t))) return 0;
748+
return bit_util::FromLittleEndian(util::SafeLoadAs<int64_t>(buffer.data()));
749+
};
750+
auto content_size = [](const Buffer& buffer) -> int64_t {
751+
return buffer.size() - sizeof(int64_t);
752+
};
753+
754+
for (auto codec : {Compression::LZ4_FRAME, Compression::ZSTD}) {
755+
if (!util::Codec::IsAvailable(codec)) {
756+
continue;
757+
}
758+
759+
auto write_options = IpcWriteOptions::Defaults();
760+
ASSERT_OK_AND_ASSIGN(write_options.codec, util::Codec::Create(codec));
761+
auto read_options = IpcReadOptions::Defaults();
762+
763+
IpcPayload payload;
764+
ASSERT_OK(GetRecordBatchPayload(*batch, write_options, &payload));
765+
ASSERT_EQ(payload.body_buffers.size(), 2);
766+
// Compute the savings when body buffers are compressed unconditionally. We also
767+
// validate that our test batch is indeed compressible.
768+
const int64_t uncompressed_size = prefixed_size(*payload.body_buffers[1]);
769+
const int64_t compressed_size = content_size(*payload.body_buffers[1]);
770+
ASSERT_LT(compressed_size, uncompressed_size);
771+
ASSERT_GT(compressed_size, 0);
772+
const double expected_savings =
773+
1.0 - static_cast<double>(compressed_size) / uncompressed_size;
774+
775+
// Using the known savings % as the minimum should yield the same outcome.
776+
write_options.min_space_savings = expected_savings;
777+
payload = IpcPayload();
778+
ASSERT_OK(GetRecordBatchPayload(*batch, write_options, &payload));
779+
ASSERT_EQ(payload.body_buffers.size(), 2);
780+
ASSERT_EQ(prefixed_size(*payload.body_buffers[1]), uncompressed_size);
781+
ASSERT_EQ(content_size(*payload.body_buffers[1]), compressed_size);
782+
CheckRoundtrip(*batch, write_options, read_options);
783+
784+
// Slightly bump the threshold. The body buffer should now be prefixed with -1 and its
785+
// content left uncompressed.
786+
write_options.min_space_savings = std::nextafter(expected_savings, 1.0);
787+
payload = IpcPayload();
788+
ASSERT_OK(GetRecordBatchPayload(*batch, write_options, &payload));
789+
ASSERT_EQ(payload.body_buffers.size(), 2);
790+
ASSERT_EQ(prefixed_size(*payload.body_buffers[1]), -1);
791+
ASSERT_EQ(content_size(*payload.body_buffers[1]), uncompressed_size);
792+
CheckRoundtrip(*batch, write_options, read_options);
793+
794+
for (double out_of_range : {std::nextafter(1.0, 2.0), std::nextafter(0.0, -1.0)}) {
795+
write_options.min_space_savings = out_of_range;
796+
EXPECT_RAISES_WITH_MESSAGE_THAT(
797+
Invalid, ::testing::StartsWith("Invalid: min_space_savings not in range [0,1]"),
798+
SerializeRecordBatch(*batch, write_options));
799+
}
800+
}
801+
}
802+
738803
TEST_F(TestWriteRecordBatch, SliceTruncatesBinaryOffsets) {
739804
// ARROW-6046
740805
std::shared_ptr<Array> array;

‎cpp/src/arrow/ipc/reader.cc‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -463,6 +463,10 @@ Result<std::shared_ptr<Buffer>> DecompressBuffer(const std::shared_ptr<Buffer>&
463463
int64_t compressed_size = buf->size() - sizeof(int64_t);
464464
int64_t uncompressed_size = bit_util::FromLittleEndian(util::SafeLoadAs<int64_t>(data));
465465

466+
if (uncompressed_size == -1) {
467+
return SliceBuffer(buf, sizeof(int64_t), compressed_size);
468+
}
469+
466470
ARROW_ASSIGN_OR_RAISE(auto uncompressed,
467471
AllocateBuffer(uncompressed_size, options.memory_pool));
468472

‎cpp/src/arrow/ipc/writer.cc‎

Lines changed: 42 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#include <algorithm>
2121
#include <cstdint>
2222
#include <cstring>
23+
#include <iomanip>
2324
#include <limits>
2425
#include <memory>
2526
#include <sstream>
@@ -176,19 +177,47 @@ class RecordBatchSerializer {
176177
field_nodes_, buffer_meta_, options_, &out_->metadata);
177178
}
178179

180+
bool ShouldCompress(int64_t uncompressed_size, int64_t compressed_size) const {
181+
DCHECK_GT(uncompressed_size, 0);
182+
if (!options_.min_space_savings) return true;
183+
const double space_savings =
184+
1.0 - static_cast<double>(compressed_size) / uncompressed_size;
185+
return space_savings >= *options_.min_space_savings;
186+
}
187+
179188
Status CompressBuffer(const Buffer& buffer, util::Codec* codec,
180189
std::shared_ptr<Buffer>* out) {
181-
// Convert buffer to uncompressed-length-prefixed compressed buffer
190+
// Convert buffer to uncompressed-length-prefixed buffer. The actual body may or may
191+
// not be compressed, depending on user-preference and projected size reduction.
182192
int64_t maximum_length = codec->MaxCompressedLen(buffer.size(), buffer.data());
183-
ARROW_ASSIGN_OR_RAISE(auto result, AllocateBuffer(maximum_length + sizeof(int64_t)));
193+
int64_t prefixed_length = buffer.size();
184194

185-
int64_t actual_length;
186-
ARROW_ASSIGN_OR_RAISE(actual_length,
195+
ARROW_ASSIGN_OR_RAISE(auto result,
196+
AllocateResizableBuffer(maximum_length + sizeof(int64_t)));
197+
ARROW_ASSIGN_OR_RAISE(auto actual_length,
187198
codec->Compress(buffer.size(), buffer.data(), maximum_length,
188199
result->mutable_data() + sizeof(int64_t)));
200+
// FIXME: Not the most sophisticated way to handle this. Ideally, you'd want to avoid
201+
// pre-compressing the entire buffer via some kind of sampling method. As the feature
202+
// gains adoption, this may become a worthwhile optimization.
203+
//
204+
// See: GH-33885
205+
if (!ShouldCompress(buffer.size(), actual_length)) {
206+
if (buffer.size() < actual_length || buffer.size() > maximum_length) {
207+
RETURN_NOT_OK(
208+
result->Resize(buffer.size() + sizeof(int64_t), /*shrink_to_fit=*/false));
209+
result->ZeroPadding();
210+
}
211+
std::memcpy(result->mutable_data() + sizeof(int64_t), buffer.data(),
212+
static_cast<size_t>(buffer.size()));
213+
actual_length = buffer.size();
214+
// Size of -1 indicates to the reader that the body doesn't need to be decompressed
215+
prefixed_length = -1;
216+
}
189217
*reinterpret_cast<int64_t*>(result->mutable_data()) =
190-
bit_util::ToLittleEndian(buffer.size());
218+
bit_util::ToLittleEndian(prefixed_length);
191219
*out = SliceBuffer(std::move(result), /*offset=*/0, actual_length + sizeof(int64_t));
220+
192221
return Status::OK();
193222
}
194223

@@ -230,6 +259,14 @@ class RecordBatchSerializer {
230259
out_->raw_body_length = raw_size;
231260

232261
if (options_.codec != nullptr) {
262+
if (options_.min_space_savings) {
263+
double percentage = *options_.min_space_savings;
264+
if (percentage < 0 || percentage > 1) {
265+
return Status::Invalid(
266+
"min_space_savings not in range [0,1]. Provided: ",
267+
std::setprecision(std::numeric_limits<double>::max_digits10), percentage);
268+
}
269+
}
233270
RETURN_NOT_OK(CompressBodyBuffers());
234271
}
235272

‎go/arrow/ipc/file_reader.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -411,7 +411,7 @@ func (src *ipcSource) buffer(i int) *memory.Buffer {
411411
src.codec.Reset(sr)
412412
r = src.codec
413413
} else {
414-
raw.Resize(int(buf.Length()))
414+
raw.Resize(int(buf.Length() - 8))
415415
}
416416

417417
if _, err = io.ReadFull(r, raw.Bytes()); err != nil {

0 commit comments

Comments
 (0)