Skip to content

Commit 700a983

Browse files
robert3005raulcd
authored andcommitted
GH-52161: [C++][Parquet] Add support for string/binary view to content defined chunking writer (#52162)
### Rationale for this change View types are supported in parquet writer but they error when using content defined chunking ### What changes are included in this PR? Teach the internal chunker how to calculate the size for view types ### Are these changes tested? Added tests ### Are there any user-facing changes? No ### Was AI used for this PR? In accordance to the [AI generation guidelines](https://arrow.apache.org/docs/dev/developers/overview.html#ai-generated-code), please disclose below whether and how AI was used in this PR. **PR code and description written by:** - [x] Human - [ ] AI **Reviewed before submission by:** - [x] Human - [x] AI - [ ] Not reviewed * GitHub Issue: #52161 Authored-by: Robert Kruszewski <[email protected]> Signed-off-by: Zehua Zou <[email protected]>
1 parent ba754c4 commit 700a983

2 files changed

Lines changed: 27 additions & 2 deletions

File tree

‎cpp/src/parquet/chunker_internal.cc‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
#include <cstdint>
2222
#include <iterator>
2323
#include <string>
24+
#include <string_view>
2425
#include <vector>
2526

2627
#include "arrow/array.h"
@@ -349,6 +350,17 @@ class ContentDefinedChunker::Impl {
349350
});
350351
}
351352

353+
std::vector<Chunk> CalculateBinaryView(const int16_t* def_levels,
354+
const int16_t* rep_levels, int64_t num_levels,
355+
const ::arrow::Array& values) {
356+
const auto& array = checked_cast<const ::arrow::BinaryViewArray&>(values);
357+
return Calculate(def_levels, rep_levels, num_levels, [&](int64_t i) {
358+
const std::string_view value = array.GetView(i);
359+
Roll(reinterpret_cast<const uint8_t*>(value.data()),
360+
static_cast<int64_t>(value.size()));
361+
});
362+
}
363+
352364
std::vector<Chunk> GetChunks(const int16_t* def_levels, const int16_t* rep_levels,
353365
int64_t num_levels, const ::arrow::Array& values) {
354366
auto handle_type = [&](auto&& type) -> std::vector<Chunk> {
@@ -380,6 +392,8 @@ class ContentDefinedChunker::Impl {
380392
} else if constexpr (::arrow::is_large_binary_like(ArrowType::type_id)) {
381393
return CalculateBinaryLike<::arrow::LargeBinaryArray>(def_levels, rep_levels,
382394
num_levels, values);
395+
} else if constexpr (::arrow::is_binary_view_like(ArrowType::type_id)) {
396+
return CalculateBinaryView(def_levels, rep_levels, num_levels, values);
383397
} else if constexpr (::arrow::is_dictionary(ArrowType::type_id)) {
384398
return GetChunks(def_levels, rep_levels, num_levels,
385399
*static_cast<const ::arrow::DictionaryArray&>(values).indices());

‎cpp/src/parquet/chunker_internal_test.cc‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
#include "arrow/testing/generator.h"
3131
#include "arrow/type_fwd.h"
3232
#include "arrow/util/float16.h"
33+
#include "arrow/visit_data_inline.h"
3334
#include "parquet/arrow/reader.h"
3435
#include "parquet/arrow/reader_internal.h"
3536
#include "parquet/arrow/test_util.h"
@@ -170,6 +171,10 @@ Result<std::shared_ptr<Array>> GenerateArray(const std::shared_ptr<Field>& field
170171
std::string("bin_") + std::to_string(val))
171172
GENERATE_CASE(LARGE_BINARY, ::arrow::LargeBinaryBuilder,
172173
std::string("bin_") + std::to_string(val))
174+
GENERATE_CASE(STRING_VIEW, ::arrow::StringViewBuilder,
175+
std::string("str_") + std::to_string(val))
176+
GENERATE_CASE(BINARY_VIEW, ::arrow::BinaryViewBuilder,
177+
std::string("bin_") + std::to_string(val))
173178
case ::arrow::Type::FIXED_SIZE_BINARY: {
174179
auto size =
175180
checked_cast<const ::arrow::FixedSizeBinaryType*>(type.get())->byte_width();
@@ -756,6 +761,10 @@ Result<int64_t> CalculateCdcSize(const std::shared_ptr<Array>& array, bool nulla
756761
} else if (::arrow::is_large_binary_like(type_id)) {
757762
auto binary_array = checked_cast<const ::arrow::LargeBinaryArray*>(array.get());
758763
result += binary_array->total_values_length();
764+
} else if (::arrow::is_binary_view_like(type_id)) {
765+
::arrow::VisitArraySpanInline<::arrow::BinaryViewType>(
766+
*array->data(),
767+
[&](std::string_view v) { result += static_cast<int64_t>(v.size()); }, [] {});
759768
} else {
760769
return Status::NotImplemented("CDC size calculation for type ",
761770
array->type()->ToString(), " is not implemented");
@@ -798,7 +807,8 @@ void AssertContentDefinedChunkSizes(const std::shared_ptr<::arrow::ChunkedArray>
798807
ASSERT_EQ(column_info.has_dictionary_page, expect_dictionary_page);
799808
}
800809

801-
if (::arrow::is_fixed_width(type_id) || ::arrow::is_base_binary_like(type_id)) {
810+
if (::arrow::is_fixed_width(type_id) || ::arrow::is_base_binary_like(type_id) ||
811+
::arrow::is_binary_view_like(type_id)) {
802812
int64_t offset = 0;
803813

804814
auto page_lengths = column_info.page_lengths;
@@ -1465,7 +1475,7 @@ INSTANTIATE_TEST_SUITE_P(
14651475
testing::Values(
14661476
CaseConfig{::arrow::boolean(), false}, CaseConfig{::arrow::int64(), true},
14671477
// Binary-like
1468-
CaseConfig{::arrow::utf8(), false},
1478+
CaseConfig{::arrow::utf8(), false}, CaseConfig{::arrow::utf8_view(), true},
14691479
CaseConfig{::arrow::fixed_size_binary(16), true},
14701480
// Nested types
14711481
CaseConfig{::arrow::list(::arrow::int32()), false},
@@ -1489,6 +1499,7 @@ INSTANTIATE_TEST_SUITE_P(
14891499
// Binary-like
14901500
CaseConfig{::arrow::utf8(), false}, CaseConfig{::arrow::binary(), true},
14911501
CaseConfig{::arrow::fixed_size_binary(16), true},
1502+
CaseConfig{::arrow::utf8_view(), false}, CaseConfig{::arrow::binary_view(), true},
14921503
// Temporal
14931504
CaseConfig{::arrow::date32(), false},
14941505
CaseConfig{::arrow::time32(::arrow::TimeUnit::MILLI), true},

0 commit comments

Comments
 (0)