Skip to content

Commit f622139

Browse files
authored
GH-37710: [C++][Integration] Add C++ Utf8View implementation (#37792)
### Rationale for this change After the PR changing the spec and schema ( #37526 ) is accepted, this PR will be undrafted. It adds the minimal addition of a C++ implementation and was extracted from the original C++ Utf8View pr ( #35628 ) for ease of review. ### What changes are included in this PR? - The new types are available with new subclasses of DataType, Array, ArrayBuilder, ... - The values of string view arrays can be visited as `std::string_view` as with StringArray - String view arrays can be round tripped through IPC and integration JSON * Closes: #37710 Relevant mailing list discussions: * https://lists.apache.org/thread/l8t1vj5x1wdf75mdw3wfjvnxrfy5xomy * https://lists.apache.org/thread/3qhkomvvc69v3gkotbwldyko7yk9cs9k Authored-by: Benjamin Kietzman <[email protected]> Signed-off-by: Benjamin Kietzman <[email protected]>
1 parent 818f71d commit f622139

69 files changed

Lines changed: 1951 additions & 375 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎cpp/src/arrow/array/array_base.cc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,8 @@ struct ScalarFromArraySlotImpl {
8787
return Finish(a.GetString(index_));
8888
}
8989

90+
Status Visit(const BinaryViewArray& a) { return Finish(a.GetString(index_)); }
91+
9092
Status Visit(const FixedSizeBinaryArray& a) { return Finish(a.GetString(index_)); }
9193

9294
Status Visit(const DayTimeIntervalArray& a) { return Finish(a.Value(index_)); }

‎cpp/src/arrow/array/array_binary.cc‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
#include "arrow/array/validate.h"
2525
#include "arrow/type.h"
2626
#include "arrow/type_traits.h"
27+
#include "arrow/util/binary_view_util.h"
2728
#include "arrow/util/checked_cast.h"
2829
#include "arrow/util/logging.h"
2930

@@ -89,6 +90,33 @@ LargeStringArray::LargeStringArray(int64_t length,
8990

9091
Status LargeStringArray::ValidateUTF8() const { return internal::ValidateUTF8(*data_); }
9192

93+
BinaryViewArray::BinaryViewArray(std::shared_ptr<ArrayData> data) {
94+
ARROW_CHECK_EQ(data->type->id(), Type::BINARY_VIEW);
95+
SetData(std::move(data));
96+
}
97+
98+
BinaryViewArray::BinaryViewArray(std::shared_ptr<DataType> type, int64_t length,
99+
std::shared_ptr<Buffer> views, BufferVector buffers,
100+
std::shared_ptr<Buffer> null_bitmap, int64_t null_count,
101+
int64_t offset) {
102+
buffers.insert(buffers.begin(), std::move(views));
103+
buffers.insert(buffers.begin(), std::move(null_bitmap));
104+
SetData(
105+
ArrayData::Make(std::move(type), length, std::move(buffers), null_count, offset));
106+
}
107+
108+
std::string_view BinaryViewArray::GetView(int64_t i) const {
109+
const std::shared_ptr<Buffer>* data_buffers = data_->buffers.data() + 2;
110+
return util::FromBinaryView(raw_values_[i], data_buffers);
111+
}
112+
113+
StringViewArray::StringViewArray(std::shared_ptr<ArrayData> data) {
114+
ARROW_CHECK_EQ(data->type->id(), Type::STRING_VIEW);
115+
SetData(std::move(data));
116+
}
117+
118+
Status StringViewArray::ValidateUTF8() const { return internal::ValidateUTF8(*data_); }
119+
92120
FixedSizeBinaryArray::FixedSizeBinaryArray(const std::shared_ptr<ArrayData>& data) {
93121
SetData(data);
94122
}

‎cpp/src/arrow/array/array_binary.h‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222

2323
#include <cstdint>
2424
#include <memory>
25+
#include <optional>
2526
#include <string>
2627
#include <string_view>
2728
#include <vector>
@@ -217,6 +218,65 @@ class ARROW_EXPORT LargeStringArray : public LargeBinaryArray {
217218
Status ValidateUTF8() const;
218219
};
219220

221+
// ----------------------------------------------------------------------
222+
// BinaryView and StringView
223+
224+
/// Concrete Array class for variable-size binary view data using the
225+
/// BinaryViewType::c_type struct to reference in-line or out-of-line string values
226+
class ARROW_EXPORT BinaryViewArray : public FlatArray {
227+
public:
228+
using TypeClass = BinaryViewType;
229+
using IteratorType = stl::ArrayIterator<BinaryViewArray>;
230+
using c_type = BinaryViewType::c_type;
231+
232+
explicit BinaryViewArray(std::shared_ptr<ArrayData> data);
233+
234+
BinaryViewArray(std::shared_ptr<DataType> type, int64_t length,
235+
std::shared_ptr<Buffer> views, BufferVector data_buffers,
236+
std::shared_ptr<Buffer> null_bitmap = NULLPTR,
237+
int64_t null_count = kUnknownNullCount, int64_t offset = 0);
238+
239+
// For API compatibility with BinaryArray etc.
240+
std::string_view GetView(int64_t i) const;
241+
std::string GetString(int64_t i) const { return std::string{GetView(i)}; }
242+
243+
const auto& values() const { return data_->buffers[1]; }
244+
const c_type* raw_values() const { return raw_values_; }
245+
246+
std::optional<std::string_view> operator[](int64_t i) const {
247+
return *IteratorType(*this, i);
248+
}
249+
250+
IteratorType begin() const { return IteratorType(*this); }
251+
IteratorType end() const { return IteratorType(*this, length()); }
252+
253+
protected:
254+
using FlatArray::FlatArray;
255+
256+
void SetData(std::shared_ptr<ArrayData> data) {
257+
FlatArray::SetData(std::move(data));
258+
raw_values_ = data_->GetValuesSafe<c_type>(1);
259+
}
260+
261+
const c_type* raw_values_;
262+
};
263+
264+
/// Concrete Array class for variable-size string view (utf-8) data using
265+
/// BinaryViewType::c_type to reference in-line or out-of-line string values
266+
class ARROW_EXPORT StringViewArray : public BinaryViewArray {
267+
public:
268+
using TypeClass = StringViewType;
269+
270+
explicit StringViewArray(std::shared_ptr<ArrayData> data);
271+
272+
using BinaryViewArray::BinaryViewArray;
273+
274+
/// \brief Validate that this array contains only valid UTF8 entries
275+
///
276+
/// This check is also implied by ValidateFull()
277+
Status ValidateUTF8() const;
278+
};
279+
220280
// ----------------------------------------------------------------------
221281
// Fixed width binary
222282

‎cpp/src/arrow/array/array_binary_test.cc‎

Lines changed: 121 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -27,17 +27,21 @@
2727

2828
#include "arrow/array.h"
2929
#include "arrow/array/builder_binary.h"
30+
#include "arrow/array/validate.h"
3031
#include "arrow/buffer.h"
3132
#include "arrow/memory_pool.h"
3233
#include "arrow/status.h"
3334
#include "arrow/testing/builder.h"
3435
#include "arrow/testing/gtest_util.h"
36+
#include "arrow/testing/matchers.h"
3537
#include "arrow/testing/util.h"
3638
#include "arrow/type.h"
3739
#include "arrow/type_traits.h"
3840
#include "arrow/util/bit_util.h"
3941
#include "arrow/util/bitmap_builders.h"
4042
#include "arrow/util/checked_cast.h"
43+
#include "arrow/util/key_value_metadata.h"
44+
#include "arrow/util/logging.h"
4145
#include "arrow/visit_data_inline.h"
4246

4347
namespace arrow {
@@ -365,38 +369,134 @@ TYPED_TEST(TestStringArray, TestValidateOffsets) { this->TestValidateOffsets();
365369

366370
TYPED_TEST(TestStringArray, TestValidateData) { this->TestValidateData(); }
367371

372+
// Produce an Array of index/offset views from a std::vector of index/offset
373+
// BinaryViewType::c_type
374+
Result<std::shared_ptr<StringViewArray>> MakeBinaryViewArray(
375+
BufferVector data_buffers, const std::vector<BinaryViewType::c_type>& views,
376+
bool validate = true) {
377+
auto length = static_cast<int64_t>(views.size());
378+
auto arr = std::make_shared<StringViewArray>(
379+
utf8_view(), length, Buffer::FromVector(views), std::move(data_buffers));
380+
if (validate) {
381+
RETURN_NOT_OK(arr->ValidateFull());
382+
}
383+
return arr;
384+
}
385+
386+
TEST(StringViewArray, Validate) {
387+
// Since this is a test of validation, we need to be able to construct invalid arrays.
388+
auto buffer_s = Buffer::FromString("supercalifragilistic(sp?)");
389+
auto buffer_y = Buffer::FromString("yyyyyyyyyyyyyyyyyyyyyyyyy");
390+
391+
// empty array is valid
392+
EXPECT_THAT(MakeBinaryViewArray({}, {}), Ok());
393+
394+
// empty array with some data buffers is valid
395+
EXPECT_THAT(MakeBinaryViewArray({buffer_s, buffer_y}, {}), Ok());
396+
397+
// inline views need not have a corresponding buffer
398+
EXPECT_THAT(MakeBinaryViewArray({},
399+
{
400+
util::ToInlineBinaryView("hello"),
401+
util::ToInlineBinaryView("world"),
402+
util::ToInlineBinaryView("inline me"),
403+
}),
404+
Ok());
405+
406+
// non-inline views are expected to reference only buffers managed by the array
407+
EXPECT_THAT(
408+
MakeBinaryViewArray(
409+
{buffer_s, buffer_y},
410+
{util::ToBinaryView("supe", static_cast<int32_t>(buffer_s->size()), 0, 0),
411+
util::ToBinaryView("yyyy", static_cast<int32_t>(buffer_y->size()), 1, 0)}),
412+
Ok());
413+
414+
// views may not reference data buffers not present in the array
415+
EXPECT_THAT(
416+
MakeBinaryViewArray(
417+
{}, {util::ToBinaryView("supe", static_cast<int32_t>(buffer_s->size()), 0, 0)}),
418+
Raises(StatusCode::IndexError));
419+
// ... or ranges which overflow the referenced data buffer
420+
EXPECT_THAT(
421+
MakeBinaryViewArray(
422+
{buffer_s}, {util::ToBinaryView(
423+
"supe", static_cast<int32_t>(buffer_s->size() + 50), 0, 0)}),
424+
Raises(StatusCode::IndexError));
425+
426+
// Additionally, the prefixes of non-inline views must match the data buffer
427+
EXPECT_THAT(
428+
MakeBinaryViewArray(
429+
{buffer_s, buffer_y},
430+
{util::ToBinaryView("SUPE", static_cast<int32_t>(buffer_s->size()), 0, 0),
431+
util::ToBinaryView("yyyy", static_cast<int32_t>(buffer_y->size()), 1, 0)}),
432+
Raises(StatusCode::Invalid));
433+
434+
// Invalid string views which are masked by a null bit do not cause validation to fail
435+
auto invalid_but_masked =
436+
MakeBinaryViewArray(
437+
{buffer_s},
438+
{util::ToBinaryView("SUPE", static_cast<int32_t>(buffer_s->size()), 0, 0),
439+
util::ToBinaryView("yyyy", 50, 40, 30)},
440+
/*validate=*/false)
441+
.ValueOrDie()
442+
->data();
443+
invalid_but_masked->null_count = 2;
444+
invalid_but_masked->buffers[0] = *AllocateEmptyBitmap(2);
445+
EXPECT_THAT(internal::ValidateArrayFull(*invalid_but_masked), Ok());
446+
447+
// overlapping views are allowed
448+
EXPECT_THAT(
449+
MakeBinaryViewArray(
450+
{buffer_s},
451+
{
452+
util::ToBinaryView("supe", static_cast<int32_t>(buffer_s->size()), 0, 0),
453+
util::ToBinaryView("uper", static_cast<int32_t>(buffer_s->size() - 1), 0,
454+
1),
455+
util::ToBinaryView("perc", static_cast<int32_t>(buffer_s->size() - 2), 0,
456+
2),
457+
util::ToBinaryView("erca", static_cast<int32_t>(buffer_s->size() - 3), 0,
458+
3),
459+
}),
460+
Ok());
461+
}
462+
368463
template <typename T>
369464
class TestUTF8Array : public ::testing::Test {
370465
public:
371466
using TypeClass = T;
372-
using offset_type = typename TypeClass::offset_type;
373467
using ArrayType = typename TypeTraits<TypeClass>::ArrayType;
374468

375-
Status ValidateUTF8(int64_t length, std::vector<offset_type> offsets,
376-
std::string_view data, int64_t offset = 0) {
377-
ArrayType arr(length, Buffer::Wrap(offsets), std::make_shared<Buffer>(data),
378-
/*null_bitmap=*/nullptr, /*null_count=*/0, offset);
379-
return arr.ValidateUTF8();
469+
std::shared_ptr<DataType> type() const {
470+
if constexpr (is_binary_view_like_type<TypeClass>::value) {
471+
return TypeClass::is_utf8 ? utf8_view() : binary_view();
472+
} else {
473+
return TypeTraits<TypeClass>::type_singleton();
474+
}
380475
}
381476

382-
Status ValidateUTF8(const std::string& json) {
383-
auto ty = TypeTraits<T>::type_singleton();
384-
auto arr = ArrayFromJSON(ty, json);
385-
return checked_cast<const ArrayType&>(*arr).ValidateUTF8();
477+
Status ValidateUTF8(const Array& arr) {
478+
return checked_cast<const ArrayType&>(arr).ValidateUTF8();
479+
}
480+
481+
Status ValidateUTF8(std::vector<std::string> values) {
482+
std::shared_ptr<Array> arr;
483+
ArrayFromVector<T, std::string>(type(), values, &arr);
484+
return ValidateUTF8(*arr);
386485
}
387486

388487
void TestValidateUTF8() {
389-
ASSERT_OK(ValidateUTF8(R"(["Voix", "ambiguë", "d’un", "cœur"])"));
390-
ASSERT_OK(ValidateUTF8(1, {0, 4}, "\xf4\x8f\xbf\xbf")); // \U0010ffff
488+
ASSERT_OK(
489+
ValidateUTF8(*ArrayFromJSON(type(), R"(["Voix", "ambiguë", "d’un", "cœur"])")));
490+
ASSERT_OK(ValidateUTF8({"\xf4\x8f\xbf\xbf"})); // \U0010ffff
391491

392-
ASSERT_RAISES(Invalid, ValidateUTF8(1, {0, 1}, "\xf4"));
492+
ASSERT_RAISES(Invalid, ValidateUTF8({"\xf4"}));
393493

394494
// More tests in TestValidateData() above
395495
// (ValidateFull() calls ValidateUTF8() internally)
396496
}
397497
};
398498

399-
TYPED_TEST_SUITE(TestUTF8Array, StringArrowTypes);
499+
TYPED_TEST_SUITE(TestUTF8Array, StringOrStringViewArrowTypes);
400500

401501
TYPED_TEST(TestUTF8Array, TestValidateUTF8) { this->TestValidateUTF8(); }
402502

@@ -883,11 +983,15 @@ class TestBaseBinaryDataVisitor : public ::testing::Test {
883983
void SetUp() override { type_ = TypeTraits<TypeClass>::type_singleton(); }
884984

885985
void TestBasics() {
886-
auto array = ArrayFromJSON(type_, R"(["foo", null, "bar"])");
986+
auto array = ArrayFromJSON(
987+
type_,
988+
R"(["foo", null, "bar", "inline_me", "allocate_me_aaaaa", "allocate_me_bbbb"])");
887989
BinaryAppender appender;
888990
ArraySpanVisitor<TypeClass> visitor;
889991
ASSERT_OK(visitor.Visit(*array->data(), &appender));
890-
ASSERT_THAT(appender.data, ::testing::ElementsAreArray({"foo", "(null)", "bar"}));
992+
ASSERT_THAT(appender.data,
993+
::testing::ElementsAreArray({"foo", "(null)", "bar", "inline_me",
994+
"allocate_me_aaaaa", "allocate_me_bbbb"}));
891995
ARROW_UNUSED(visitor); // Workaround weird MSVC warning
892996
}
893997

@@ -904,7 +1008,7 @@ class TestBaseBinaryDataVisitor : public ::testing::Test {
9041008
std::shared_ptr<DataType> type_;
9051009
};
9061010

907-
TYPED_TEST_SUITE(TestBaseBinaryDataVisitor, BaseBinaryArrowTypes);
1011+
TYPED_TEST_SUITE(TestBaseBinaryDataVisitor, BaseBinaryOrBinaryViewLikeArrowTypes);
9081012

9091013
TYPED_TEST(TestBaseBinaryDataVisitor, Basics) { this->TestBasics(); }
9101014

‎cpp/src/arrow/array/array_test.cc‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -382,10 +382,12 @@ static std::vector<std::shared_ptr<DataType>> TestArrayUtilitiesAgainstTheseType
382382
float64(),
383383
binary(),
384384
large_binary(),
385+
binary_view(),
385386
fixed_size_binary(3),
386387
decimal(16, 4),
387388
utf8(),
388389
large_utf8(),
390+
utf8_view(),
389391
list(utf8()),
390392
list(int64()), // NOTE: Regression case for ARROW-9071/MakeArrayOfNull
391393
list(large_utf8()),
@@ -601,12 +603,15 @@ static ScalarVector GetScalars() {
601603
std::make_shared<DurationScalar>(60, duration(TimeUnit::SECOND)),
602604
std::make_shared<BinaryScalar>(hello),
603605
std::make_shared<LargeBinaryScalar>(hello),
606+
std::make_shared<BinaryViewScalar>(hello),
604607
std::make_shared<FixedSizeBinaryScalar>(
605608
hello, fixed_size_binary(static_cast<int32_t>(hello->size()))),
606609
std::make_shared<Decimal128Scalar>(Decimal128(10), decimal(16, 4)),
607610
std::make_shared<Decimal256Scalar>(Decimal256(10), decimal(76, 38)),
608611
std::make_shared<StringScalar>(hello),
609612
std::make_shared<LargeStringScalar>(hello),
613+
std::make_shared<StringViewScalar>(hello),
614+
std::make_shared<StringViewScalar>(Buffer::FromString("long string; not inlined")),
610615
std::make_shared<ListScalar>(ArrayFromJSON(int8(), "[1, 2, 3]")),
611616
ScalarFromJSON(map(int8(), utf8()), R"([[1, "foo"], [2, "bar"]])"),
612617
std::make_shared<LargeListScalar>(ArrayFromJSON(int8(), "[1, 1, 2, 2, 3, 3]")),
@@ -647,13 +652,14 @@ TEST_F(TestArray, TestMakeArrayFromScalar) {
647652

648653
for (int64_t length : {16}) {
649654
for (auto scalar : scalars) {
655+
ARROW_SCOPED_TRACE("scalar type: ", scalar->type->ToString());
650656
ASSERT_OK_AND_ASSIGN(auto array, MakeArrayFromScalar(*scalar, length));
651657
ASSERT_OK(array->ValidateFull());
652658
ASSERT_EQ(array->length(), length);
653659
ASSERT_EQ(array->null_count(), 0);
654660

655661
// test case for ARROW-13321
656-
for (int64_t i : std::vector<int64_t>{0, length / 2, length - 1}) {
662+
for (int64_t i : {int64_t{0}, length / 2, length - 1}) {
657663
ASSERT_OK_AND_ASSIGN(auto s, array->GetScalar(i));
658664
AssertScalarsEqual(*s, *scalar, /*verbose=*/true);
659665
}

0 commit comments

Comments
 (0)