Skip to content

Commit 4b48ecc

Browse files
authored
GH-33607: [C++] Support optional additional arguments for inline visit functions (#33608)
# Which issue does this PR close? Closes #33607 # Rationale for this change Sometimes we need extra arguments for the `Visit` function, the most common of which is output parameters to save the result. For now, we need to have a member variable and a member function to get the result: ```cpp class ExampleVisitor { public: template <typename T> Status Visit(const T& arr) { /// Do stuff and save result to output_; return Status::OK(); } std::shared_ptr<Array> GetOutput() { return output_; } private: std::shared_ptr<Array> output_; }; ExampleVisitor visitor; RETURN_NOT_OK(VisitArrayInline(*arr, &visitor)); *output = visitor.GetOutput(); ``` It will be more convenient to write a Visitor if the VisitArrayInline function supports additional args for the `Visit` method: ```cpp class ExampleVisitorWithArg { public: template <typename T> Status Visit(const T& arr, std::shared_ptr<Array>* output) { /// Do stuff and save result to output directly; return Status::OK(); } }; ExampleVisitorWithArg visitor; RETURN_NOT_OK(VisitArrayInline(*arr, &visitor, &output)); ``` # Are these changes tested? Not sure if we need to explicitly test this feature, since no runtime behavior is changed. The existing tests all pass. # Are there any user-facing changes? There is a small chance, like in [diff.cc:402](https://github.com/apache/arrow/pull/33608/files#diff-ef9d394f251ee719f9896b864246bd97a81165c08195d4846d5d99e589e5c1c0L402), `VisitTypeInline` is declared as a friend function. The declaration needs to be changed. But I am not sure if any outside user would do this. Authored-by: Jin Shang <[email protected]> Signed-off-by: Antoine Pitrou <[email protected]>
1 parent 4d9228f commit 4b48ecc

6 files changed

Lines changed: 68 additions & 60 deletions

File tree

‎cpp/src/arrow/array/diff.cc‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -399,8 +399,8 @@ class MakeFormatterImpl {
399399
}
400400

401401
private:
402-
template <typename VISITOR>
403-
friend Status VisitTypeInline(const DataType&, VISITOR*);
402+
template <typename VISITOR, typename... ARGS>
403+
friend Status VisitTypeInline(const DataType&, VISITOR*, ARGS&&... args);
404404

405405
// factory implementation
406406
Status Visit(const BooleanType&) {

‎cpp/src/arrow/testing/builder.h‎

Lines changed: 9 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
#include <cstdint>
2121
#include <memory>
22+
#include <type_traits>
2223
#include <vector>
2324

2425
#include "arrow/array.h"
@@ -27,6 +28,7 @@
2728
#include "arrow/array/builder_time.h"
2829
#include "arrow/buffer.h"
2930
#include "arrow/testing/gtest_util.h"
31+
#include "arrow/type_fwd.h"
3032
#include "arrow/util/bit_util.h"
3133
#include "arrow/visit_type_inline.h"
3234

@@ -159,31 +161,21 @@ Status MakeArray(const std::vector<uint8_t>& valid_bytes, const std::vector<T>&
159161
}
160162

161163
template <typename Fn>
162-
struct VisitBuilderImpl {
164+
struct VisitBuilder {
163165
template <typename T, typename BuilderType = typename TypeTraits<T>::BuilderType,
164166
// need to let SFINAE drop this Visit when it would result in
165167
// [](NullBuilder*){}(double_builder)
166168
typename = decltype(std::declval<Fn>()(std::declval<BuilderType*>()))>
167-
Status Visit(const T&) {
168-
fn_(internal::checked_cast<BuilderType*>(builder_));
169+
Status Visit(const T&, ArrayBuilder* builder, Fn&& fn) {
170+
fn(internal::checked_cast<BuilderType*>(builder));
169171
return Status::OK();
170172
}
171173

172-
Status Visit(const DataType& t) {
174+
Status Visit(const DataType& t, ArrayBuilder* builder, Fn&& fn) {
173175
return Status::NotImplemented("visiting builders of type ", t);
174176
}
175-
176-
Status Visit() { return VisitTypeInline(*builder_->type(), this); }
177-
178-
ArrayBuilder* builder_;
179-
Fn fn_;
180177
};
181178

182-
template <typename Fn>
183-
Status VisitBuilder(ArrayBuilder* builder, Fn&& fn) {
184-
return VisitBuilderImpl<Fn>{builder, std::forward<Fn>(fn)}.Visit();
185-
}
186-
187179
template <typename Fn>
188180
Result<std::shared_ptr<Array>> ArrayFromBuilderVisitor(
189181
const std::shared_ptr<DataType>& type, int64_t initial_capacity,
@@ -195,8 +187,10 @@ Result<std::shared_ptr<Array>> ArrayFromBuilderVisitor(
195187
RETURN_NOT_OK(builder->Resize(initial_capacity));
196188
}
197189

190+
VisitBuilder<Fn> visitor;
198191
for (int64_t i = 0; i < visitor_repetitions; ++i) {
199-
RETURN_NOT_OK(VisitBuilder(builder.get(), std::forward<Fn>(fn)));
192+
RETURN_NOT_OK(
193+
VisitTypeInline(*builder->type(), &visitor, builder.get(), std::forward<Fn>(fn)));
200194
}
201195

202196
std::shared_ptr<Array> out;

‎cpp/src/arrow/visit_array_inline.h‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,11 +27,14 @@ namespace arrow {
2727
case TYPE_CLASS##Type::type_id: \
2828
return visitor->Visit( \
2929
internal::checked_cast<const typename TypeTraits<TYPE_CLASS##Type>::ArrayType&>( \
30-
array));
30+
array), \
31+
std::forward<ARGS>(args)...);
3132

3233
/// \brief Apply the visitors Visit() method specialized to the array type
3334
///
3435
/// \tparam VISITOR Visitor type that implements Visit() for all array types.
36+
/// \tparam ARGS Additional arguments, if any, will be passed to the Visit function after
37+
/// the `arr` argument
3538
/// \return Status
3639
///
3740
/// A visitor is a type that implements specialized logic for each Arrow type.
@@ -46,8 +49,8 @@ namespace arrow {
4649
/// ExampleVisitor visitor;
4750
/// VisitArrayInline(some_array, &visitor);
4851
/// ```
49-
template <typename VISITOR>
50-
inline Status VisitArrayInline(const Array& array, VISITOR* visitor) {
52+
template <typename VISITOR, typename... ARGS>
53+
inline Status VisitArrayInline(const Array& array, VISITOR* visitor, ARGS&&... args) {
5154
switch (array.type_id()) {
5255
ARROW_GENERATE_FOR_ALL_TYPES(ARRAY_VISIT_INLINE);
5356
default:

‎cpp/src/arrow/visit_scalar_inline.h‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -28,29 +28,32 @@
2828

2929
namespace arrow {
3030

31-
#define SCALAR_VISIT_INLINE(TYPE_CLASS) \
32-
case TYPE_CLASS##Type::type_id: \
33-
return visitor->Visit(internal::checked_cast<const TYPE_CLASS##Scalar&>(scalar));
31+
#define SCALAR_VISIT_INLINE(TYPE_CLASS) \
32+
case TYPE_CLASS##Type::type_id: \
33+
return visitor->Visit(internal::checked_cast<const TYPE_CLASS##Scalar&>(scalar), \
34+
std::forward<ARGS>(args)...);
3435

3536
/// \brief Apply the visitors Visit() method specialized to the scalar type
3637
///
3738
/// \tparam VISITOR Visitor type that implements Visit() for all scalar types.
39+
/// \tparam ARGS Additional arguments, if any, will be passed to the Visit function after
40+
/// the `scalar` argument
3841
/// \return Status
3942
///
4043
/// A visitor is a type that implements specialized logic for each Arrow type.
4144
/// Example usage:
4245
///
4346
/// ```
4447
/// class ExampleVisitor {
45-
/// arrow::Status Visit(arrow::Int32Scalar arr) { ... }
46-
/// arrow::Status Visit(arrow::Int64Scalar arr) { ... }
48+
/// arrow::Status Visit(arrow::Int32Scalar scalar) { ... }
49+
/// arrow::Status Visit(arrow::Int64Scalar scalar) { ... }
4750
/// ...
4851
/// }
4952
/// ExampleVisitor visitor;
5053
/// VisitScalarInline(some_scalar, &visitor);
5154
/// ```
52-
template <typename VISITOR>
53-
inline Status VisitScalarInline(const Scalar& scalar, VISITOR* visitor) {
55+
template <typename VISITOR, typename... ARGS>
56+
inline Status VisitScalarInline(const Scalar& scalar, VISITOR* visitor, ARGS&&... args) {
5457
switch (scalar.type->id()) {
5558
ARROW_GENERATE_FOR_ALL_TYPES(SCALAR_VISIT_INLINE);
5659
default:

‎cpp/src/arrow/visit_type_inline.h‎

Lines changed: 27 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,16 @@
2424

2525
namespace arrow {
2626

27-
#define TYPE_VISIT_INLINE(TYPE_CLASS) \
28-
case TYPE_CLASS##Type::type_id: \
29-
return visitor->Visit(internal::checked_cast<const TYPE_CLASS##Type&>(type));
27+
#define TYPE_VISIT_INLINE(TYPE_CLASS) \
28+
case TYPE_CLASS##Type::type_id: \
29+
return visitor->Visit(internal::checked_cast<const TYPE_CLASS##Type&>(type), \
30+
std::forward<ARGS>(args)...);
3031

3132
/// \brief Calls `visitor` with the corresponding concrete type class
3233
///
3334
/// \tparam VISITOR Visitor type that implements Visit() for all Arrow types.
35+
/// \tparam ARGS Additional arguments, if any, will be passed to the Visit function after
36+
/// the `type` argument
3437
/// \return Status
3538
///
3639
/// A visitor is a type that implements specialized logic for each Arrow type.
@@ -45,8 +48,8 @@ namespace arrow {
4548
/// ExampleVisitor visitor;
4649
/// VisitTypeInline(some_type, &visitor);
4750
/// ```
48-
template <typename VISITOR>
49-
inline Status VisitTypeInline(const DataType& type, VISITOR* visitor) {
51+
template <typename VISITOR, typename... ARGS>
52+
inline Status VisitTypeInline(const DataType& type, VISITOR* visitor, ARGS&&... args) {
5053
switch (type.id()) {
5154
ARROW_GENERATE_FOR_ALL_TYPES(TYPE_VISIT_INLINE);
5255
default:
@@ -57,12 +60,15 @@ inline Status VisitTypeInline(const DataType& type, VISITOR* visitor) {
5760

5861
#undef TYPE_VISIT_INLINE
5962

60-
#define TYPE_VISIT_INLINE(TYPE_CLASS) \
61-
case TYPE_CLASS##Type::type_id: \
62-
return std::forward<VISITOR>(visitor)( \
63-
internal::checked_cast<const TYPE_CLASS##Type&>(type));
63+
#define TYPE_VISIT_INLINE(TYPE_CLASS) \
64+
case TYPE_CLASS##Type::type_id: \
65+
return std::forward<VISITOR>(visitor)( \
66+
internal::checked_cast<const TYPE_CLASS##Type&>(type), \
67+
std::forward<ARGS>(args)...);
6468

6569
/// \brief Call `visitor` with the corresponding concrete type class
70+
/// \tparam ARGS Additional arguments, if any, will be passed to the Visit function after
71+
/// the `type` argument
6672
///
6773
/// Unlike VisitTypeInline which calls `visitor.Visit`, here `visitor`
6874
/// itself is called.
@@ -71,31 +77,33 @@ inline Status VisitTypeInline(const DataType& type, VISITOR* visitor) {
7177
///
7278
/// The intent is for this to be called on a generic lambda
7379
/// that may internally use `if constexpr` or similar constructs.
74-
template <typename VISITOR>
75-
inline auto VisitType(const DataType& type, VISITOR&& visitor)
76-
-> decltype(std::forward<VISITOR>(visitor)(type)) {
80+
template <typename VISITOR, typename... ARGS>
81+
inline auto VisitType(const DataType& type, VISITOR&& visitor, ARGS&&... args)
82+
-> decltype(std::forward<VISITOR>(visitor)(type, args...)) {
7783
switch (type.id()) {
7884
ARROW_GENERATE_FOR_ALL_TYPES(TYPE_VISIT_INLINE);
7985
default:
8086
break;
8187
}
82-
return std::forward<VISITOR>(visitor)(type);
88+
return std::forward<VISITOR>(visitor)(type, std::forward<ARGS>(args)...);
8389
}
8490

8591
#undef TYPE_VISIT_INLINE
8692

87-
#define TYPE_ID_VISIT_INLINE(TYPE_CLASS) \
88-
case TYPE_CLASS##Type::type_id: { \
89-
const TYPE_CLASS##Type* concrete_ptr = NULLPTR; \
90-
return visitor->Visit(concrete_ptr); \
93+
#define TYPE_ID_VISIT_INLINE(TYPE_CLASS) \
94+
case TYPE_CLASS##Type::type_id: { \
95+
const TYPE_CLASS##Type* concrete_ptr = NULLPTR; \
96+
return visitor->Visit(concrete_ptr, std::forward<ARGS>(args)...); \
9197
}
9298

9399
/// \brief Calls `visitor` with a nullptr of the corresponding concrete type class
94100
///
95101
/// \tparam VISITOR Visitor type that implements Visit() for all Arrow types.
102+
/// \tparam ARGS Additional arguments, if any, will be passed to the Visit function after
103+
/// the `type` argument
96104
/// \return Status
97-
template <typename VISITOR>
98-
inline Status VisitTypeIdInline(Type::type id, VISITOR* visitor) {
105+
template <typename VISITOR, typename... ARGS>
106+
inline Status VisitTypeIdInline(Type::type id, VISITOR* visitor, ARGS&&... args) {
99107
switch (id) {
100108
ARROW_GENERATE_FOR_ALL_TYPES(TYPE_ID_VISIT_INLINE);
101109
default:

‎cpp/src/parquet/column_writer.cc‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -74,19 +74,20 @@ namespace {
7474
// Visitor that exracts the value buffer from a FlatArray at a given offset.
7575
struct ValueBufferSlicer {
7676
template <typename T>
77-
::arrow::enable_if_base_binary<typename T::TypeClass, Status> Visit(const T& array) {
77+
::arrow::enable_if_base_binary<typename T::TypeClass, Status> Visit(
78+
const T& array, std::shared_ptr<Buffer>* buffer) {
7879
auto data = array.data();
79-
buffer_ =
80+
*buffer =
8081
SliceBuffer(data->buffers[1], data->offset * sizeof(typename T::offset_type),
8182
data->length * sizeof(typename T::offset_type));
8283
return Status::OK();
8384
}
8485

8586
template <typename T>
8687
::arrow::enable_if_fixed_size_binary<typename T::TypeClass, Status> Visit(
87-
const T& array) {
88+
const T& array, std::shared_ptr<Buffer>* buffer) {
8889
auto data = array.data();
89-
buffer_ = SliceBuffer(data->buffers[1], data->offset * array.byte_width(),
90+
*buffer = SliceBuffer(data->buffers[1], data->offset * array.byte_width(),
9091
data->length * array.byte_width());
9192
return Status::OK();
9293
}
@@ -95,29 +96,30 @@ struct ValueBufferSlicer {
9596
::arrow::enable_if_t<::arrow::has_c_type<typename T::TypeClass>::value &&
9697
!std::is_same<BooleanType, typename T::TypeClass>::value,
9798
Status>
98-
Visit(const T& array) {
99+
Visit(const T& array, std::shared_ptr<Buffer>* buffer) {
99100
auto data = array.data();
100-
buffer_ = SliceBuffer(
101+
*buffer = SliceBuffer(
101102
data->buffers[1],
102103
::arrow::TypeTraits<typename T::TypeClass>::bytes_required(data->offset),
103104
::arrow::TypeTraits<typename T::TypeClass>::bytes_required(data->length));
104105
return Status::OK();
105106
}
106107

107-
Status Visit(const ::arrow::BooleanArray& array) {
108+
Status Visit(const ::arrow::BooleanArray& array, std::shared_ptr<Buffer>* buffer) {
108109
auto data = array.data();
109110
if (bit_util::IsMultipleOf8(data->offset)) {
110-
buffer_ = SliceBuffer(data->buffers[1], bit_util::BytesForBits(data->offset),
111+
*buffer = SliceBuffer(data->buffers[1], bit_util::BytesForBits(data->offset),
111112
bit_util::BytesForBits(data->length));
112113
return Status::OK();
113114
}
114-
PARQUET_ASSIGN_OR_THROW(buffer_,
115+
PARQUET_ASSIGN_OR_THROW(*buffer,
115116
::arrow::internal::CopyBitmap(pool_, data->buffers[1]->data(),
116117
data->offset, data->length));
117118
return Status::OK();
118119
}
119120
#define NOT_IMPLEMENTED_VISIT(ArrowTypePrefix) \
120-
Status Visit(const ::arrow::ArrowTypePrefix##Array& array) { \
121+
Status Visit(const ::arrow::ArrowTypePrefix##Array& array, \
122+
std::shared_ptr<Buffer>* buffer) { \
121123
return Status::NotImplemented("Slicing not implemented for " #ArrowTypePrefix); \
122124
}
123125

@@ -133,7 +135,6 @@ struct ValueBufferSlicer {
133135
#undef NOT_IMPLEMENTED_VISIT
134136

135137
MemoryPool* pool_;
136-
std::shared_ptr<Buffer> buffer_;
137138
};
138139

139140
internal::LevelInfo ComputeLevelInfo(const ColumnDescriptor* descr) {
@@ -1316,10 +1317,9 @@ class TypedColumnWriterImpl : public ColumnWriterImpl, public TypedColumnWriter<
13161317
buffers[0] = bits_buffer_;
13171318
// Should be a leaf array.
13181319
DCHECK_GT(buffers.size(), 1);
1319-
ValueBufferSlicer slicer{memory_pool, /*buffer=*/nullptr};
1320+
ValueBufferSlicer slicer{memory_pool};
13201321
if (array->data()->offset > 0) {
1321-
RETURN_NOT_OK(::arrow::VisitArrayInline(*array, &slicer));
1322-
buffers[1] = slicer.buffer_;
1322+
RETURN_NOT_OK(::arrow::VisitArrayInline(*array, &slicer, &buffers[1]));
13231323
}
13241324
return ::arrow::MakeArray(std::make_shared<ArrayData>(
13251325
array->type(), array->length(), std::move(buffers), new_null_count));

0 commit comments

Comments
 (0)