Skip to content

Commit 4303be4

Browse files
committed
chore: use Vec instead of OffsetBuilder
1 parent a0e9887 commit 4303be4

8 files changed

Lines changed: 54 additions & 52 deletions

File tree

‎datafusion/functions-nested/src/array_add.rs‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,9 @@
1919
2020
use crate::utils::{coerce_array_math_arg_types, make_scalar_function};
2121
use arrow::array::{
22-
Array, ArrayRef, Float64Array, GenericListArray, NullBufferBuilder,
23-
OffsetBufferBuilder, OffsetSizeTrait,
22+
Array, ArrayRef, Float64Array, GenericListArray, NullBufferBuilder, OffsetSizeTrait,
2423
};
25-
use arrow::buffer::NullBuffer;
24+
use arrow::buffer::{NullBuffer, OffsetBuffer};
2625
use arrow::datatypes::{
2726
DataType,
2827
DataType::{LargeList, List},
@@ -147,12 +146,13 @@ fn general_array_add<O: OffsetSizeTrait>(
147146

148147
let mut out_values: Vec<f64> = Vec::with_capacity(lhs_values.len());
149148
let mut out_inner_nulls = NullBufferBuilder::new(lhs_values.len());
150-
let mut out_offsets = OffsetBufferBuilder::<O>::new(lhs.len());
149+
let mut out_offsets = Vec::<O>::with_capacity(lhs.len() + 1);
150+
out_offsets.push(O::zero());
151151

152152
for row in 0..lhs.len() {
153153
// Whole-row NULL on either side -> NULL output row, no elements.
154154
if row_nulls.as_ref().is_some_and(|nb| nb.is_null(row)) {
155-
out_offsets.push_length(0);
155+
out_offsets.push(out_offsets[row]);
156156
continue;
157157
}
158158

@@ -185,7 +185,7 @@ fn general_array_add<O: OffsetSizeTrait>(
185185
None => out_inner_nulls.append_n_non_nulls(len1),
186186
}
187187

188-
out_offsets.push_length(len1);
188+
out_offsets.push(out_offsets[row] + O::usize_as(len1));
189189
}
190190

191191
let values_array = Arc::new(Float64Array::new(
@@ -196,7 +196,7 @@ fn general_array_add<O: OffsetSizeTrait>(
196196

197197
Ok(Arc::new(GenericListArray::<O>::try_new(
198198
field,
199-
out_offsets.finish(),
199+
OffsetBuffer::new(out_offsets.into()),
200200
values_array,
201201
row_nulls,
202202
)?))

‎datafusion/functions-nested/src/array_filter.rs‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
use arrow::{
2121
array::{
2222
Array, ArrayRef, AsArray, BooleanArray, LargeListArray, ListArray,
23-
OffsetBufferBuilder, OffsetSizeTrait, new_empty_array,
23+
OffsetSizeTrait, new_empty_array,
2424
},
2525
buffer::{OffsetBuffer, ScalarBuffer},
2626
compute::{filter as arrow_filter, take_arrays},
@@ -252,7 +252,8 @@ fn filter_list_values<O: OffsetSizeTrait>(
252252
offsets: &OffsetBuffer<O>,
253253
) -> Result<(ArrayRef, OffsetBuffer<O>)> {
254254
let num_sublists = offsets.len().saturating_sub(1);
255-
let mut builder = OffsetBufferBuilder::<O>::new(num_sublists);
255+
let mut new_offsets = Vec::<O>::with_capacity(num_sublists + 1);
256+
new_offsets.push(O::zero());
256257

257258
let has_nulls = predicate.null_count() > 0;
258259
for i in 0..num_sublists {
@@ -268,10 +269,10 @@ fn filter_list_values<O: OffsetSizeTrait>(
268269
.slice(start, end - start)
269270
.count_set_bits()
270271
};
271-
builder.push_length(count);
272+
new_offsets.push(new_offsets[i] + O::usize_as(count));
272273
}
273274

274-
let new_offsets = builder.finish();
275+
let new_offsets = OffsetBuffer::new(new_offsets.into());
275276

276277
if new_offsets.last() == offsets.last() {
277278
return Ok((Arc::clone(values), offsets.clone()));

‎datafusion/functions-nested/src/array_normalize.rs‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,9 @@
1919
2020
use crate::utils::make_scalar_function;
2121
use arrow::array::{
22-
Array, ArrayRef, Float64Array, GenericListArray, NullBufferBuilder,
23-
OffsetBufferBuilder, OffsetSizeTrait,
22+
Array, ArrayRef, Float64Array, GenericListArray, NullBufferBuilder, OffsetSizeTrait,
2423
};
24+
use arrow::buffer::OffsetBuffer;
2525
use arrow::datatypes::{
2626
DataType,
2727
DataType::{FixedSizeList, LargeList, List, Null},
@@ -144,13 +144,14 @@ fn general_array_normalize<O: OffsetSizeTrait>(arrays: &[ArrayRef]) -> Result<Ar
144144
let offsets = list_array.value_offsets();
145145

146146
let mut new_values: Vec<f64> = Vec::with_capacity(values.len());
147-
let mut new_offsets = OffsetBufferBuilder::<O>::new(list_array.len());
147+
let mut new_offsets = Vec::<O>::with_capacity(list_array.len() + 1);
148+
new_offsets.push(O::zero());
148149
let mut nulls = NullBufferBuilder::new(list_array.len());
149150

150151
for row in 0..list_array.len() {
151152
if list_array.is_null(row) {
152153
nulls.append_null();
153-
new_offsets.push_length(0);
154+
new_offsets.push(new_offsets[row]);
154155
continue;
155156
}
156157

@@ -161,7 +162,7 @@ fn general_array_normalize<O: OffsetSizeTrait>(arrays: &[ArrayRef]) -> Result<Ar
161162
let slice = values.slice(start, len);
162163
if slice.null_count() != 0 {
163164
nulls.append_null();
164-
new_offsets.push_length(0);
165+
new_offsets.push(new_offsets[row]);
165166
continue;
166167
}
167168

@@ -170,7 +171,7 @@ fn general_array_normalize<O: OffsetSizeTrait>(arrays: &[ArrayRef]) -> Result<Ar
170171
// Empty array: return empty array (no normalization needed, no division by zero risk)
171172
if len == 0 {
172173
nulls.append_non_null();
173-
new_offsets.push_length(0);
174+
new_offsets.push(new_offsets[row]);
174175
continue;
175176
}
176177

@@ -183,7 +184,7 @@ fn general_array_normalize<O: OffsetSizeTrait>(arrays: &[ArrayRef]) -> Result<Ar
183184
// Zero magnitude: undefined normalization. Emit NULL row.
184185
if sq_sum == 0.0 {
185186
nulls.append_null();
186-
new_offsets.push_length(0);
187+
new_offsets.push(new_offsets[row]);
187188
continue;
188189
}
189190

@@ -192,15 +193,15 @@ fn general_array_normalize<O: OffsetSizeTrait>(arrays: &[ArrayRef]) -> Result<Ar
192193
new_values.push(vals[i] / mag);
193194
}
194195
nulls.append_non_null();
195-
new_offsets.push_length(len);
196+
new_offsets.push(new_offsets[row] + O::usize_as(len));
196197
}
197198

198199
let values_array = Arc::new(Float64Array::from(new_values));
199200
let field = Arc::new(Field::new_list_field(DataType::Float64, true));
200201

201202
Ok(Arc::new(GenericListArray::<O>::try_new(
202203
field,
203-
new_offsets.finish(),
204+
OffsetBuffer::new(new_offsets.into()),
204205
values_array,
205206
nulls.finish(),
206207
)?))

‎datafusion/functions-nested/src/array_scale.rs‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,8 @@
1818
//! [`ScalarUDFImpl`] definitions for array_scale function.
1919
2020
use crate::utils::make_scalar_function;
21-
use arrow::array::{
22-
Array, ArrayRef, Float64Array, GenericListArray, OffsetBufferBuilder, OffsetSizeTrait,
23-
};
24-
use arrow::buffer::NullBuffer;
21+
use arrow::array::{Array, ArrayRef, Float64Array, GenericListArray, OffsetSizeTrait};
22+
use arrow::buffer::{NullBuffer, OffsetBuffer};
2523
use arrow::datatypes::{
2624
DataType,
2725
DataType::{FixedSizeList, LargeList, List, Null},
@@ -172,11 +170,12 @@ fn general_array_scale<O: OffsetSizeTrait>(
172170
let row_nulls = NullBuffer::union(list_array.nulls(), scalar_array.nulls());
173171

174172
let mut value_builder = Float64Array::builder(values.len());
175-
let mut new_offsets = OffsetBufferBuilder::<O>::new(list_array.len());
173+
let mut new_offsets = Vec::<O>::with_capacity(list_array.len() + 1);
174+
new_offsets.push(O::zero());
176175

177176
for row in 0..list_array.len() {
178177
if row_nulls.as_ref().is_some_and(|nb| nb.is_null(row)) {
179-
new_offsets.push_length(0);
178+
new_offsets.push(new_offsets[row]);
180179
continue;
181180
}
182181

@@ -196,7 +195,7 @@ fn general_array_scale<O: OffsetSizeTrait>(
196195
}
197196
}
198197

199-
new_offsets.push_length(len);
198+
new_offsets.push(new_offsets[row] + O::usize_as(len));
200199
}
201200

202201
let values_array = Arc::new(value_builder.finish());
@@ -213,7 +212,7 @@ fn general_array_scale<O: OffsetSizeTrait>(
213212

214213
Ok(Arc::new(GenericListArray::<O>::try_new(
215214
field,
216-
new_offsets.finish(),
215+
OffsetBuffer::new(new_offsets.into()),
217216
values_array,
218217
row_nulls,
219218
)?))

‎datafusion/functions-nested/src/remove.rs‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,7 @@
2020
use crate::utils;
2121
use arrow::array::{
2222
Array, ArrayRef, Capacities, GenericListArray, MutableArrayData, NullBufferBuilder,
23-
OffsetBufferBuilder, OffsetSizeTrait, Scalar, cast::AsArray, make_array,
24-
new_null_array,
23+
OffsetSizeTrait, Scalar, cast::AsArray, make_array, new_null_array,
2524
};
2625
use arrow::buffer::OffsetBuffer;
2726
use arrow::datatypes::{DataType, FieldRef};
@@ -584,7 +583,8 @@ fn general_remove_with_scalar<OffsetSize: OffsetSizeTrait>(
584583
let values_range_len = last_offset - first_offset;
585584
let values_slice = list_array.values().slice(first_offset, values_range_len);
586585
let original_data = values_slice.to_data();
587-
let mut offsets = OffsetBufferBuilder::<OffsetSize>::new(list_array.len());
586+
let mut offsets = Vec::<OffsetSize>::with_capacity(list_array.len() + 1);
587+
offsets.push(OffsetSize::zero());
588588

589589
let mut mutable = MutableArrayData::with_capacities(
590590
vec![&original_data],
@@ -598,7 +598,7 @@ fn general_remove_with_scalar<OffsetSize: OffsetSizeTrait>(
598598

599599
for (row_index, offset_window) in list_offsets.windows(2).enumerate() {
600600
if nulls.as_ref().is_some_and(|nulls| nulls.is_null(row_index)) {
601-
offsets.push_length(0);
601+
offsets.push(offsets[row_index]);
602602
continue;
603603
}
604604

@@ -611,7 +611,7 @@ fn general_remove_with_scalar<OffsetSize: OffsetSizeTrait>(
611611

612612
if num_to_remove == 0 {
613613
mutable.extend(0, start, end);
614-
offsets.push_length(row_len);
614+
offsets.push(offsets[row_index] + OffsetSize::usize_as(row_len));
615615
continue;
616616
}
617617

@@ -641,13 +641,13 @@ fn general_remove_with_scalar<OffsetSize: OffsetSizeTrait>(
641641
copied += end - prev_end;
642642
}
643643

644-
offsets.push_length(copied);
644+
offsets.push(offsets[row_index] + OffsetSize::usize_as(copied));
645645
}
646646

647647
let new_values = make_array(mutable.freeze());
648648
Ok(Arc::new(GenericListArray::<OffsetSize>::try_new(
649649
Arc::clone(list_field),
650-
offsets.finish(),
650+
OffsetBuffer::new(offsets.into()),
651651
new_values,
652652
nulls,
653653
)?))

‎datafusion/functions-nested/src/replace.rs‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
2020
use arrow::array::{
2121
Array, ArrayRef, AsArray, Capacities, GenericListArray, MutableArrayData,
22-
NullBufferBuilder, OffsetBufferBuilder, OffsetSizeTrait, Scalar, new_null_array,
22+
NullBufferBuilder, OffsetSizeTrait, Scalar, new_null_array,
2323
};
2424
use arrow::buffer::OffsetBuffer;
2525
use arrow::datatypes::{DataType, Field};
@@ -538,7 +538,8 @@ fn general_replace_with_scalar<O: OffsetSizeTrait>(
538538
capacity,
539539
);
540540

541-
let mut offsets = OffsetBufferBuilder::<O>::new(list_array.len());
541+
let mut offsets = Vec::<O>::with_capacity(list_array.len() + 1);
542+
offsets.push(O::zero());
542543

543544
// Single bulk comparison over the visible values only.
544545
let match_bitmap = arrow_ord::cmp::not_distinct(&visible_values, needle)?;
@@ -551,7 +552,7 @@ fn general_replace_with_scalar<O: OffsetSizeTrait>(
551552
let row_len = end - start;
552553

553554
if list_array.is_null(row_index) {
554-
offsets.push_length(0);
555+
offsets.push(offsets[row_index]);
555556
continue;
556557
}
557558

@@ -563,7 +564,7 @@ fn general_replace_with_scalar<O: OffsetSizeTrait>(
563564
.peekable();
564565
if match_positions.peek().is_none() {
565566
mutable.extend(0, start, end);
566-
offsets.push_length(row_len);
567+
offsets.push(offsets[row_index] + O::usize_as(row_len));
567568
continue;
568569
}
569570

@@ -586,14 +587,14 @@ fn general_replace_with_scalar<O: OffsetSizeTrait>(
586587
mutable.extend(0, start + prev_end, end);
587588
}
588589

589-
offsets.push_length(row_len);
590+
offsets.push(offsets[row_index] + O::usize_as(row_len));
590591
}
591592

592593
let data = mutable.freeze();
593594

594595
Ok(Arc::new(GenericListArray::<O>::try_new(
595596
Arc::new(Field::new_list_field(list_array.value_type(), true)),
596-
offsets.finish(),
597+
OffsetBuffer::new(offsets.into()),
597598
arrow::array::make_array(data),
598599
list_array.nulls().cloned(),
599600
)?))

‎datafusion/functions-nested/src/utils.rs‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ use arrow::datatypes::{DataType, Field, Fields};
2323

2424
use arrow::array::{
2525
Array, ArrayRef, BooleanArray, Float64Array, GenericListArray, NullBufferBuilder,
26-
OffsetBufferBuilder, OffsetSizeTrait, Scalar,
26+
OffsetSizeTrait, Scalar,
2727
};
2828
use arrow::buffer::{NullBuffer, OffsetBuffer};
2929
use datafusion_common::cast::{
@@ -361,11 +361,12 @@ where
361361

362362
let mut out_values: Vec<f64> = Vec::with_capacity(lhs_values.len());
363363
let mut out_inner_nulls = NullBufferBuilder::new(lhs_values.len());
364-
let mut out_offsets = OffsetBufferBuilder::<O>::new(lhs.len());
364+
let mut out_offsets = Vec::<O>::with_capacity(lhs.len() + 1);
365+
out_offsets.push(O::zero());
365366

366367
for row in 0..lhs.len() {
367368
if row_nulls.as_ref().is_some_and(|nb| nb.is_null(row)) {
368-
out_offsets.push_length(0);
369+
out_offsets.push(out_offsets[row]);
369370
continue;
370371
}
371372

@@ -395,7 +396,7 @@ where
395396
None => out_inner_nulls.append_n_non_nulls(len1),
396397
}
397398

398-
out_offsets.push_length(len1);
399+
out_offsets.push(out_offsets[row] + O::usize_as(len1));
399400
}
400401

401402
let values_array = Arc::new(Float64Array::new(
@@ -406,7 +407,7 @@ where
406407

407408
Ok(Arc::new(GenericListArray::<O>::try_new(
408409
field,
409-
out_offsets.finish(),
410+
OffsetBuffer::new(out_offsets.into()),
410411
values_array,
411412
row_nulls,
412413
)?))

‎datafusion/functions/src/encoding/inner.rs‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ use arrow::{
2424
},
2525
datatypes::DataType,
2626
};
27-
use arrow_buffer::{Buffer, OffsetBufferBuilder};
27+
use arrow_buffer::{Buffer, OffsetBuffer};
2828
use base64::{
2929
Engine as _,
3030
engine::{DecodePaddingMode, GeneralPurpose, GeneralPurposeConfig},
@@ -470,22 +470,21 @@ where
470470
OutputOffset: OffsetSizeTrait,
471471
{
472472
let mut values = vec![0; conservative_upper_bound_size];
473-
let mut offsets = OffsetBufferBuilder::new(input.len());
473+
let mut offsets = Vec::<OutputOffset>::with_capacity(input.len() + 1);
474+
offsets.push(OutputOffset::zero());
474475
let mut total_bytes_decoded = 0;
475476
for v in input.iter() {
476477
if let Some(v) = v {
477478
let cursor = &mut values[total_bytes_decoded..];
478479
let decoded = decode(v, cursor)?;
479480
total_bytes_decoded += decoded;
480-
offsets.push_length(decoded);
481-
} else {
482-
offsets.push_length(0);
483481
}
482+
offsets.push(OutputOffset::usize_as(total_bytes_decoded));
484483
}
485484
// We reserved an upper bound size for the values buffer, but we only use the actual size
486485
values.truncate(total_bytes_decoded);
487486
let binary_array = GenericBinaryArray::<OutputOffset>::try_new(
488-
offsets.finish(),
487+
OffsetBuffer::new(offsets.into()),
489488
Buffer::from_vec(values),
490489
input.nulls().cloned(),
491490
)?;

0 commit comments

Comments
 (0)