Skip to content

Commit b3e261d

Browse files
authored
GH-46771: [Python][C++] Implement pa.arange function to generate array sequences (#46778)
### Rationale for this change When slicing arrays with non-trivial steps we were using `numpy.arange` to generate the indices for take. As numpy is an optional dependency, implementing it via Python caused a performance penalty. Creating a pyarrow function to build our own ranges that mimics Python range or numpy arange is useful for that uses case and might also be useful for other use cases. Currently we only generate `Array[Int64]` we could potentially generate more types. ### What changes are included in this PR? provide a `pa.arange` function that allows us to generate indices when slicing arrays. ### Are these changes tested? Yes new tests added. ### Are there any user-facing changes? No but a new pyarrow.arange function has been added. * GitHub Issue: #46771 Authored-by: Raúl Cumplido <[email protected]> Signed-off-by: Raúl Cumplido <[email protected]>
1 parent 5f0aeb5 commit b3e261d

8 files changed

Lines changed: 159 additions & 6 deletions

File tree

‎python/CMakeLists.txt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -368,7 +368,8 @@ set(PYARROW_CPP_SRCS
368368
${PYARROW_CPP_SOURCE_DIR}/python_test.cc
369369
${PYARROW_CPP_SOURCE_DIR}/python_to_arrow.cc
370370
${PYARROW_CPP_SOURCE_DIR}/pyarrow.cc
371-
${PYARROW_CPP_SOURCE_DIR}/udf.cc)
371+
${PYARROW_CPP_SOURCE_DIR}/udf.cc
372+
${PYARROW_CPP_SOURCE_DIR}/util.cc)
372373
set_source_files_properties(${PYARROW_CPP_SOURCE_DIR}/numpy_init.cc
373374
PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON)
374375

‎python/pyarrow/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,7 @@ def print_entry(label, value):
191191
SparseCOOTensor, SparseCSRMatrix, SparseCSCMatrix,
192192
SparseCSFTensor,
193193
infer_type, from_numpy_dtype,
194+
arange,
194195
NullArray,
195196
NumericArray, IntegerArray, FloatingPointArray,
196197
BooleanArray,

‎python/pyarrow/array.pxi‎

Lines changed: 36 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -572,22 +572,53 @@ def infer_type(values, mask=None, from_pandas=False):
572572
return pyarrow_wrap_data_type(out)
573573

574574

575+
def arange(int64_t start, int64_t stop, int64_t step=1, *, memory_pool=None):
576+
"""
577+
Create an array of evenly spaced values within a given interval.
578+
579+
This function is similar to Python's `range` function.
580+
The resulting array will contain values starting from `start` up to but not
581+
including `stop`, with a step size of `step`.
582+
583+
Parameters
584+
----------
585+
start : int
586+
The starting value for the sequence. The returned array will include this value.
587+
stop : int
588+
The stopping value for the sequence. The returned array will not include this value.
589+
step : int, default 1
590+
The spacing between values.
591+
memory_pool : MemoryPool, optional
592+
A memory pool to use for memory allocations.
593+
594+
Raises
595+
------
596+
ArrowInvalid
597+
If `step` is zero.
598+
599+
Returns
600+
-------
601+
arange : Array
602+
"""
603+
cdef CMemoryPool* pool = maybe_unbox_memory_pool(memory_pool)
604+
with nogil:
605+
c_array = GetResultValue(Arange(start, stop, step, pool))
606+
return pyarrow_wrap_array(c_array)
607+
608+
575609
def _normalize_slice(object arrow_obj, slice key):
576610
"""
577611
Slices with step not equal to 1 (or None) will produce a copy
578612
rather than a zero-copy view
579613
"""
580614
cdef:
581-
Py_ssize_t start, stop, step
615+
int64_t start, stop, step
582616
Py_ssize_t n = len(arrow_obj)
583617

584618
start, stop, step = key.indices(n)
585619

586620
if step != 1:
587-
indices = list(range(start, stop, step))
588-
if len(indices) == 0:
589-
return arrow_obj.slice(0, 0)
590-
return arrow_obj.take(indices)
621+
return arrow_obj.take(arange(start, stop, step))
591622
else:
592623
length = max(stop - start, 0)
593624
return arrow_obj.slice(start, length)

‎python/pyarrow/includes/libarrow_python.pxd‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,9 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
7373
object obj, object mask, const PyConversionOptions& options,
7474
CMemoryPool* pool)
7575

76+
CResult[shared_ptr[CArray]] Arange(int64_t start, int64_t stop,
77+
int64_t step, CMemoryPool* pool)
78+
7679
CResult[shared_ptr[CDataType]] NumPyDtypeToArrow(object dtype)
7780

7881
CStatus NdarrayToArrow(CMemoryPool* pool, object ao, object mo,

‎python/pyarrow/src/arrow/python/api.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,3 +26,4 @@
2626
#include "arrow/python/numpy_convert.h"
2727
#include "arrow/python/numpy_to_arrow.h"
2828
#include "arrow/python/python_to_arrow.h"
29+
#include "arrow/python/util.h"
Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#include "arrow/python/util.h"
19+
20+
#include "arrow/array.h"
21+
#include "arrow/python/common.h"
22+
23+
namespace arrow ::py {
24+
25+
Result<std::shared_ptr<Array>> Arange(int64_t start, int64_t stop, int64_t step,
26+
MemoryPool* pool) {
27+
int64_t size;
28+
if (step == 0) {
29+
return Status::Invalid("Step must not be zero");
30+
}
31+
if (step > 0 && stop > start) {
32+
// Ceiling division for positive step
33+
size = (stop - start + step - 1) / step;
34+
} else if (step < 0 && stop < start) {
35+
// Ceiling division for negative step
36+
size = (start - stop - step - 1) / (-step);
37+
} else {
38+
return MakeEmptyArray(int64());
39+
}
40+
std::shared_ptr<Buffer> data_buffer;
41+
ARROW_ASSIGN_OR_RAISE(data_buffer, AllocateBuffer(size * sizeof(int64_t), pool));
42+
auto values = reinterpret_cast<int64_t*>(data_buffer->mutable_data());
43+
for (int64_t i = 0; i < size; ++i) {
44+
values[i] = start + i * step;
45+
}
46+
auto data = ArrayData::Make(int64(), size, {nullptr, data_buffer}, 0);
47+
return MakeArray(data);
48+
}
49+
50+
} // namespace arrow::py
Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#pragma once
19+
20+
#include "arrow/python/common.h"
21+
#include "arrow/python/visibility.h"
22+
23+
namespace arrow::py {
24+
25+
/// \brief Create an array of evenly spaced values within a given interval.
26+
/// This function is similar to Python's `range` function.
27+
/// The resulting array will contain values starting from `start` up to but not
28+
/// including `stop`, with a step size of `step`. If `step` is zero, the function
29+
/// will return an error.
30+
/// The resulting array will have a data type of `int64`.
31+
/// \param[in] start initial value of the sequence.
32+
/// \param[in] stop final value of the sequence (exclusive).
33+
/// \param[in] step step size between consecutive values.
34+
/// \param[in] pool Memory pool for any memory allocations.
35+
/// \return Result Array
36+
ARROW_PYTHON_EXPORT
37+
Result<std::shared_ptr<Array>> Arange(int64_t start, int64_t stop, int64_t step,
38+
MemoryPool* pool);
39+
40+
} // namespace arrow::py

‎python/pyarrow/tests/test_array.py‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -536,6 +536,32 @@ def test_array_slice_negative_step():
536536
assert result.equals(expected)
537537

538538

539+
def test_arange():
540+
cases = [
541+
(5, 103), # Default step
542+
(-2, 128, 3),
543+
(4, 103, 5),
544+
(10, -7, -1),
545+
(100, -20, -3),
546+
(0, 0), # Empty array
547+
(2, 10, -1), # Empty array
548+
(10, 3, 1), # Empty array
549+
]
550+
for case in cases:
551+
result = pa.arange(*case)
552+
result.validate(full=True)
553+
assert result.equals(pa.array(list(range(*case)), type=pa.int64()))
554+
555+
# Validate memory_pool keyword argument
556+
result = pa.arange(-1, 101, memory_pool=pa.default_memory_pool())
557+
result.validate(full=True)
558+
assert result.equals(pa.array(list(range(-1, 101)), type=pa.int64()))
559+
560+
# Special case for invalid step (arange does not accept step of 0)
561+
with pytest.raises(pa.ArrowInvalid):
562+
pa.arange(0, 10, 0)
563+
564+
539565
def test_array_diff():
540566
# ARROW-6252
541567
arr1 = pa.array(['foo'], type=pa.utf8())

0 commit comments

Comments
 (0)