Skip to content

Commit a521d7a

Browse files
committed
fix(python): handle non-row-major and zero-size input arrays
Two input-handling bugs in the bindings, both reachable from Zarr: 1. The bindings handed the input array's raw buffer to the conversion kernels via numpy::PyReadonlyArray::as_slice, whose guard accepts column-major arrays as well as row-major ones. The kernels then wrote those elements into a freshly allocated row-major output of the same shape, silently transposing the data. Arrays that were neither C- nor F-contiguous were rejected with "Input array must be contiguous". The Zarr transpose codec hands the next codec a transposed view -- column-major in 2-D, strided in higher dimensions -- so both cases occur in routine pipelines (zarr-developers/zarr-python#4237). The Python wrapper now normalizes the input with np.asarray(arr, order="C"): a no-op for row-major arrays, a copy for anything else, and 0-d-preserving (unlike np.ascontiguousarray). The binding keeps a strict backstop for direct callers of the private module, rejecting non-row-major input instead of misreading it. Normalizing on the Rust side instead (ndarray's as_standard_layout) was tried and rejected: ndarray's raw-view stride assertions panic on layouts numpy considers legal, and numpy's own normalization is authoritative. 2. numpy gives every zero-size array strides of 0, which ndarray's debug-build stride assertions reject as self-overlapping -- so casting something as plain as np.zeros((4, 0)) panicked in any maturin develop build (release builds compile the assertion out and were unaffected). Zero-size arrays now skip the conversion entirely; there is nothing to convert. Also corrects the cast_array_into output error message, which said "contiguous" where it meant row-major, and gives cast_array_into a Python wrapper (it was previously re-exported raw) so both entry points normalize identically. Assisted-by: ClaudeCode:claude-opus-4.8
1 parent 7f46619 commit a521d7a

2 files changed

Lines changed: 148 additions & 46 deletions

File tree

‎python/cast_value_rs/__init__.py‎

Lines changed: 55 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
import numpy as np
66

77
from cast_value_rs._cast_value_rs import cast_array as _cast_array
8-
from cast_value_rs._cast_value_rs import cast_array_into
8+
from cast_value_rs._cast_value_rs import cast_array_into as _cast_array_into
99

1010
if TYPE_CHECKING:
1111
from collections.abc import Iterable
@@ -64,6 +64,21 @@ def _resolve_dtype(target_dtype: DTypeName | np.dtype[np.generic] | type[np.gene
6464
return name
6565

6666

67+
def _as_row_major(arr: npt.NDArray[np.generic]) -> npt.NDArray[np.generic]:
68+
"""Return ``arr`` in row-major (C) layout with well-formed strides.
69+
70+
The extension module requires row-major input; ``np.asarray`` with
71+
``order="C"`` is a no-op for arrays that already are, and copies views of
72+
any other layout (e.g. the transposed views the Zarr transpose codec
73+
produces). Unlike ``np.ascontiguousarray``, it preserves 0-d shapes.
74+
75+
Zero-size arrays pass through unchanged (numpy flags them all
76+
C-contiguous, whatever their strides); the extension module skips the
77+
conversion for them.
78+
"""
79+
return np.asarray(arr, order="C")
80+
81+
6782
def cast_array(
6883
arr: npt.NDArray[np.generic],
6984
*,
@@ -98,12 +113,50 @@ def cast_array(
98113
A new numpy array with the target dtype.
99114
"""
100115
return _cast_array(
101-
arr,
116+
_as_row_major(arr),
102117
target_dtype=_resolve_dtype(target_dtype),
103118
rounding_mode=rounding_mode,
104119
out_of_range_mode=out_of_range_mode,
105120
scalar_map_entries=scalar_map_entries,
106121
)
107122

108123

124+
def cast_array_into(
125+
arr: npt.NDArray[np.generic],
126+
out: npt.NDArray[np.generic],
127+
*,
128+
rounding_mode: RoundingMode,
129+
out_of_range_mode: OutOfRangeMode | None = None,
130+
scalar_map_entries: (
131+
dict[float, float] | Iterable[tuple[float, float]] | None
132+
) = None,
133+
) -> None:
134+
"""Cast a numpy array to a new dtype, writing into a pre-allocated array.
135+
136+
Parameters
137+
----------
138+
arr
139+
Input numpy array.
140+
out
141+
Output numpy array. Determines the target dtype; must be row-major
142+
(C-contiguous), writeable, and the same shape as ``arr``.
143+
rounding_mode
144+
How to round values during conversion.
145+
out_of_range_mode
146+
How to handle values outside the target type's range.
147+
``None`` means out-of-range values raise an error.
148+
scalar_map_entries
149+
Mapping of special source values to target values.
150+
"""
151+
# The output array is the caller's buffer, so it cannot be copied; the
152+
# extension module rejects it if it is not row-major and writeable.
153+
_cast_array_into(
154+
_as_row_major(arr),
155+
out,
156+
rounding_mode=rounding_mode,
157+
out_of_range_mode=out_of_range_mode,
158+
scalar_map_entries=scalar_map_entries,
159+
)
160+
161+
109162
__all__ = ["cast_array", "cast_array_into"]

‎python/src/lib.rs‎

Lines changed: 93 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,39 @@ fn array_dtype_key(arr: &Bound<'_, PyUntypedArray>) -> PyResult<&'static str> {
217217
// Per-path conversion helpers (avoid duplicating the numpy I/O boilerplate)
218218
// ---------------------------------------------------------------------------
219219

220+
/// Borrow a readonly view of `arr`, requiring row-major (C) memory layout.
221+
///
222+
/// The conversion kernels read the input buffer as a flat slice and write
223+
/// their results into a freshly allocated row-major output, so the buffer
224+
/// they read must be in row-major element order.
225+
///
226+
/// `PyReadonlyArray::as_slice` alone is not a sufficient guard: it accepts
227+
/// column-major arrays too, and handing a column-major buffer to the kernels
228+
/// silently transposes the data. Normalizing the layout here (e.g. via
229+
/// `ndarray::as_standard_layout`) is also off the table: `ndarray`'s raw-view
230+
/// stride checks panic on layouts numpy considers legal, such as zero-size
231+
/// arrays with negative strides. So the binding strictly validates, and the
232+
/// Python wrapper normalizes arbitrary layouts with
233+
/// `np.asarray(arr, order="C")` -- numpy's own, authoritative implementation
234+
/// -- before calling in. For callers of the wrapper this rejection is
235+
/// unreachable; it is a backstop for direct users of the private extension
236+
/// module.
237+
fn readonly_row_major<'py, T: numpy::Element>(
238+
arr: &Bound<'py, PyUntypedArray>,
239+
) -> PyResult<PyReadonlyArrayDyn<'py, T>> {
240+
// NB: numpy flags 0-d and zero-size arrays as C-contiguous, so those
241+
// always pass.
242+
if !arr.is_c_contiguous() {
243+
return Err(pyo3::exceptions::PyValueError::new_err(
244+
"Input array must be row-major (C-contiguous)",
245+
));
246+
}
247+
Ok(arr.downcast::<PyArrayDyn<T>>()?.readonly())
248+
}
249+
250+
/// Panic message for `as_slice` on an array `readonly_row_major` accepted.
251+
const C_CONTIGUOUS_SLICE: &str = "a C-contiguous array's buffer is a valid slice";
252+
220253
/// Perform a float→int conversion on numpy arrays.
221254
fn do_float_to_int_alloc<'py, Src, Dst>(
222255
py: Python<'py>,
@@ -231,18 +264,19 @@ where
231264
Src: CastFloat + CastInto<Dst> + ExtractFromPy + numpy::Element + 'static,
232265
Dst: CastInt + ExtractFromPy + numpy::Element + 'static,
233266
{
234-
let input_arr: PyReadonlyArrayDyn<'_, Src> = arr.downcast::<PyArrayDyn<Src>>()?.readonly();
235-
let src_slice = input_arr
236-
.as_slice()
237-
.map_err(|_| pyo3::exceptions::PyValueError::new_err("Input array must be contiguous"))?;
267+
let input_arr: PyReadonlyArrayDyn<'_, Src> = readonly_row_major(arr)?;
268+
let src_slice = input_arr.as_slice().expect(C_CONTIGUOUS_SLICE);
238269
let config = FloatToIntConfig {
239270
map_entries: parse_map_entries::<Src, Dst>(map_entries_py, src_dtype, tgt_dtype)?,
240271
rounding,
241272
out_of_range: oor,
242273
};
243274
let shape: Vec<usize> = arr.shape().to_vec();
244275
let output = PyArrayDyn::<Dst>::zeros(py, &shape[..], false);
245-
{
276+
// Zero-size arrays skip the conversion: there is nothing to convert, and
277+
// numpy gives all zero-size arrays strides of 0, which `as_array_mut`'s
278+
// debug-build stride assertions reject as self-overlapping.
279+
if !shape.contains(&0) {
246280
// SAFETY: We just created `output` and hold the GIL, so no other
247281
// code can alias this array. The mutable reference is valid for
248282
// the duration of this block.
@@ -269,17 +303,18 @@ where
269303
Src: CastInt + CastInto<Dst> + ExtractFromPy + numpy::Element,
270304
Dst: CastInt + ExtractFromPy + numpy::Element,
271305
{
272-
let input_arr: PyReadonlyArrayDyn<'_, Src> = arr.downcast::<PyArrayDyn<Src>>()?.readonly();
273-
let src_slice = input_arr
274-
.as_slice()
275-
.map_err(|_| pyo3::exceptions::PyValueError::new_err("Input array must be contiguous"))?;
306+
let input_arr: PyReadonlyArrayDyn<'_, Src> = readonly_row_major(arr)?;
307+
let src_slice = input_arr.as_slice().expect(C_CONTIGUOUS_SLICE);
276308
let config = IntToIntConfig {
277309
map_entries: parse_map_entries::<Src, Dst>(map_entries_py, src_dtype, tgt_dtype)?,
278310
out_of_range: oor,
279311
};
280312
let shape: Vec<usize> = arr.shape().to_vec();
281313
let output = PyArrayDyn::<Dst>::zeros(py, &shape[..], false);
282-
{
314+
// Zero-size arrays skip the conversion: there is nothing to convert, and
315+
// numpy gives all zero-size arrays strides of 0, which `as_array_mut`'s
316+
// debug-build stride assertions reject as self-overlapping.
317+
if !shape.contains(&0) {
283318
// SAFETY: We just created `output` and hold the GIL, so no other
284319
// code can alias this array.
285320
let mut output_rw = unsafe { output.as_array_mut() };
@@ -306,18 +341,19 @@ where
306341
Src: CastFloat + CastInto<Dst> + ExtractFromPy + numpy::Element + 'static,
307342
Dst: CastFloat + ExtractFromPy + numpy::Element + 'static,
308343
{
309-
let input_arr: PyReadonlyArrayDyn<'_, Src> = arr.downcast::<PyArrayDyn<Src>>()?.readonly();
310-
let src_slice = input_arr
311-
.as_slice()
312-
.map_err(|_| pyo3::exceptions::PyValueError::new_err("Input array must be contiguous"))?;
344+
let input_arr: PyReadonlyArrayDyn<'_, Src> = readonly_row_major(arr)?;
345+
let src_slice = input_arr.as_slice().expect(C_CONTIGUOUS_SLICE);
313346
let config = FloatToFloatConfig {
314347
map_entries: parse_map_entries::<Src, Dst>(map_entries_py, src_dtype, tgt_dtype)?,
315348
rounding,
316349
out_of_range: oor,
317350
};
318351
let shape: Vec<usize> = arr.shape().to_vec();
319352
let output = PyArrayDyn::<Dst>::zeros(py, &shape[..], false);
320-
{
353+
// Zero-size arrays skip the conversion: there is nothing to convert, and
354+
// numpy gives all zero-size arrays strides of 0, which `as_array_mut`'s
355+
// debug-build stride assertions reject as self-overlapping.
356+
if !shape.contains(&0) {
321357
// SAFETY: We just created `output` and hold the GIL, so no other
322358
// code can alias this array.
323359
let mut output_rw = unsafe { output.as_array_mut() };
@@ -343,17 +379,18 @@ where
343379
Src: CastInt + CastInto<Dst> + ExtractFromPy + numpy::Element,
344380
Dst: CastFloat + ExtractFromPy + numpy::Element,
345381
{
346-
let input_arr: PyReadonlyArrayDyn<'_, Src> = arr.downcast::<PyArrayDyn<Src>>()?.readonly();
347-
let src_slice = input_arr
348-
.as_slice()
349-
.map_err(|_| pyo3::exceptions::PyValueError::new_err("Input array must be contiguous"))?;
382+
let input_arr: PyReadonlyArrayDyn<'_, Src> = readonly_row_major(arr)?;
383+
let src_slice = input_arr.as_slice().expect(C_CONTIGUOUS_SLICE);
350384
let config = IntToFloatConfig {
351385
map_entries: parse_map_entries::<Src, Dst>(map_entries_py, src_dtype, tgt_dtype)?,
352386
rounding,
353387
};
354388
let shape: Vec<usize> = arr.shape().to_vec();
355389
let output = PyArrayDyn::<Dst>::zeros(py, &shape[..], false);
356-
{
390+
// Zero-size arrays skip the conversion: there is nothing to convert, and
391+
// numpy gives all zero-size arrays strides of 0, which `as_array_mut`'s
392+
// debug-build stride assertions reject as self-overlapping.
393+
if !shape.contains(&0) {
357394
// SAFETY: We just created `output` and hold the GIL, so no other
358395
// code can alias this array.
359396
let mut output_rw = unsafe { output.as_array_mut() };
@@ -384,22 +421,25 @@ where
384421
Src: CastFloat + CastInto<Dst> + ExtractFromPy + numpy::Element + 'static,
385422
Dst: CastInt + ExtractFromPy + numpy::Element + 'static,
386423
{
387-
let input_arr: PyReadonlyArrayDyn<'_, Src> = arr.downcast::<PyArrayDyn<Src>>()?.readonly();
388-
let src_slice = input_arr
389-
.as_slice()
390-
.map_err(|_| pyo3::exceptions::PyValueError::new_err("Input array must be contiguous"))?;
424+
let input_arr: PyReadonlyArrayDyn<'_, Src> = readonly_row_major(arr)?;
425+
let src_slice = input_arr.as_slice().expect(C_CONTIGUOUS_SLICE);
391426
let config = FloatToIntConfig {
392427
map_entries: parse_map_entries::<Src, Dst>(map_entries_py, src_dtype, tgt_dtype)?,
393428
rounding,
394429
out_of_range: oor,
395430
};
396431
let out_arr: &Bound<'_, PyArrayDyn<Dst>> = out.downcast()?;
397-
{
432+
// Zero-size arrays skip the conversion: there is nothing to convert, and
433+
// numpy gives all zero-size arrays strides of 0, which `as_array_mut`'s
434+
// debug-build stride assertions reject as self-overlapping.
435+
if !out.shape().contains(&0) {
398436
// SAFETY: The GIL is held and `out_arr` is a distinct array from
399437
// `input_arr` (different dtypes). No aliasing occurs.
400438
let mut output_rw = unsafe { out_arr.as_array_mut() };
401439
let dst_slice = output_rw.as_slice_mut().ok_or_else(|| {
402-
pyo3::exceptions::PyValueError::new_err("Output array must be contiguous and writeable")
440+
pyo3::exceptions::PyValueError::new_err(
441+
"Output array must be row-major (C-contiguous) and writeable",
442+
)
403443
})?;
404444
zarr_cast_value::convert_slice_float_to_int(src_slice, dst_slice, &config)
405445
.map_err(cast_error_to_pyerr)?;
@@ -420,21 +460,24 @@ where
420460
Src: CastInt + CastInto<Dst> + ExtractFromPy + numpy::Element,
421461
Dst: CastInt + ExtractFromPy + numpy::Element,
422462
{
423-
let input_arr: PyReadonlyArrayDyn<'_, Src> = arr.downcast::<PyArrayDyn<Src>>()?.readonly();
424-
let src_slice = input_arr
425-
.as_slice()
426-
.map_err(|_| pyo3::exceptions::PyValueError::new_err("Input array must be contiguous"))?;
463+
let input_arr: PyReadonlyArrayDyn<'_, Src> = readonly_row_major(arr)?;
464+
let src_slice = input_arr.as_slice().expect(C_CONTIGUOUS_SLICE);
427465
let config = IntToIntConfig {
428466
map_entries: parse_map_entries::<Src, Dst>(map_entries_py, src_dtype, tgt_dtype)?,
429467
out_of_range: oor,
430468
};
431469
let out_arr: &Bound<'_, PyArrayDyn<Dst>> = out.downcast()?;
432-
{
470+
// Zero-size arrays skip the conversion: there is nothing to convert, and
471+
// numpy gives all zero-size arrays strides of 0, which `as_array_mut`'s
472+
// debug-build stride assertions reject as self-overlapping.
473+
if !out.shape().contains(&0) {
433474
// SAFETY: The GIL is held and `out_arr` is a distinct array from
434475
// `input_arr`. No aliasing occurs.
435476
let mut output_rw = unsafe { out_arr.as_array_mut() };
436477
let dst_slice = output_rw.as_slice_mut().ok_or_else(|| {
437-
pyo3::exceptions::PyValueError::new_err("Output array must be contiguous and writeable")
478+
pyo3::exceptions::PyValueError::new_err(
479+
"Output array must be row-major (C-contiguous) and writeable",
480+
)
438481
})?;
439482
zarr_cast_value::convert_slice_int_to_int(src_slice, dst_slice, &config)
440483
.map_err(cast_error_to_pyerr)?;
@@ -456,22 +499,25 @@ where
456499
Src: CastFloat + CastInto<Dst> + ExtractFromPy + numpy::Element + 'static,
457500
Dst: CastFloat + ExtractFromPy + numpy::Element + 'static,
458501
{
459-
let input_arr: PyReadonlyArrayDyn<'_, Src> = arr.downcast::<PyArrayDyn<Src>>()?.readonly();
460-
let src_slice = input_arr
461-
.as_slice()
462-
.map_err(|_| pyo3::exceptions::PyValueError::new_err("Input array must be contiguous"))?;
502+
let input_arr: PyReadonlyArrayDyn<'_, Src> = readonly_row_major(arr)?;
503+
let src_slice = input_arr.as_slice().expect(C_CONTIGUOUS_SLICE);
463504
let config = FloatToFloatConfig {
464505
map_entries: parse_map_entries::<Src, Dst>(map_entries_py, src_dtype, tgt_dtype)?,
465506
rounding,
466507
out_of_range: oor,
467508
};
468509
let out_arr: &Bound<'_, PyArrayDyn<Dst>> = out.downcast()?;
469-
{
510+
// Zero-size arrays skip the conversion: there is nothing to convert, and
511+
// numpy gives all zero-size arrays strides of 0, which `as_array_mut`'s
512+
// debug-build stride assertions reject as self-overlapping.
513+
if !out.shape().contains(&0) {
470514
// SAFETY: The GIL is held and `out_arr` is a distinct array from
471515
// `input_arr`. No aliasing occurs.
472516
let mut output_rw = unsafe { out_arr.as_array_mut() };
473517
let dst_slice = output_rw.as_slice_mut().ok_or_else(|| {
474-
pyo3::exceptions::PyValueError::new_err("Output array must be contiguous and writeable")
518+
pyo3::exceptions::PyValueError::new_err(
519+
"Output array must be row-major (C-contiguous) and writeable",
520+
)
475521
})?;
476522
zarr_cast_value::convert_slice_float_to_float(src_slice, dst_slice, &config)
477523
.map_err(cast_error_to_pyerr)?;
@@ -492,21 +538,24 @@ where
492538
Src: CastInt + CastInto<Dst> + ExtractFromPy + numpy::Element,
493539
Dst: CastFloat + ExtractFromPy + numpy::Element,
494540
{
495-
let input_arr: PyReadonlyArrayDyn<'_, Src> = arr.downcast::<PyArrayDyn<Src>>()?.readonly();
496-
let src_slice = input_arr
497-
.as_slice()
498-
.map_err(|_| pyo3::exceptions::PyValueError::new_err("Input array must be contiguous"))?;
541+
let input_arr: PyReadonlyArrayDyn<'_, Src> = readonly_row_major(arr)?;
542+
let src_slice = input_arr.as_slice().expect(C_CONTIGUOUS_SLICE);
499543
let config = IntToFloatConfig {
500544
map_entries: parse_map_entries::<Src, Dst>(map_entries_py, src_dtype, tgt_dtype)?,
501545
rounding,
502546
};
503547
let out_arr: &Bound<'_, PyArrayDyn<Dst>> = out.downcast()?;
504-
{
548+
// Zero-size arrays skip the conversion: there is nothing to convert, and
549+
// numpy gives all zero-size arrays strides of 0, which `as_array_mut`'s
550+
// debug-build stride assertions reject as self-overlapping.
551+
if !out.shape().contains(&0) {
505552
// SAFETY: The GIL is held and `out_arr` is a distinct array from
506553
// `input_arr`. No aliasing occurs.
507554
let mut output_rw = unsafe { out_arr.as_array_mut() };
508555
let dst_slice = output_rw.as_slice_mut().ok_or_else(|| {
509-
pyo3::exceptions::PyValueError::new_err("Output array must be contiguous and writeable")
556+
pyo3::exceptions::PyValueError::new_err(
557+
"Output array must be row-major (C-contiguous) and writeable",
558+
)
510559
})?;
511560
zarr_cast_value::convert_slice_int_to_float(src_slice, dst_slice, &config)
512561
.map_err(cast_error_to_pyerr)?;

0 commit comments

Comments
 (0)