Skip to content

Commit 4f2b219

Browse files
authored
BUG: add missing c_distributions.pxd, enables cython use of random C-API (gh-15463)
xref gh-14778 As pointed out in the comment by @jamesthomasgriffin, we did not include a pxd file to expose the distribution functions documented in the random c-api. This PR adds a c_distributions.pxd file that exposes them. Squashed commits: * BUG: add missing c_distributions.pxd to enable cython use of random C-API * ENH, TST: add npyrandom library like npymath, test cython use of it * BUG: actually prefix f-string with f * MAINT: fixes from review, add _bit_generato_bit_generator.pxd * STY: fixes from review * BLD: don't use nprandom library for mtrand legacy build * TST: WindowsPath cannot be used in subprocess's list2cmdline * MAINT, API: move _bit_generator to bit_generator * DOC: add release note about moving bit_generator * DOC, MAINT: fixes from review * MAINT: redo dtype determination from review
1 parent f77c5e8 commit 4f2b219

14 files changed

Lines changed: 268 additions & 151 deletions

File tree

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
``numpy.random._bit_generator`` moved to ``numpy.random.bit_generator``
2+
-----------------------------------------------------------------------
3+
4+
In order to expose `numpy.random.BitGenerator` and `numpy.random.SeedSequence`
5+
to cython, the ``_bitgenerator`` module is now public as
6+
`numpy.random.bit_generator`
7+
8+
Cython access to the random distributions is provided via a `pxd` file
9+
----------------------------------------------------------------------
10+
11+
``c_distributions.pxd`` provides access to the c functions behind many of the
12+
random distributions from Cython, making it convenient to use and extend them.

‎doc/source/reference/random/extending.rst‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,9 +47,18 @@ struct.
4747
.. literalinclude:: ../../../../numpy/random/_examples/cython/extending_distributions.pyx
4848
:language: cython
4949
:start-after: example 2
50+
:end-before: example 3
5051

51-
See :ref:`extending_cython_example` for a complete working example including a
52-
minimal setup and cython files.
52+
Cython can be used to directly access the functions in
53+
``numpy/random/c_distributions.pxd``. This requires linking with the
54+
``npyrandom`` library located in ``numpy/random/lib``.
55+
56+
.. literalinclude:: ../../../../numpy/random/_examples/cython/extending_distributions.pyx
57+
:language: cython
58+
:start-after: example 3
59+
60+
See :ref:`extending_cython_example` for the complete listings of these examples
61+
and a minimal ``setup.py`` to build the c-extension modules.
5362

5463
CFFI
5564
====

‎numpy/random/__init__.pxd‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,4 +11,4 @@ cdef extern from "numpy/random/bitgen.h":
1111

1212
ctypedef bitgen bitgen_t
1313

14-
from numpy.random._bit_generator cimport BitGenerator, SeedSequence
14+
from numpy.random.bit_generator cimport BitGenerator, SeedSequence

‎numpy/random/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -181,7 +181,7 @@
181181
from . import _bounded_integers
182182

183183
from ._generator import Generator, default_rng
184-
from ._bit_generator import SeedSequence, BitGenerator
184+
from .bit_generator import SeedSequence, BitGenerator
185185
from ._mt19937 import MT19937
186186
from ._pcg64 import PCG64
187187
from ._philox import Philox

‎numpy/random/_examples/cython/extending_distributions.pyx‎

Lines changed: 44 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ from cpython.pycapsule cimport PyCapsule_IsValid, PyCapsule_GetPointer
1010
from libc.stdint cimport uint16_t, uint64_t
1111
from numpy.random cimport bitgen_t
1212
from numpy.random import PCG64
13+
from numpy.random.c_distributions cimport (
14+
random_standard_uniform_fill, random_standard_uniform_fill_f)
1315

1416

1517
@cython.boundscheck(False)
@@ -40,7 +42,7 @@ def uniforms(Py_ssize_t n):
4042
randoms = np.asarray(random_values)
4143

4244
return randoms
43-
45+
4446
# cython example 2
4547
@cython.boundscheck(False)
4648
@cython.wraparound(False)
@@ -72,3 +74,44 @@ def uint10_uniforms(Py_ssize_t n):
7274
randoms = np.asarray(random_values)
7375
return randoms
7476

77+
# cython example 3
78+
def uniforms_ex(bit_generator, Py_ssize_t n, dtype=np.float64):
79+
"""
80+
Create an array of `n` uniformly distributed doubles via a "fill" function.
81+
82+
A 'real' distribution would want to process the values into
83+
some non-uniform distribution
84+
85+
Parameters
86+
----------
87+
bit_generator: BitGenerator instance
88+
n: int
89+
Output vector length
90+
dtype: {str, dtype}, optional
91+
Desired dtype, either 'd' (or 'float64') or 'f' (or 'float32'). The
92+
default dtype value is 'd'
93+
"""
94+
cdef Py_ssize_t i
95+
cdef bitgen_t *rng
96+
cdef const char *capsule_name = "BitGenerator"
97+
cdef np.ndarray randoms
98+
99+
capsule = bit_generator.capsule
100+
# Optional check that the capsule if from a BitGenerator
101+
if not PyCapsule_IsValid(capsule, capsule_name):
102+
raise ValueError("Invalid pointer to anon_func_state")
103+
# Cast the pointer
104+
rng = <bitgen_t *> PyCapsule_GetPointer(capsule, capsule_name)
105+
106+
_dtype = np.dtype(dtype)
107+
randoms = np.empty(n, dtype=_dtype)
108+
if _dtype == np.float32:
109+
with bit_generator.lock:
110+
random_standard_uniform_fill_f(rng, n, <float*>np.PyArray_DATA(randoms))
111+
elif _dtype == np.float64:
112+
with bit_generator.lock:
113+
random_standard_uniform_fill(rng, n, <double*>np.PyArray_DATA(randoms))
114+
else:
115+
raise TypeError('Unsupported dtype %r for random' % _dtype)
116+
return randoms
117+

‎numpy/random/_examples/cython/setup.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,11 @@
1212
from os.path import join, dirname
1313

1414
path = dirname(__file__)
15+
src_dir = join(dirname(path), '..', 'src')
1516
defs = [('NPY_NO_DEPRECATED_API', 0)]
17+
inc_path = np.get_include()
18+
# not so nice. We need the random/lib library from numpy
19+
lib_path = join(np.get_include(), '..', '..', 'random', 'lib')
1620

1721
extending = Extension("extending",
1822
sources=[join(path, 'extending.pyx')],
@@ -24,7 +28,9 @@
2428
)
2529
distributions = Extension("extending_distributions",
2630
sources=[join(path, 'extending_distributions.pyx')],
27-
include_dirs=[np.get_include()],
31+
include_dirs=[inc_path],
32+
library_dirs=[lib_path],
33+
libraries=['npyrandom'],
2834
define_macros=defs,
2935
)
3036

‎numpy/random/_generator.pyx‎

Lines changed: 1 addition & 109 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import numpy as np
1111
cimport numpy as np
1212
from numpy.core.multiarray import normalize_axis_index
1313

14+
from .c_distributions cimport *
1415
from libc cimport string
1516
from libc.stdint cimport (uint8_t, uint16_t, uint32_t, uint64_t,
1617
int32_t, int64_t, INT64_MAX, SIZE_MAX)
@@ -26,117 +27,8 @@ from ._common cimport (POISSON_LAM_MAX, CONS_POSITIVE, CONS_NONE,
2627
check_array_constraint, check_constraint, disc, discrete_broadcast_iii,
2728
)
2829

29-
30-
cdef extern from "numpy/random/distributions.h":
31-
32-
struct s_binomial_t:
33-
int has_binomial
34-
double psave
35-
int64_t nsave
36-
double r
37-
double q
38-
double fm
39-
int64_t m
40-
double p1
41-
double xm
42-
double xl
43-
double xr
44-
double c
45-
double laml
46-
double lamr
47-
double p2
48-
double p3
49-
double p4
50-
51-
ctypedef s_binomial_t binomial_t
52-
53-
double random_standard_uniform(bitgen_t *bitgen_state) nogil
54-
void random_standard_uniform_fill(bitgen_t* bitgen_state, np.npy_intp cnt, double *out) nogil
55-
double random_standard_exponential(bitgen_t *bitgen_state) nogil
56-
double random_standard_exponential_f(bitgen_t *bitgen_state) nogil
57-
void random_standard_exponential_fill(bitgen_t *bitgen_state, np.npy_intp cnt, double *out) nogil
58-
void random_standard_exponential_fill_f(bitgen_t *bitgen_state, np.npy_intp cnt, double *out) nogil
59-
void random_standard_exponential_inv_fill(bitgen_t *bitgen_state, np.npy_intp cnt, double *out) nogil
60-
void random_standard_exponential_inv_fill_f(bitgen_t *bitgen_state, np.npy_intp cnt, double *out) nogil
61-
double random_standard_normal(bitgen_t* bitgen_state) nogil
62-
void random_standard_normal_fill(bitgen_t *bitgen_state, np.npy_intp count, double *out) nogil
63-
void random_standard_normal_fill_f(bitgen_t *bitgen_state, np.npy_intp count, float *out) nogil
64-
double random_standard_gamma(bitgen_t *bitgen_state, double shape) nogil
65-
66-
float random_standard_uniform_f(bitgen_t *bitgen_state) nogil
67-
void random_standard_uniform_fill_f(bitgen_t* bitgen_state, np.npy_intp cnt, float *out) nogil
68-
float random_standard_normal_f(bitgen_t* bitgen_state) nogil
69-
float random_standard_gamma_f(bitgen_t *bitgen_state, float shape) nogil
70-
71-
int64_t random_positive_int64(bitgen_t *bitgen_state) nogil
72-
int32_t random_positive_int32(bitgen_t *bitgen_state) nogil
73-
int64_t random_positive_int(bitgen_t *bitgen_state) nogil
74-
uint64_t random_uint(bitgen_t *bitgen_state) nogil
75-
76-
double random_normal(bitgen_t *bitgen_state, double loc, double scale) nogil
77-
78-
double random_gamma(bitgen_t *bitgen_state, double shape, double scale) nogil
79-
float random_gamma_f(bitgen_t *bitgen_state, float shape, float scale) nogil
80-
81-
double random_exponential(bitgen_t *bitgen_state, double scale) nogil
82-
double random_uniform(bitgen_t *bitgen_state, double lower, double range) nogil
83-
double random_beta(bitgen_t *bitgen_state, double a, double b) nogil
84-
double random_chisquare(bitgen_t *bitgen_state, double df) nogil
85-
double random_f(bitgen_t *bitgen_state, double dfnum, double dfden) nogil
86-
double random_standard_cauchy(bitgen_t *bitgen_state) nogil
87-
double random_pareto(bitgen_t *bitgen_state, double a) nogil
88-
double random_weibull(bitgen_t *bitgen_state, double a) nogil
89-
double random_power(bitgen_t *bitgen_state, double a) nogil
90-
double random_laplace(bitgen_t *bitgen_state, double loc, double scale) nogil
91-
double random_gumbel(bitgen_t *bitgen_state, double loc, double scale) nogil
92-
double random_logistic(bitgen_t *bitgen_state, double loc, double scale) nogil
93-
double random_lognormal(bitgen_t *bitgen_state, double mean, double sigma) nogil
94-
double random_rayleigh(bitgen_t *bitgen_state, double mode) nogil
95-
double random_standard_t(bitgen_t *bitgen_state, double df) nogil
96-
double random_noncentral_chisquare(bitgen_t *bitgen_state, double df,
97-
double nonc) nogil
98-
double random_noncentral_f(bitgen_t *bitgen_state, double dfnum,
99-
double dfden, double nonc) nogil
100-
double random_wald(bitgen_t *bitgen_state, double mean, double scale) nogil
101-
double random_vonmises(bitgen_t *bitgen_state, double mu, double kappa) nogil
102-
double random_triangular(bitgen_t *bitgen_state, double left, double mode,
103-
double right) nogil
104-
105-
int64_t random_poisson(bitgen_t *bitgen_state, double lam) nogil
106-
int64_t random_negative_binomial(bitgen_t *bitgen_state, double n, double p) nogil
107-
int64_t random_binomial(bitgen_t *bitgen_state, double p, int64_t n, binomial_t *binomial) nogil
108-
int64_t random_logseries(bitgen_t *bitgen_state, double p) nogil
109-
int64_t random_geometric_search(bitgen_t *bitgen_state, double p) nogil
110-
int64_t random_geometric_inversion(bitgen_t *bitgen_state, double p) nogil
111-
int64_t random_geometric(bitgen_t *bitgen_state, double p) nogil
112-
int64_t random_zipf(bitgen_t *bitgen_state, double a) nogil
113-
int64_t random_hypergeometric(bitgen_t *bitgen_state, int64_t good, int64_t bad,
114-
int64_t sample) nogil
115-
116-
uint64_t random_interval(bitgen_t *bitgen_state, uint64_t max) nogil
117-
118-
# Generate random uint64 numbers in closed interval [off, off + rng].
119-
uint64_t random_bounded_uint64(bitgen_t *bitgen_state,
120-
uint64_t off, uint64_t rng,
121-
uint64_t mask, bint use_masked) nogil
122-
123-
void random_multinomial(bitgen_t *bitgen_state, int64_t n, int64_t *mnix,
124-
double *pix, np.npy_intp d, binomial_t *binomial) nogil
125-
126-
int random_multivariate_hypergeometric_count(bitgen_t *bitgen_state,
127-
int64_t total,
128-
size_t num_colors, int64_t *colors,
129-
int64_t nsample,
130-
size_t num_variates, int64_t *variates) nogil
131-
void random_multivariate_hypergeometric_marginals(bitgen_t *bitgen_state,
132-
int64_t total,
133-
size_t num_colors, int64_t *colors,
134-
int64_t nsample,
135-
size_t num_variates, int64_t *variates) nogil
136-
13730
np.import_array()
13831

139-
14032
cdef int64_t _safe_sum_nonneg_int64(size_t num_colors, int64_t *colors):
14133
"""
14234
Sum the values in the array `colors`.
Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ def _coerce_to_uint32_array(x):
114114
Examples
115115
--------
116116
>>> import numpy as np
117-
>>> from numpy.random._bit_generator import _coerce_to_uint32_array
117+
>>> from numpy.random.bit_generator import _coerce_to_uint32_array
118118
>>> _coerce_to_uint32_array(12345)
119119
array([12345], dtype=uint32)
120120
>>> _coerce_to_uint32_array('12345')

‎numpy/random/c_distributions.pxd‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
#!python
2+
#cython: wraparound=False, nonecheck=False, boundscheck=False, cdivision=True, language_level=3
3+
from numpy cimport npy_intp
4+
5+
from libc.stdint cimport (uint64_t, int32_t, int64_t)
6+
from numpy.random cimport bitgen_t
7+
8+
cdef extern from "numpy/random/distributions.h":
9+
10+
struct s_binomial_t:
11+
int has_binomial
12+
double psave
13+
int64_t nsave
14+
double r
15+
double q
16+
double fm
17+
int64_t m
18+
double p1
19+
double xm
20+
double xl
21+
double xr
22+
double c
23+
double laml
24+
double lamr
25+
double p2
26+
double p3
27+
double p4
28+
29+
ctypedef s_binomial_t binomial_t
30+
31+
double random_standard_uniform(bitgen_t *bitgen_state) nogil
32+
void random_standard_uniform_fill(bitgen_t* bitgen_state, npy_intp cnt, double *out) nogil
33+
double random_standard_exponential(bitgen_t *bitgen_state) nogil
34+
double random_standard_exponential_f(bitgen_t *bitgen_state) nogil
35+
void random_standard_exponential_fill(bitgen_t *bitgen_state, npy_intp cnt, double *out) nogil
36+
void random_standard_exponential_fill_f(bitgen_t *bitgen_state, npy_intp cnt, double *out) nogil
37+
void random_standard_exponential_inv_fill(bitgen_t *bitgen_state, npy_intp cnt, double *out) nogil
38+
void random_standard_exponential_inv_fill_f(bitgen_t *bitgen_state, npy_intp cnt, double *out) nogil
39+
double random_standard_normal(bitgen_t* bitgen_state) nogil
40+
void random_standard_normal_fill(bitgen_t *bitgen_state, npy_intp count, double *out) nogil
41+
void random_standard_normal_fill_f(bitgen_t *bitgen_state, npy_intp count, float *out) nogil
42+
double random_standard_gamma(bitgen_t *bitgen_state, double shape) nogil
43+
44+
float random_standard_uniform_f(bitgen_t *bitgen_state) nogil
45+
void random_standard_uniform_fill_f(bitgen_t* bitgen_state, npy_intp cnt, float *out) nogil
46+
float random_standard_normal_f(bitgen_t* bitgen_state) nogil
47+
float random_standard_gamma_f(bitgen_t *bitgen_state, float shape) nogil
48+
49+
int64_t random_positive_int64(bitgen_t *bitgen_state) nogil
50+
int32_t random_positive_int32(bitgen_t *bitgen_state) nogil
51+
int64_t random_positive_int(bitgen_t *bitgen_state) nogil
52+
uint64_t random_uint(bitgen_t *bitgen_state) nogil
53+
54+
double random_normal(bitgen_t *bitgen_state, double loc, double scale) nogil
55+
56+
double random_gamma(bitgen_t *bitgen_state, double shape, double scale) nogil
57+
float random_gamma_f(bitgen_t *bitgen_state, float shape, float scale) nogil
58+
59+
double random_exponential(bitgen_t *bitgen_state, double scale) nogil
60+
double random_uniform(bitgen_t *bitgen_state, double lower, double range) nogil
61+
double random_beta(bitgen_t *bitgen_state, double a, double b) nogil
62+
double random_chisquare(bitgen_t *bitgen_state, double df) nogil
63+
double random_f(bitgen_t *bitgen_state, double dfnum, double dfden) nogil
64+
double random_standard_cauchy(bitgen_t *bitgen_state) nogil
65+
double random_pareto(bitgen_t *bitgen_state, double a) nogil
66+
double random_weibull(bitgen_t *bitgen_state, double a) nogil
67+
double random_power(bitgen_t *bitgen_state, double a) nogil
68+
double random_laplace(bitgen_t *bitgen_state, double loc, double scale) nogil
69+
double random_gumbel(bitgen_t *bitgen_state, double loc, double scale) nogil
70+
double random_logistic(bitgen_t *bitgen_state, double loc, double scale) nogil
71+
double random_lognormal(bitgen_t *bitgen_state, double mean, double sigma) nogil
72+
double random_rayleigh(bitgen_t *bitgen_state, double mode) nogil
73+
double random_standard_t(bitgen_t *bitgen_state, double df) nogil
74+
double random_noncentral_chisquare(bitgen_t *bitgen_state, double df,
75+
double nonc) nogil
76+
double random_noncentral_f(bitgen_t *bitgen_state, double dfnum,
77+
double dfden, double nonc) nogil
78+
double random_wald(bitgen_t *bitgen_state, double mean, double scale) nogil
79+
double random_vonmises(bitgen_t *bitgen_state, double mu, double kappa) nogil
80+
double random_triangular(bitgen_t *bitgen_state, double left, double mode,
81+
double right) nogil
82+
83+
int64_t random_poisson(bitgen_t *bitgen_state, double lam) nogil
84+
int64_t random_negative_binomial(bitgen_t *bitgen_state, double n, double p) nogil
85+
int64_t random_binomial(bitgen_t *bitgen_state, double p, int64_t n, binomial_t *binomial) nogil
86+
int64_t random_logseries(bitgen_t *bitgen_state, double p) nogil
87+
int64_t random_geometric_search(bitgen_t *bitgen_state, double p) nogil
88+
int64_t random_geometric_inversion(bitgen_t *bitgen_state, double p) nogil
89+
int64_t random_geometric(bitgen_t *bitgen_state, double p) nogil
90+
int64_t random_zipf(bitgen_t *bitgen_state, double a) nogil
91+
int64_t random_hypergeometric(bitgen_t *bitgen_state, int64_t good, int64_t bad,
92+
int64_t sample) nogil
93+
94+
uint64_t random_interval(bitgen_t *bitgen_state, uint64_t max) nogil
95+
96+
# Generate random uint64 numbers in closed interval [off, off + rng].
97+
uint64_t random_bounded_uint64(bitgen_t *bitgen_state,
98+
uint64_t off, uint64_t rng,
99+
uint64_t mask, bint use_masked) nogil
100+
101+
void random_multinomial(bitgen_t *bitgen_state, int64_t n, int64_t *mnix,
102+
double *pix, npy_intp d, binomial_t *binomial) nogil
103+
104+
int random_multivariate_hypergeometric_count(bitgen_t *bitgen_state,
105+
int64_t total,
106+
size_t num_colors, int64_t *colors,
107+
int64_t nsample,
108+
size_t num_variates, int64_t *variates) nogil
109+
void random_multivariate_hypergeometric_marginals(bitgen_t *bitgen_state,
110+
int64_t total,
111+
size_t num_colors, int64_t *colors,
112+
int64_t nsample,
113+
size_t num_variates, int64_t *variates) nogil
114+

0 commit comments

Comments
 (0)