Repository navigation
Expand file tree
/
Copy path_resampling.py
More file actions
2425 lines (2013 loc) · 104 KB
/
Copy path_resampling.py
File metadata and controls
2425 lines (2013 loc) · 104 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
import warnings
import numpy as np
from itertools import combinations, permutations, product, accumulate
from collections.abc import Sequence
from dataclasses import dataclass, field
import inspect
import math
from scipy._lib._util import (check_random_state, _rename_parameter, rng_integers,
_transition_to_rng)
from scipy._lib._array_api import (
array_namespace,
is_numpy,
is_array_api_strict,
xp_capabilities,
xp_result_type,
xp_size,
xp_device,
xp_swapaxes,
is_lazy_array,
)
from scipy._external import array_api_extra as xpx
from scipy.special import ndtr, ndtri
from scipy import stats
from ._common import ConfidenceInterval
from ._axis_nan_policy import _broadcast_concatenate, _broadcast_arrays
from ._warnings_errors import DegenerateDataWarning
__all__ = ['bootstrap', 'monte_carlo_test', 'permutation_test']
def _vectorize_statistic(statistic):
"""Vectorize an n-sample statistic"""
# This is a little cleaner than np.nditer at the expense of some data
# copying: concatenate samples together, then use np.apply_along_axis
def stat_nd(*data, axis=0):
lengths = [sample.shape[axis] for sample in data]
split_indices = np.cumsum(lengths)[:-1]
z = _broadcast_concatenate(data, axis)
# move working axis to position 0 so that new dimensions in the output
# of `statistic` are _prepended_. ("This axis is removed, and replaced
# with new dimensions...")
z = np.moveaxis(z, axis, 0)
def stat_1d(z):
data = np.split(z, split_indices)
return statistic(*data)
return np.apply_along_axis(stat_1d, 0, z)[()]
return stat_nd
def _jackknife_resample(sample, batch=None, *, xp):
"""Jackknife resample the sample. Only one-sample stats for now."""
n = sample.shape[-1]
batch_nominal = batch or n
for k in range(0, n, batch_nominal):
# col_start:col_end are the observations to remove
batch_actual = min(batch_nominal, n-k)
# jackknife - each row leaves out one observation
j = np.ones((batch_actual, n), dtype=bool)
np.fill_diagonal(j[:, k:k+batch_actual], False)
i = np.arange(n)
i = np.broadcast_to(i, (batch_actual, n))
i = i[j].reshape((batch_actual, n-1))
i = xp.asarray(i, device=xp_device(sample))
yield _get_from_last_axis(sample, i, xp=xp)
def _bootstrap_resample(sample, n_resamples=None, rng=None, *, xp):
"""Bootstrap resample the sample."""
n = sample.shape[-1]
# bootstrap - each row is a random resample of original observations
i = rng_integers(rng, 0, n, (n_resamples, n))
i = xp.asarray(i, device=xp_device(sample))
return _get_from_last_axis(sample, i, xp=xp)
def _get_from_last_axis(sample, i, xp):
if not is_array_api_strict(xp):
return sample[..., i]
# Equivalent to `sample[..., i]` as used in `bootstrap`. Assumes i.ndim <=2.
if i.ndim == 2:
sample = xp.expand_dims(sample, axis=-2)
sample, i = _broadcast_arrays((sample, i), axis=-1, xp=xp)
return xp.take_along_axis(sample, i, axis=-1)
def _percentile_of_score(a, score, axis, xp):
"""Vectorized, simplified `scipy.stats.percentileofscore`.
Uses logic of the 'mean' value of percentileofscore's kind parameter.
Unlike `stats.percentileofscore`, the percentile returned is a fraction
in [0, 1].
"""
B = a.shape[axis]
nonzeros = (xp.count_nonzero(a < score, axis=axis)
+ xp.count_nonzero(a <= score, axis=axis))
return xp.astype(nonzeros, score.dtype) / (2 * B)
def _bca_interval(data, statistic, axis, alpha, theta_hat_b, batch, xp):
"""Bias-corrected and accelerated interval."""
# closely follows [1] 14.3 and 15.4 (Eq. 15.36)
# calculate z0_hat
theta_hat = xp.expand_dims(statistic(*data, axis=axis), axis=-1)
percentile = _percentile_of_score(theta_hat_b, theta_hat, axis=-1, xp=xp)
percentile = xp.asarray(percentile, dtype=theta_hat.dtype,
device=xp_device(theta_hat))
z0_hat = ndtri(percentile)
# calculate a_hat
theta_hat_ji = [] # j is for sample of data, i is for jackknife resample
for j, sample in enumerate(data):
# _jackknife_resample will add an axis prior to the last axis that
# corresponds with the different jackknife resamples. Do the same for
# each sample of the data to ensure broadcastability. We need to
# create a copy of the list containing the samples anyway, so do this
# in the loop to simplify the code. This is not the bottleneck...
samples = [xp.expand_dims(sample, axis=-2) for sample in data]
theta_hat_i = []
for jackknife_sample in _jackknife_resample(sample, batch, xp=xp):
samples[j] = jackknife_sample
broadcasted = _broadcast_arrays(samples, axis=-1, xp=xp)
theta_hat_i.append(statistic(*broadcasted, axis=-1))
theta_hat_ji.append(theta_hat_i)
theta_hat_ji = [xp.concat(theta_hat_i, axis=-1)
for theta_hat_i in theta_hat_ji]
# CuPy array dtype is unstable in 1.13.6 when divided by large Python int:
# (cp.ones(1, dtype=cp.float32)/10000).dtype # float32
# (cp.ones(1, dtype=cp.float32)/100000).dtype # float64
# Solution for now is to make the divisor a Python float
# Looks like this will be fixed in 1.14.0
n_j = [float(theta_hat_i.shape[-1]) for theta_hat_i in theta_hat_ji]
theta_hat_j_dot = [xp.mean(theta_hat_i, axis=-1, keepdims=True)
for theta_hat_i in theta_hat_ji]
U_ji = [(n - 1) * (theta_hat_dot - theta_hat_i)
for theta_hat_dot, theta_hat_i, n
in zip(theta_hat_j_dot, theta_hat_ji, n_j)]
nums = [xp.sum(U_i**3, axis=-1)/n**3 for U_i, n in zip(U_ji, n_j)]
dens = [xp.sum(U_i**2, axis=-1)/n**2 for U_i, n in zip(U_ji, n_j)]
a_hat = 1/6 * sum(nums) / sum(dens)**(3/2)
# calculate alpha_1, alpha_2
# `float` because dtype of ndtri output (float64) should not promote other vals
z_alpha = float(ndtri(alpha))
z_1alpha = -z_alpha
num1 = z0_hat + z_alpha
alpha_1 = ndtr(z0_hat + num1/(1 - a_hat*num1))
num2 = z0_hat + z_1alpha
alpha_2 = ndtr(z0_hat + num2/(1 - a_hat*num2))
return alpha_1, alpha_2, a_hat # return a_hat for testing
def _bootstrap_iv(data, statistic, vectorized, paired, axis, confidence_level,
alternative, n_resamples, batch, method, bootstrap_result,
rng):
"""Input validation and standardization for `bootstrap`."""
xp = array_namespace(*data)
if vectorized not in {True, False, None}:
raise ValueError("`vectorized` must be `True`, `False`, or `None`.")
if vectorized is None:
vectorized = 'axis' in inspect.signature(statistic).parameters
if not vectorized:
if not is_numpy(xp):
message = (f"When using array library {xp.__name__}, `func` must be "
"vectorized and accept argument `axis`.")
raise TypeError(message)
statistic = _vectorize_statistic(statistic)
axis_int = int(axis)
if axis != axis_int:
raise ValueError("`axis` must be an integer.")
n_samples = 0
try:
n_samples = len(data)
except TypeError:
raise ValueError("`data` must be a sequence of samples.")
if n_samples == 0:
raise ValueError("`data` must contain at least one sample.")
data = _broadcast_arrays(data, axis_int, xp=xp)
data_iv = []
for sample in data:
if sample.shape[axis_int] <= 1:
raise ValueError("each sample in `data` must contain two or more "
"observations along `axis`.")
sample = xp.moveaxis(sample, axis_int, -1)
data_iv.append(sample)
if paired not in {True, False}:
raise ValueError("`paired` must be `True` or `False`.")
if paired:
n = data_iv[0].shape[-1]
for sample in data_iv[1:]:
if sample.shape[-1] != n:
message = ("When `paired is True`, all samples must have the "
"same length along `axis`")
raise ValueError(message)
# to generate the bootstrap distribution for paired-sample statistics,
# resample the indices of the observations
def statistic(i, axis=-1, data=data_iv, unpaired_statistic=statistic):
# data = [sample[..., i] for sample in data]
data = [_get_from_last_axis(sample, i, xp=xp) for sample in data]
return unpaired_statistic(*data, axis=axis)
data_iv = [xp.arange(n)]
confidence_level_float = float(confidence_level)
alternative = alternative.lower()
alternatives = {'two-sided', 'less', 'greater'}
if alternative not in alternatives:
raise ValueError(f"`alternative` must be one of {alternatives}")
n_resamples_int = int(n_resamples)
if n_resamples != n_resamples_int or n_resamples_int < 0:
raise ValueError("`n_resamples` must be a non-negative integer.")
if batch is None:
batch_iv = batch
else:
batch_iv = int(batch)
if batch != batch_iv or batch_iv <= 0:
raise ValueError("`batch` must be a positive integer or None.")
methods = {'percentile', 'basic', 'bca'}
method = method.lower()
if method not in methods:
raise ValueError(f"`method` must be in {methods}")
message = "`bootstrap_result` must have attribute `bootstrap_distribution'"
if (bootstrap_result is not None
and not hasattr(bootstrap_result, "bootstrap_distribution")):
raise ValueError(message)
message = ("Either `bootstrap_result.bootstrap_distribution.size` or "
"`n_resamples` must be positive.")
if ((not bootstrap_result or
not xp_size(bootstrap_result.bootstrap_distribution))
and n_resamples_int == 0):
raise ValueError(message)
rng = check_random_state(rng)
return (data_iv, statistic, vectorized, paired, axis_int,
confidence_level_float, alternative, n_resamples_int, batch_iv,
method, bootstrap_result, rng, xp)
@dataclass
class BootstrapResult:
"""Result object returned by `scipy.stats.bootstrap`.
Attributes
----------
confidence_interval : ConfidenceInterval
The bootstrap confidence interval as an instance of
`collections.namedtuple` with attributes `low` and `high`.
bootstrap_distribution : ndarray
The bootstrap distribution, that is, the value of `statistic` for
each resample. The last dimension corresponds with the resamples
(e.g. ``res.bootstrap_distribution.shape[-1] == n_resamples``).
standard_error : float or ndarray
The bootstrap standard error, that is, the sample standard
deviation of the bootstrap distribution.
"""
confidence_interval: ConfidenceInterval
bootstrap_distribution: np.ndarray
standard_error: float | np.ndarray
@xp_capabilities(skip_backends=[("dask.array", "Dask doesn't have take_along_axis.")],
jax_jit=False) # a few failed assertions - not sure what's going on
@_transition_to_rng('random_state')
def bootstrap(data, statistic, *, n_resamples=9999, batch=None,
vectorized=None, paired=False, axis=0, confidence_level=0.95,
alternative='two-sided', method='BCa', bootstrap_result=None,
rng=None):
r"""
Compute a two-sided bootstrap confidence interval of a statistic.
When `method` is ``'percentile'`` and `alternative` is ``'two-sided'``,
a bootstrap confidence interval is computed according to the following
procedure.
1. Resample the data: for each sample in `data` and for each of
`n_resamples`, take a random sample of the original sample
(with replacement) of the same size as the original sample.
2. Compute the bootstrap distribution of the statistic: for each set of
resamples, compute the test statistic.
3. Determine the confidence interval: find the interval of the bootstrap
distribution that is
- symmetric about the median and
- contains `confidence_level` of the resampled statistic values.
While the ``'percentile'`` method is the most intuitive, it is rarely
used in practice. Two more common methods are available, ``'basic'``
('reverse percentile') and ``'BCa'`` ('bias-corrected and accelerated');
they differ in how step 3 is performed.
If the samples in `data` are taken at random from their respective
distributions :math:`n` times, the confidence interval returned by
`bootstrap` will contain the true value of the statistic for those
distributions approximately `confidence_level`:math:`\, \times \, n` times.
Parameters
----------
data : sequence of array-like
Each element of `data` is a sample containing scalar observations from an
underlying distribution. Elements of `data` must be broadcastable to the
same shape (with the possible exception of the dimension specified by `axis`).
statistic : callable
Statistic for which the confidence interval is to be calculated.
`statistic` must be a callable that accepts ``len(data)`` samples
as separate arguments and returns the resulting statistic.
If `vectorized` is set ``True``,
`statistic` must also accept a keyword argument `axis` and be
vectorized to compute the statistic along the provided `axis`.
n_resamples : int, default: ``9999``
The number of resamples performed to form the bootstrap distribution
of the statistic.
batch : int, optional
The number of resamples to process in each vectorized call to
`statistic`. Memory usage is O( `batch` * ``n`` ), where ``n`` is the
sample size. Default is ``None``, in which case ``batch = n_resamples``
(or ``batch = max(n_resamples, n)`` for ``method='BCa'``).
vectorized : bool, optional
If `vectorized` is set ``False``, `statistic` will not be passed
keyword argument `axis` and is expected to calculate the statistic
only for 1D samples. If ``True``, `statistic` will be passed keyword
argument `axis` and is expected to calculate the statistic along `axis`
when passed an ND sample array. If ``None`` (default), `vectorized`
will be set ``True`` if ``axis`` is a parameter of `statistic`. Use of
a vectorized statistic typically reduces computation time.
paired : bool, default: ``False``
Whether the statistic treats corresponding elements of the samples
in `data` as paired. If True, `bootstrap` resamples an array of
*indices* and uses the same indices for all arrays in `data`; otherwise,
`bootstrap` independently resamples the elements of each array.
axis : int, default: ``0``
The axis of the samples in `data` along which the `statistic` is
calculated.
confidence_level : float, default: ``0.95``
The confidence level of the confidence interval.
alternative : {'two-sided', 'less', 'greater'}, default: ``'two-sided'``
Choose ``'two-sided'`` (default) for a two-sided confidence interval,
``'less'`` for a one-sided confidence interval with the lower bound
at ``-np.inf``, and ``'greater'`` for a one-sided confidence interval
with the upper bound at ``np.inf``. The other bound of the one-sided
confidence intervals is the same as that of a two-sided confidence
interval with `confidence_level` twice as far from 1.0; e.g. the upper
bound of a 95% ``'less'`` confidence interval is the same as the upper
bound of a 90% ``'two-sided'`` confidence interval.
method : {'percentile', 'basic', 'bca'}, default: ``'BCa'``
Whether to return the 'percentile' bootstrap confidence interval
(``'percentile'``), the 'basic' (AKA 'reverse') bootstrap confidence
interval (``'basic'``), or the bias-corrected and accelerated bootstrap
confidence interval (``'BCa'``).
bootstrap_result : BootstrapResult, optional
Provide the result object returned by a previous call to `bootstrap`
to include the previous bootstrap distribution in the new bootstrap
distribution. This can be used, for example, to change
`confidence_level`, change `method`, or see the effect of performing
additional resampling without repeating computations.
rng : `numpy.random.Generator`, optional
Pseudorandom number generator state. When `rng` is None, a new
`numpy.random.Generator` is created using entropy from the
operating system. Types other than `numpy.random.Generator` are
passed to `numpy.random.default_rng` to instantiate a ``Generator``.
Returns
-------
res : BootstrapResult
An object with attributes:
confidence_interval : ConfidenceInterval
The bootstrap confidence interval as an instance of
`collections.namedtuple` with attributes `low` and `high`.
bootstrap_distribution : ndarray
The bootstrap distribution, that is, the value of `statistic` for
each resample. The last dimension corresponds with the resamples
(e.g. ``res.bootstrap_distribution.shape[-1] == n_resamples``).
standard_error : float or ndarray
The bootstrap standard error, that is, the sample standard
deviation of the bootstrap distribution.
Warns
-----
`~scipy.stats.DegenerateDataWarning`
Generated when ``method='BCa'`` and the bootstrap distribution is
degenerate (e.g. all elements are identical).
Notes
-----
Elements of the confidence interval may be NaN for ``method='BCa'`` if
the bootstrap distribution is degenerate (e.g. all elements are identical).
In this case, consider using another `method` or inspecting `data` for
indications that other analysis may be more appropriate (e.g. all
observations are identical).
References
----------
.. [1] B. Efron and R. J. Tibshirani, An Introduction to the Bootstrap,
Chapman & Hall/CRC, Boca Raton, FL, USA (1993)
.. [2] Nathaniel E. Helwig, "Bootstrap Confidence Intervals",
http://users.stat.umn.edu/~helwig/notes/bootci-Notes.pdf
.. [3] Bootstrapping (statistics), Wikipedia,
https://en.wikipedia.org/wiki/Bootstrapping_%28statistics%29
Examples
--------
Suppose we have sampled data from an unknown distribution.
>>> import numpy as np
>>> rng = np.random.default_rng()
>>> from scipy.stats import norm
>>> dist = norm(loc=2, scale=4) # our "unknown" distribution
>>> data = dist.rvs(size=100, random_state=rng)
We are interested in the standard deviation of the distribution.
>>> std_true = dist.std() # the true value of the statistic
>>> print(std_true)
4.0
>>> std_sample = np.std(data) # the sample statistic
>>> print(std_sample)
3.9460644295563863
The bootstrap is used to approximate the variability we would expect if we
were to repeatedly sample from the unknown distribution and calculate the
statistic of the sample each time. It does this by repeatedly resampling
values *from the original sample* with replacement and calculating the
statistic of each resample. This results in a "bootstrap distribution" of
the statistic.
>>> import matplotlib.pyplot as plt
>>> from scipy.stats import bootstrap
>>> data = (data,) # samples must be in a sequence
>>> res = bootstrap(data, np.std, confidence_level=0.9, rng=rng)
>>> fig, ax = plt.subplots()
>>> ax.hist(res.bootstrap_distribution, bins=25)
>>> ax.set_title('Bootstrap Distribution')
>>> ax.set_xlabel('statistic value')
>>> ax.set_ylabel('frequency')
>>> plt.show()
The standard error quantifies this variability. It is calculated as the
standard deviation of the bootstrap distribution.
>>> res.standard_error
0.24427002125829136
>>> res.standard_error == np.std(res.bootstrap_distribution, ddof=1)
True
The bootstrap distribution of the statistic is often approximately normal
with scale equal to the standard error.
>>> x = np.linspace(3, 5)
>>> pdf = norm.pdf(x, loc=std_sample, scale=res.standard_error)
>>> fig, ax = plt.subplots()
>>> ax.hist(res.bootstrap_distribution, bins=25, density=True)
>>> ax.plot(x, pdf)
>>> ax.set_title('Normal Approximation of the Bootstrap Distribution')
>>> ax.set_xlabel('statistic value')
>>> ax.set_ylabel('pdf')
>>> plt.show()
This suggests that we could construct a 90% confidence interval on the
statistic based on quantiles of this normal distribution.
>>> norm.interval(0.9, loc=std_sample, scale=res.standard_error)
(3.5442759991341726, 4.3478528599786)
Due to central limit theorem, this normal approximation is accurate for a
variety of statistics and distributions underlying the samples; however,
the approximation is not reliable in all cases. Because `bootstrap` is
designed to work with arbitrary underlying distributions and statistics,
it uses more advanced techniques to generate an accurate confidence
interval.
>>> print(res.confidence_interval)
ConfidenceInterval(low=3.57655333533867, high=4.382043696342881)
If we sample from the original distribution 100 times and form a bootstrap
confidence interval for each sample, the confidence interval
contains the true value of the statistic approximately 90% of the time.
>>> n_trials = 100
>>> ci_contains_true_std = 0
>>> for i in range(n_trials):
... data = (dist.rvs(size=100, random_state=rng),)
... res = bootstrap(data, np.std, confidence_level=0.9,
... n_resamples=999, rng=rng)
... ci = res.confidence_interval
... if ci[0] < std_true < ci[1]:
... ci_contains_true_std += 1
>>> print(ci_contains_true_std)
88
Rather than writing a loop, we can also determine the confidence intervals
for all 100 samples at once.
>>> data = (dist.rvs(size=(n_trials, 100), random_state=rng),)
>>> res = bootstrap(data, np.std, axis=-1, confidence_level=0.9,
... n_resamples=999, rng=rng)
>>> ci_l, ci_u = res.confidence_interval
Here, `ci_l` and `ci_u` contain the confidence interval for each of the
``n_trials = 100`` samples.
>>> print(ci_l[:5])
[3.86401283 3.33304394 3.52474647 3.54160981 3.80569252]
>>> print(ci_u[:5])
[4.80217409 4.18143252 4.39734707 4.37549713 4.72843584]
And again, approximately 90% contain the true value, ``std_true = 4``.
>>> print(np.sum((ci_l < std_true) & (std_true < ci_u)))
93
`bootstrap` can also be used to estimate confidence intervals of
multi-sample statistics. For example, to get a confidence interval
for the difference between means, we write a function that accepts
two sample arguments and returns only the statistic. The use of the
``axis`` argument ensures that all mean calculations are perform in
a single vectorized call, which is faster than looping over pairs
of resamples in Python.
>>> def my_statistic(sample1, sample2, axis=-1):
... mean1 = np.mean(sample1, axis=axis)
... mean2 = np.mean(sample2, axis=axis)
... return mean1 - mean2
Here, we use the 'percentile' method with the default 95% confidence level.
>>> sample1 = norm.rvs(scale=1, size=100, random_state=rng)
>>> sample2 = norm.rvs(scale=2, size=100, random_state=rng)
>>> data = (sample1, sample2)
>>> res = bootstrap(data, my_statistic, method='basic', rng=rng)
>>> print(my_statistic(sample1, sample2))
0.16661030792089523
>>> print(res.confidence_interval)
ConfidenceInterval(low=-0.29087973240818693, high=0.6371338699912273)
The bootstrap estimate of the standard error is also available.
>>> print(res.standard_error)
0.238323948262459
Paired-sample statistics work, too. For example, consider the Pearson
correlation coefficient.
>>> from scipy.stats import pearsonr
>>> n = 100
>>> x = np.linspace(0, 10, n)
>>> y = x + rng.uniform(size=n)
>>> print(pearsonr(x, y)[0]) # element 0 is the statistic
0.9954306665125647
We wrap `pearsonr` so that it returns only the statistic, ensuring
that we use the `axis` argument because it is available.
>>> def my_statistic(x, y, axis=-1):
... return pearsonr(x, y, axis=axis)[0]
We call `bootstrap` using ``paired=True``.
>>> res = bootstrap((x, y), my_statistic, paired=True, rng=rng)
>>> print(res.confidence_interval)
ConfidenceInterval(low=0.9941504301315878, high=0.996377412215445)
The result object can be passed back into `bootstrap` to perform additional
resampling:
>>> len(res.bootstrap_distribution)
9999
>>> res = bootstrap((x, y), my_statistic, paired=True,
... n_resamples=1000, rng=rng,
... bootstrap_result=res)
>>> len(res.bootstrap_distribution)
10999
or to change the confidence interval options:
>>> res2 = bootstrap((x, y), my_statistic, paired=True,
... n_resamples=0, rng=rng, bootstrap_result=res,
... method='percentile', confidence_level=0.9)
>>> np.testing.assert_equal(res2.bootstrap_distribution,
... res.bootstrap_distribution)
>>> res.confidence_interval
ConfidenceInterval(low=0.9941574828235082, high=0.9963781698210212)
without repeating computation of the original bootstrap distribution.
"""
# Input validation
args = _bootstrap_iv(data, statistic, vectorized, paired, axis,
confidence_level, alternative, n_resamples, batch,
method, bootstrap_result, rng)
(data, statistic, vectorized, paired, axis, confidence_level,
alternative, n_resamples, batch, method, bootstrap_result,
rng, xp) = args
theta_hat_b = ([] if bootstrap_result is None
else [bootstrap_result.bootstrap_distribution])
batch_nominal = batch or n_resamples or 1
for k in range(0, n_resamples, batch_nominal):
batch_actual = min(batch_nominal, n_resamples-k)
# Generate resamples
resampled_data = []
for sample in data:
resample = _bootstrap_resample(sample, n_resamples=batch_actual,
rng=rng, xp=xp)
resampled_data.append(resample)
# Compute bootstrap distribution of statistic
theta_hat_b.append(statistic(*resampled_data, axis=-1))
theta_hat_b = xp.concat(theta_hat_b, axis=-1)
# Calculate percentile interval
alpha = ((1 - confidence_level)/2 if alternative == 'two-sided'
else (1 - confidence_level))
if method == 'bca':
interval = _bca_interval(data, statistic, axis=-1, alpha=alpha,
theta_hat_b=theta_hat_b, batch=batch, xp=xp)[:2]
else:
alpha = xp.asarray(alpha, dtype=theta_hat_b.dtype,
device=xp_device(theta_hat_b))
interval = alpha, 1 - alpha
# Calculate confidence interval of statistic
interval = xp.stack(interval, axis=-1)
ci = stats.quantile(theta_hat_b, interval, axis=-1)
if not is_lazy_array(ci) and xp.any(xp.isnan(ci)):
msg = (
"The BCa confidence interval cannot be calculated. "
"This problem is known to occur when the distribution "
"is degenerate or the statistic is np.min."
)
warnings.warn(DegenerateDataWarning(msg), stacklevel=2)
ci_l = ci[..., 0]
ci_u = ci[..., 1]
if method == 'basic': # see [3]
theta_hat = statistic(*data, axis=-1)
ci_l, ci_u = 2*theta_hat - ci_u, 2*theta_hat - ci_l
if alternative == 'less':
ci_l = xp.full_like(ci_l, -xp.inf)
elif alternative == 'greater':
ci_u = xp.full_like(ci_u, xp.inf)
standard_error = xp.std(theta_hat_b, correction=1, axis=-1)
ci_l = ci_l[()] if ci_l.ndim == 0 else ci_l
ci_u = ci_u[()] if ci_u.ndim == 0 else ci_u
standard_error = standard_error[()] if standard_error.ndim == 0 else standard_error
return BootstrapResult(confidence_interval=ConfidenceInterval(ci_l, ci_u),
bootstrap_distribution=theta_hat_b,
standard_error=standard_error)
def _monte_carlo_test_iv(data, rvs, statistic, vectorized, n_resamples,
batch, alternative, axis):
"""Input validation for `monte_carlo_test`."""
axis_int = int(axis)
if axis != axis_int:
raise ValueError("`axis` must be an integer.")
if vectorized not in {True, False, None}:
raise ValueError("`vectorized` must be `True`, `False`, or `None`.")
if not isinstance(rvs, Sequence):
rvs = (rvs,)
data = (data,)
for rvs_i in rvs:
if not callable(rvs_i):
raise TypeError("`rvs` must be callable or sequence of callables.")
# At this point, `data` should be a sequence
# If it isn't, the user passed a sequence for `rvs` but not `data`
message = "If `rvs` is a sequence, `len(rvs)` must equal `len(data)`."
try:
len(data)
except TypeError as e:
raise ValueError(message) from e
if not len(rvs) == len(data):
raise ValueError(message)
if not callable(statistic):
raise TypeError("`statistic` must be callable.")
if vectorized is None:
try:
signature = inspect.signature(statistic).parameters
except ValueError as e:
message = (f"Signature inspection of {statistic=} failed; "
"pass `vectorize` explicitly.")
raise ValueError(message) from e
vectorized = 'axis' in signature
xp = array_namespace(*data)
dtype = xp_result_type(*data, force_floating=True, xp=xp)
if not vectorized:
if is_numpy(xp):
statistic_vectorized = _vectorize_statistic(statistic)
else:
message = ("`statistic` must be vectorized (i.e. support an `axis` "
f"argument) when `data` contains {xp.__name__} arrays.")
raise ValueError(message)
else:
statistic_vectorized = statistic
data = _broadcast_arrays(data, axis, xp=xp)
data_iv = []
for sample in data:
sample = xp.broadcast_to(sample, (1,)) if sample.ndim == 0 else sample
sample = xp.moveaxis(sample, axis_int, -1)
data_iv.append(sample)
n_resamples_int = int(n_resamples)
if n_resamples != n_resamples_int or n_resamples_int <= 0:
raise ValueError("`n_resamples` must be a positive integer.")
if batch is None:
batch_iv = batch
else:
batch_iv = int(batch)
if batch != batch_iv or batch_iv <= 0:
raise ValueError("`batch` must be a positive integer or None.")
alternatives = {'two-sided', 'greater', 'less'}
alternative = alternative.lower()
if alternative not in alternatives:
raise ValueError(f"`alternative` must be in {alternatives}")
return (data_iv, rvs, statistic_vectorized, vectorized, n_resamples_int,
batch_iv, alternative, axis_int, dtype, xp)
@dataclass
class MonteCarloTestResult:
"""Result object returned by `scipy.stats.monte_carlo_test`.
Attributes
----------
statistic : float or ndarray
The observed test statistic of the sample.
pvalue : float or ndarray
The p-value for the given alternative.
null_distribution : ndarray
The values of the test statistic generated under the null
hypothesis.
"""
statistic: float | np.ndarray
pvalue: float | np.ndarray
null_distribution: np.ndarray
@xp_capabilities()
@_rename_parameter('sample', 'data')
def monte_carlo_test(data, rvs, statistic, *, vectorized=None,
n_resamples=9999, batch=None, alternative="two-sided",
axis=0):
r"""Perform a Monte Carlo hypothesis test.
`data` contains a sample or a sequence of one or more samples. `rvs`
specifies the distribution(s) of the sample(s) in `data` under the null
hypothesis. The value of `statistic` for the given `data` is compared
against a Monte Carlo null distribution: the value of the statistic for
each of `n_resamples` sets of samples generated using `rvs`. This gives
the p-value, the probability of observing such an extreme value of the
test statistic under the null hypothesis.
Parameters
----------
data : array-like or sequence of array-like
An array or sequence of arrays of observations.
rvs : callable or tuple of callables
A callable or sequence of callables that generates random variates
under the null hypothesis. Each element of `rvs` must be a callable
that accepts keyword argument ``size`` (e.g. ``rvs(size=(m, n))``) and
returns an N-d array sample of that shape. If `rvs` is a sequence, the
number of callables in `rvs` must match the number of samples in
`data`, i.e. ``len(rvs) == len(data)``. If `rvs` is a single callable,
`data` is treated as a single sample.
statistic : callable
Statistic for which the p-value of the hypothesis test is to be
calculated. `statistic` must be a callable that accepts a sample
(e.g. ``statistic(sample)``) or ``len(rvs)`` separate samples (e.g.
``statistic(samples1, sample2)`` if `rvs` contains two callables and
`data` contains two samples) and returns the resulting statistic.
If `vectorized` is set ``True``, `statistic` must also accept a keyword
argument `axis` and be vectorized to compute the statistic along the
provided `axis` of the samples in `data`.
vectorized : bool, optional
If `vectorized` is set ``False``, `statistic` will not be passed
keyword argument `axis` and is expected to calculate the statistic
only for 1D samples. If ``True``, `statistic` will be passed keyword
argument `axis` and is expected to calculate the statistic along `axis`
when passed ND sample arrays. If ``None`` (default), `vectorized`
will be set ``True`` if ``axis`` is a parameter of `statistic`. Use of
a vectorized statistic typically reduces computation time.
n_resamples : int, default: 9999
Number of samples drawn from each of the callables of `rvs`.
Equivalently, the number statistic values under the null hypothesis
used as the Monte Carlo null distribution.
batch : int, optional
The number of Monte Carlo samples to process in each call to
`statistic`. Memory usage is O( `batch` * ``sample.size[axis]`` ). Default
is ``None``, in which case `batch` equals `n_resamples`.
alternative : {'two-sided', 'less', 'greater'}
The alternative hypothesis for which the p-value is calculated.
For each alternative, the p-value is defined as follows.
- ``'greater'`` : the percentage of the null distribution that is
greater than or equal to the observed value of the test statistic.
- ``'less'`` : the percentage of the null distribution that is
less than or equal to the observed value of the test statistic.
- ``'two-sided'`` : twice the smaller of the p-values above.
axis : int, default: 0
The axis of `data` (or each sample within `data`) over which to
calculate the statistic.
Returns
-------
res : MonteCarloTestResult
An object with attributes:
statistic : float or ndarray
The test statistic of the observed `data`.
pvalue : float or ndarray
The p-value for the given alternative.
null_distribution : ndarray
The values of the test statistic generated under the null
hypothesis.
.. warning::
The p-value is calculated by counting the elements of the null
distribution that are as extreme or more extreme than the observed
value of the statistic. Due to the use of finite precision arithmetic,
some statistic functions return numerically distinct values when the
theoretical values would be exactly equal. In some cases, this could
lead to a large error in the calculated p-value. `monte_carlo_test`
guards against this by considering elements in the null distribution
that are "close" (within a relative tolerance of 100 times the
floating point epsilon of inexact dtypes) to the observed
value of the test statistic as equal to the observed value of the
test statistic. However, the user is advised to inspect the null
distribution to assess whether this method of comparison is
appropriate, and if not, calculate the p-value manually.
References
----------
.. [1] B. Phipson and G. K. Smyth. "Permutation P-values Should Never Be
Zero: Calculating Exact P-values When Permutations Are Randomly Drawn."
Statistical Applications in Genetics and Molecular Biology 9.1 (2010).
Examples
--------
Suppose we wish to test whether a small sample has been drawn from a normal
distribution. We decide that we will use the skew of the sample as a
test statistic, and we will consider a p-value of 0.05 to be statistically
significant.
>>> import numpy as np
>>> from scipy import stats
>>> def statistic(x, axis):
... return stats.skew(x, axis)
After collecting our data, we calculate the observed value of the test
statistic.
>>> rng = np.random.default_rng()
>>> x = stats.skewnorm.rvs(a=1, size=50, random_state=rng)
>>> statistic(x, axis=0)
0.12457412450240658
To determine the probability of observing such an extreme value of the
skewness by chance if the sample were drawn from the normal distribution,
we can perform a Monte Carlo hypothesis test. The test will draw many
samples at random from their normal distribution, calculate the skewness
of each sample, and compare our original skewness against this
distribution to determine an approximate p-value.
>>> from scipy.stats import monte_carlo_test
>>> # because our statistic is vectorized, we pass `vectorized=True`
>>> rvs = lambda size: stats.norm.rvs(size=size, random_state=rng)
>>> res = monte_carlo_test(x, rvs, statistic, vectorized=True)
>>> print(res.statistic)
0.12457412450240658
>>> print(res.pvalue)
0.7012
The probability of obtaining a test statistic less than or equal to the
observed value under the null hypothesis is ~70%. This is greater than
our chosen threshold of 5%, so we cannot consider this to be significant
evidence against the null hypothesis.
Note that this p-value essentially matches that of
`scipy.stats.skewtest`, which relies on an asymptotic distribution of a
test statistic based on the sample skewness.
>>> stats.skewtest(x).pvalue
0.6892046027110614
This asymptotic approximation is not valid for small sample sizes, but
`monte_carlo_test` can be used with samples of any size.
>>> x = stats.skewnorm.rvs(a=1, size=7, random_state=rng)
>>> # stats.skewtest(x) would produce an error due to small sample
>>> res = monte_carlo_test(x, rvs, statistic, vectorized=True)
The Monte Carlo distribution of the test statistic is provided for
further investigation.
>>> import matplotlib.pyplot as plt
>>> fig, ax = plt.subplots()
>>> ax.hist(res.null_distribution, bins=50)
>>> ax.set_title("Monte Carlo distribution of test statistic")
>>> ax.set_xlabel("Value of Statistic")
>>> ax.set_ylabel("Frequency")
>>> plt.show()
"""
args = _monte_carlo_test_iv(data, rvs, statistic, vectorized,
n_resamples, batch, alternative, axis)
(data, rvs, statistic, vectorized, n_resamples,
batch, alternative, axis, dtype, xp) = args
# Some statistics return plain floats; ensure they're at least a NumPy float
observed = xp.asarray(statistic(*data, axis=-1))
observed = observed[()] if observed.ndim == 0 else observed
n_observations = [sample.shape[-1] for sample in data]
batch_nominal = batch or n_resamples
null_distribution = []
for k in range(0, n_resamples, batch_nominal):
batch_actual = min(batch_nominal, n_resamples - k)
resamples = [rvs_i(size=(batch_actual, n_observations_i))
for rvs_i, n_observations_i in zip(rvs, n_observations)]
null_distribution.append(statistic(*resamples, axis=-1))
null_distribution = xp.concat(null_distribution)
null_distribution = xp.reshape(null_distribution, (-1,) + (1,)*observed.ndim)
# relative tolerance for detecting numerically distinct but
# theoretically equal values in the null distribution
eps = (0 if not xp.isdtype(observed.dtype, ('real floating'))
else xp.finfo(observed.dtype).eps*100)
gamma = xp.abs(eps * observed)
def less(null_distribution, observed):
cmps = null_distribution <= observed + gamma
cmps = xp.asarray(cmps, dtype=dtype)
pvalues = (xp.sum(cmps, axis=0, dtype=dtype) + 1.) / (n_resamples + 1.)
return pvalues
def greater(null_distribution, observed):
cmps = null_distribution >= observed - gamma
cmps = xp.asarray(cmps, dtype=dtype)
pvalues = (xp.sum(cmps, axis=0, dtype=dtype) + 1.) / (n_resamples + 1.)
return pvalues
def two_sided(null_distribution, observed):