Unstable sklearn/svm/tests/test_bounds.py::test_newrand_set_seed #25937
Description
On my local laptop I often observe a random failure for test_newrand_set_seed.
I am running macos m1 with Python 3.11 from conda-forge. scikit-learn has been built with the clang compilers from conda-forge.
Here is a typical run with 10 repetitions using pytest-repeat:
Details
pytest -v -k "test_newrand_set_seed[None" sklearn/svm/tests/test_bounds.py --count 10
================================================================================ test session starts =================================================================================
platform darwin -- Python 3.11.0, pytest-7.2.0, pluggy-1.0.0 -- /Users/ogrisel/mambaforge/envs/dev/bin/python3.11
cachedir: .pytest_cache
hypothesis profile 'default' -> database=DirectoryBasedExampleDatabase('/Users/ogrisel/code/scikit-learn/.hypothesis/examples')
Using --randomly-seed=1301508857
benchmark: 4.0.0 (defaults: timer=time.perf_counter disable_gc=False min_rounds=5 min_time=0.000005 max_time=1.0 calibration_precision=10 warmup=False warmup_iterations=100000)
rootdir: /Users/ogrisel/code/scikit-learn, configfile: setup.cfg
plugins: snapshot-0.9.0, xdist-3.1.0, mock-3.10.0, clarity-1.0.1, hypothesis-6.62.0, randomly-3.12.0, profiling-1.7.0, cov-4.0.0, repeat-0.9.1, anyio-3.6.2, benchmark-4.0.0
collected 260 items / 250 deselected / 10 selected
sklearn/svm/tests/test_bounds.py::test_newrand_set_seed[None-81-5-10] PASSED [ 10%]
sklearn/svm/tests/test_bounds.py::test_newrand_set_seed[None-81-4-10] FAILED [ 20%]
sklearn/svm/tests/test_bounds.py::test_newrand_set_seed[None-81-7-10] FAILED [ 30%]
sklearn/svm/tests/test_bounds.py::test_newrand_set_seed[None-81-1-10] FAILED [ 40%]
sklearn/svm/tests/test_bounds.py::test_newrand_set_seed[None-81-9-10] FAILED [ 50%]
sklearn/svm/tests/test_bounds.py::test_newrand_set_seed[None-81-8-10] FAILED [ 60%]
sklearn/svm/tests/test_bounds.py::test_newrand_set_seed[None-81-10-10] FAILED [ 70%]
sklearn/svm/tests/test_bounds.py::test_newrand_set_seed[None-81-3-10] FAILED [ 80%]
sklearn/svm/tests/test_bounds.py::test_newrand_set_seed[None-81-6-10] FAILED [ 90%]
sklearn/svm/tests/test_bounds.py::test_newrand_set_seed[None-81-2-10] FAILED [100%]
====================================================================================== FAILURES ======================================================================================
________________________________________________________________________ test_newrand_set_seed[None-81-4-10] _________________________________________________________________________
seed = None, val = 81
@pytest.mark.parametrize("seed, val", [(None, 81), (0, 54), (_MAX_UNSIGNED_INT, 9)])
def test_newrand_set_seed(seed, val):
"""Test that `set_seed` produces deterministic results"""
if seed is not None:
set_seed_wrap(seed)
x = bounded_rand_int_wrap(100)
> assert x == val, f"Expected {val} but got {x} instead"
E AssertionError: Expected 81 but got 13 instead
E assert 13 == 81
sklearn/svm/tests/test_bounds.py:81: AssertionError
________________________________________________________________________ test_newrand_set_seed[None-81-7-10] _________________________________________________________________________
seed = None, val = 81
@pytest.mark.parametrize("seed, val", [(None, 81), (0, 54), (_MAX_UNSIGNED_INT, 9)])
def test_newrand_set_seed(seed, val):
"""Test that `set_seed` produces deterministic results"""
if seed is not None:
set_seed_wrap(seed)
x = bounded_rand_int_wrap(100)
> assert x == val, f"Expected {val} but got {x} instead"
E AssertionError: Expected 81 but got 90 instead
E assert 90 == 81
sklearn/svm/tests/test_bounds.py:81: AssertionError
________________________________________________________________________ test_newrand_set_seed[None-81-1-10] _________________________________________________________________________
seed = None, val = 81
@pytest.mark.parametrize("seed, val", [(None, 81), (0, 54), (_MAX_UNSIGNED_INT, 9)])
def test_newrand_set_seed(seed, val):
"""Test that `set_seed` produces deterministic results"""
if seed is not None:
set_seed_wrap(seed)
x = bounded_rand_int_wrap(100)
> assert x == val, f"Expected {val} but got {x} instead"
E AssertionError: Expected 81 but got 83 instead
E assert 83 == 81
sklearn/svm/tests/test_bounds.py:81: AssertionError
________________________________________________________________________ test_newrand_set_seed[None-81-9-10] _________________________________________________________________________
seed = None, val = 81
@pytest.mark.parametrize("seed, val", [(None, 81), (0, 54), (_MAX_UNSIGNED_INT, 9)])
def test_newrand_set_seed(seed, val):
"""Test that `set_seed` produces deterministic results"""
if seed is not None:
set_seed_wrap(seed)
x = bounded_rand_int_wrap(100)
> assert x == val, f"Expected {val} but got {x} instead"
E AssertionError: Expected 81 but got 12 instead
E assert 12 == 81
sklearn/svm/tests/test_bounds.py:81: AssertionError
________________________________________________________________________ test_newrand_set_seed[None-81-8-10] _________________________________________________________________________
seed = None, val = 81
@pytest.mark.parametrize("seed, val", [(None, 81), (0, 54), (_MAX_UNSIGNED_INT, 9)])
def test_newrand_set_seed(seed, val):
"""Test that `set_seed` produces deterministic results"""
if seed is not None:
set_seed_wrap(seed)
x = bounded_rand_int_wrap(100)
> assert x == val, f"Expected {val} but got {x} instead"
E AssertionError: Expected 81 but got 96 instead
E assert 96 == 81
sklearn/svm/tests/test_bounds.py:81: AssertionError
________________________________________________________________________ test_newrand_set_seed[None-81-10-10] ________________________________________________________________________
seed = None, val = 81
@pytest.mark.parametrize("seed, val", [(None, 81), (0, 54), (_MAX_UNSIGNED_INT, 9)])
def test_newrand_set_seed(seed, val):
"""Test that `set_seed` produces deterministic results"""
if seed is not None:
set_seed_wrap(seed)
x = bounded_rand_int_wrap(100)
> assert x == val, f"Expected {val} but got {x} instead"
E AssertionError: Expected 81 but got 91 instead
E assert 91 == 81
sklearn/svm/tests/test_bounds.py:81: AssertionError
________________________________________________________________________ test_newrand_set_seed[None-81-3-10] _________________________________________________________________________
seed = None, val = 81
@pytest.mark.parametrize("seed, val", [(None, 81), (0, 54), (_MAX_UNSIGNED_INT, 9)])
def test_newrand_set_seed(seed, val):
"""Test that `set_seed` produces deterministic results"""
if seed is not None:
set_seed_wrap(seed)
x = bounded_rand_int_wrap(100)
> assert x == val, f"Expected {val} but got {x} instead"
E AssertionError: Expected 81 but got 22 instead
E assert 22 == 81
sklearn/svm/tests/test_bounds.py:81: AssertionError
________________________________________________________________________ test_newrand_set_seed[None-81-6-10] _________________________________________________________________________
seed = None, val = 81
@pytest.mark.parametrize("seed, val", [(None, 81), (0, 54), (_MAX_UNSIGNED_INT, 9)])
def test_newrand_set_seed(seed, val):
"""Test that `set_seed` produces deterministic results"""
if seed is not None:
set_seed_wrap(seed)
x = bounded_rand_int_wrap(100)
> assert x == val, f"Expected {val} but got {x} instead"
E AssertionError: Expected 81 but got 63 instead
E assert 63 == 81
sklearn/svm/tests/test_bounds.py:81: AssertionError
________________________________________________________________________ test_newrand_set_seed[None-81-2-10] _________________________________________________________________________
seed = None, val = 81
@pytest.mark.parametrize("seed, val", [(None, 81), (0, 54), (_MAX_UNSIGNED_INT, 9)])
def test_newrand_set_seed(seed, val):
"""Test that `set_seed` produces deterministic results"""
if seed is not None:
set_seed_wrap(seed)
x = bounded_rand_int_wrap(100)
> assert x == val, f"Expected {val} but got {x} instead"
E AssertionError: Expected 81 but got 30 instead
E assert 30 == 81
sklearn/svm/tests/test_bounds.py:81: AssertionError
==================================================================== 9 failed, 1 passed, 250 deselected in 0.28s =====================================================================I do not understand the purpose of the test though: this test check for a deterministic result without seeding the "newrand.h" RNG. Why doesn't it fail on our CI? Can someone else confirm the failure or lack of failure of this test on their machine?