Skip to content

Commit 1f660c7

Browse files
miss-islingtonserhiy-storchakaclaude
authored
[3.15] gh-155109: Run tests exhausting the C stack with a limited C stack (GH-155120) (GH-155164)
Add the @support.run_with_limited_c_stack() decorator which runs the test in a thread with a 8 MiB C stack, so that the outcome does not depend on RLIMIT_STACK. Use it in tests which recurse to a fixed depth -- @support.skip_if_huge_c_stack() failed to skip them with a 16 MiB stack. (cherry picked from commit ce5ae29) Co-authored-by: Serhiy Storchaka <[email protected]> Co-authored-by: Claude Opus 5 (1M context) <[email protected]>
1 parent 7681a44 commit 1f660c7

13 files changed

Lines changed: 102 additions & 32 deletions

‎Lib/test/list_tests.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
from functools import cmp_to_key
77

88
from test import seq_tests
9-
from test.support import ALWAYS_EQ, NEVER_EQ, skip_if_huge_c_stack
9+
from test.support import ALWAYS_EQ, NEVER_EQ, run_with_limited_c_stack
1010
from test.support import skip_emscripten_stack_overflow, skip_wasi_stack_overflow
1111

1212

@@ -60,7 +60,7 @@ def test_repr(self):
6060
self.assertEqual(str(a2), "[0, 1, 2, [...], 3]")
6161
self.assertEqual(repr(a2), "[0, 1, 2, [...], 3]")
6262

63-
@skip_if_huge_c_stack(200_000)
63+
@run_with_limited_c_stack(200_000)
6464
@skip_wasi_stack_overflow()
6565
@skip_emscripten_stack_overflow()
6666
def test_repr_deep(self):

‎Lib/test/mapping_tests.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -629,7 +629,7 @@ def __repr__(self):
629629
d = self._full_mapping({1: BadRepr()})
630630
self.assertRaises(Exc, repr, d)
631631

632-
@support.skip_if_huge_c_stack()
632+
@support.run_with_limited_c_stack()
633633
@support.skip_wasi_stack_overflow()
634634
@support.skip_emscripten_stack_overflow()
635635
@support.skip_if_sanitizer("requires deep stack", ub=True)

‎Lib/test/support/__init__.py‎

Lines changed: 80 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646
"check_disallow_instantiation", "check_sanitizer", "skip_if_sanitizer",
4747
"requires_limited_api", "requires_specialization", "thread_unsafe",
4848
"skip_if_unlimited_stack_size", "skip_if_huge_c_stack",
49+
"run_with_limited_c_stack",
4950
# sys
5051
"MS_WINDOWS", "is_jython", "is_android", "is_emscripten", "is_wasi",
5152
"is_apple_mobile", "check_impl_detail", "unix_shell", "setswitchinterval",
@@ -2806,30 +2807,94 @@ def exceeds_recursion_limit():
28062807
return 150_000
28072808

28082809

2810+
def _has_huge_c_stack(depth):
2811+
"""Check that *depth* recursive calls cannot exhaust the C stack."""
2812+
try:
2813+
from _testinternalcapi import get_c_recursion_remaining
2814+
except ImportError:
2815+
# Fall back to checking for an unlimited stack size.
2816+
if is_emscripten or is_wasi or os.name == "nt":
2817+
return False
2818+
import resource
2819+
soft, hard = resource.getrlimit(resource.RLIMIT_STACK)
2820+
return soft == hard and soft in (-1, 0xFFFF_FFFF_FFFF_FFFF)
2821+
else:
2822+
remaining = get_c_recursion_remaining()
2823+
# A negative value means integer overflow in the estimate
2824+
# (e.g. with an unlimited RLIMIT_STACK). The estimate is based on
2825+
# the size of the interpreter loop frame, so it is only a lower
2826+
# bound for recursion with smaller C frames.
2827+
return remaining >= depth or remaining < 0
2828+
2829+
28092830
def skip_if_huge_c_stack(depth=150_000):
28102831
"""Skip decorator for tests which cannot overflow the C stack.
28112832
28122833
Tests exhausting the C stack with *depth* recursive calls cannot
28132834
trigger the recursion protection if the C stack is too large (e.g.
28142835
with a large or unlimited RLIMIT_STACK), and either fail, or run
28152836
for a very long time, or crash, or consume all memory.
2837+
2838+
Prefer run_with_limited_c_stack() for tests recursing to a fixed depth.
28162839
"""
2817-
try:
2818-
from _testinternalcapi import get_c_recursion_remaining
2819-
except ImportError:
2820-
# Fall back to checking for an unlimited stack size.
2821-
huge = False
2822-
if not (is_emscripten or is_wasi) and os.name != "nt":
2823-
import resource
2824-
soft, hard = resource.getrlimit(resource.RLIMIT_STACK)
2825-
huge = soft == hard and soft in (-1, 0xFFFF_FFFF_FFFF_FFFF)
2826-
else:
2827-
remaining = get_c_recursion_remaining()
2828-
# A negative value means integer overflow in the estimate
2829-
# (e.g. with an unlimited RLIMIT_STACK).
2830-
huge = remaining >= depth or remaining < 0
28312840
return unittest.skipIf(
2832-
huge, f"the C stack is large enough for {depth} recursive calls")
2841+
_has_huge_c_stack(depth),
2842+
f"the C stack is large enough for {depth} recursive calls")
2843+
2844+
2845+
# Small enough to be exhausted by tens of thousands of recursive calls,
2846+
# but not smaller than Py_C_STACK_SIZE (4 MiB) which the interpreter
2847+
# assumes if it cannot query the thread stack size.
2848+
C_STACK_SIZE = 8 * 1024 * 1024
2849+
2850+
2851+
def run_with_limited_c_stack(depth=150_000, size=C_STACK_SIZE):
2852+
"""Decorator for tests exhausting the C stack with *depth* recursive calls.
2853+
2854+
Run the test in a separate thread with the C stack of *size* bytes, so
2855+
that the outcome does not depend on the C stack size of the main thread
2856+
(which can be large or unlimited, see RLIMIT_STACK).
2857+
2858+
If a thread with the limited C stack cannot be created, run the test in
2859+
the current thread, but skip it if the C stack is too large.
2860+
"""
2861+
reason = f"the C stack is large enough for {depth} recursive calls"
2862+
def decorator(test):
2863+
@functools.wraps(test)
2864+
def wrapper(*args, **kwargs):
2865+
def run_test():
2866+
# The C stack can still be too large if limiting it failed.
2867+
if _has_huge_c_stack(depth):
2868+
raise unittest.SkipTest(reason)
2869+
test(*args, **kwargs)
2870+
2871+
try:
2872+
import threading
2873+
old_size = threading.stack_size(size)
2874+
except (ImportError, ValueError, RuntimeError):
2875+
# Setting the thread stack size is not supported.
2876+
return run_test()
2877+
2878+
exceptions = []
2879+
def run():
2880+
try:
2881+
run_test()
2882+
except BaseException as exc:
2883+
exceptions.append(exc)
2884+
2885+
thread = threading.Thread(target=run)
2886+
try:
2887+
thread.start()
2888+
except RuntimeError:
2889+
# Threads are not supported.
2890+
return run_test()
2891+
finally:
2892+
threading.stack_size(old_size)
2893+
thread.join()
2894+
if exceptions:
2895+
raise exceptions[0]
2896+
return wrapper
2897+
return decorator
28332898

28342899

28352900
# Windows doesn't have os.uname() but it doesn't support s390x.

‎Lib/test/test_ast/test_ast.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1056,7 +1056,8 @@ def next(self):
10561056
enum._test_simple_enum(_Precedence, _ast_unparse._Precedence)
10571057

10581058
@support.cpython_only
1059-
@support.skip_if_huge_c_stack(100_000 if sys.platform == "android" else 500_000)
1059+
@support.run_with_limited_c_stack(
1060+
100_000 if sys.platform == "android" else 500_000)
10601061
@skip_wasi_stack_overflow()
10611062
@skip_emscripten_stack_overflow()
10621063
def test_ast_recursion_limit(self):

‎Lib/test/test_compile.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -724,7 +724,8 @@ def test_yet_more_evil_still_undecodable(self):
724724

725725
@support.cpython_only
726726
@unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI")
727-
@support.skip_if_huge_c_stack(100_000 if sys.platform == "android" else 500_000)
727+
@support.run_with_limited_c_stack(
728+
100_000 if sys.platform == "android" else 500_000)
728729
@support.skip_emscripten_stack_overflow()
729730
def test_compiler_recursion_limit(self):
730731
# Compiler frames are small

‎Lib/test/test_dict.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -678,7 +678,7 @@ def __repr__(self):
678678
d = {1: BadRepr()}
679679
self.assertRaises(Exc, repr, d)
680680

681-
@support.skip_if_huge_c_stack()
681+
@support.run_with_limited_c_stack()
682682
@support.skip_wasi_stack_overflow()
683683
@support.skip_emscripten_stack_overflow()
684684
def test_repr_deep(self):

‎Lib/test/test_dictviews.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
import pickle
44
import unittest
55
from test.support import (skip_emscripten_stack_overflow,
6-
skip_wasi_stack_overflow, skip_if_huge_c_stack,
6+
skip_wasi_stack_overflow, run_with_limited_c_stack,
77
exceeds_recursion_limit)
88

99
class DictSetTest(unittest.TestCase):
@@ -279,7 +279,7 @@ def test_recursive_repr(self):
279279
# Again.
280280
self.assertIsInstance(r, str)
281281

282-
@skip_if_huge_c_stack()
282+
@run_with_limited_c_stack()
283283
@skip_wasi_stack_overflow()
284284
@skip_emscripten_stack_overflow()
285285
def test_deeply_nested_repr(self):

‎Lib/test/test_exception_group.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
import types
33
import unittest
44
from test.support import (skip_emscripten_stack_overflow,
5-
skip_wasi_stack_overflow, skip_if_huge_c_stack,
5+
skip_wasi_stack_overflow, run_with_limited_c_stack,
66
exceeds_recursion_limit)
77

88
class TestExceptionGroupTypeHierarchy(unittest.TestCase):
@@ -549,15 +549,15 @@ def make_deep_eg(self):
549549
e = ExceptionGroup('eg', [e])
550550
return e
551551

552-
@skip_if_huge_c_stack()
552+
@run_with_limited_c_stack()
553553
@skip_emscripten_stack_overflow()
554554
@skip_wasi_stack_overflow()
555555
def test_deep_split(self):
556556
e = self.make_deep_eg()
557557
with self.assertRaises(RecursionError):
558558
e.split(TypeError)
559559

560-
@skip_if_huge_c_stack()
560+
@run_with_limited_c_stack()
561561
@skip_emscripten_stack_overflow()
562562
@skip_wasi_stack_overflow()
563563
def test_deep_subgroup(self):

‎Lib/test/test_json/test_recursion.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ def default(self, o):
6969

7070

7171
@support.skip_if_pgo_task # fails during PGO training w/ some stack sizes
72-
@support.skip_if_huge_c_stack(500_000)
72+
@support.run_with_limited_c_stack(500_000)
7373
@support.skip_emscripten_stack_overflow()
7474
@support.skip_wasi_stack_overflow()
7575
def test_highly_nested_objects_decoding(self):

‎Lib/test/test_pyexpat.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -855,7 +855,7 @@ def test_trigger_leak(self):
855855
parser.ElementDeclHandler = lambda _1, _2: None
856856
self.assertRaises(TypeError, parser.Parse, data, True)
857857

858-
@support.skip_if_huge_c_stack(800_000)
858+
@support.run_with_limited_c_stack(800_000)
859859
@support.skip_emscripten_stack_overflow()
860860
@support.skip_wasi_stack_overflow()
861861
def test_deeply_nested_content_model(self):

0 commit comments

Comments
 (0)