Skip to content

Commit 632ad06

Browse files
[None][perf] Run the executor loop under a CPython frame-stack reserve
CPython 3.11+ keeps frames in per-thread data-stack chunks. When a hot call sits at a chunk boundary, every call maps a fresh chunk and every return unmaps it (gh-142183). On GB200 each munmap also pays a Grace SMMU invalidation once NCCL has peer-mapped memory, so the churn costs milliseconds per executor step: MiniMax-M3 piecewise CUDA graph prefill steps spent about half their host time in munmap, all on the executor thread. CPython fixed this in 3.13.13 and 3.14.4; 3.12 will not get the fix. Add run_with_frame_stack_reserve() and call PyExecutor.event_loop through it, so the loop starts near the beginning of a large data-stack chunk and ordinary call depths never reach a boundary. The reserve is applied only on affected CPython versions and can be disabled with TLLM_DISABLE_FRAME_STACK_RESERVE=1. Cover the version gate, the disable switch, the churn itself and the executor wiring with cpu_only unit tests. Background: - python/cpython#142183 (the issue) - python/cpython#145789 (the fix; backports #145829 / #146123 for 3.13, #145828 for 3.14) - python/cpython#88756 (3.11 change that moved frames onto the data stack) - sgl-project/sglang-torchtpu#1037 (same technique) Signed-off-by: Dan Hansen <[email protected]>
1 parent 08b8c22 commit 632ad06

5 files changed

Lines changed: 262 additions & 1 deletion

File tree

‎tensorrt_llm/_torch/pyexecutor/py_executor.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
get_steady_clock_now_in_seconds,
3333
global_mpi_size, is_trace_enabled, mpi_comm,
3434
mpi_disabled, nvtx_range,
35+
run_with_frame_stack_reserve,
3536
set_thread_local_mpi_comm, trace_func)
3637
# yapf: disable
3738
from tensorrt_llm.bindings.executor import (DisServingRequestStats,
@@ -1972,7 +1973,7 @@ def _event_loop_wrapper(self) -> None:
19721973
"TLLM_LINE_PROFILER_PATH")) and not self.is_warmup
19731974
with host_profiler_context(enable=enable_profiler), \
19741975
customized_gc_thresholds(self.garbage_collection_gen0_threshold):
1975-
self.event_loop()
1976+
run_with_frame_stack_reserve(self.event_loop)
19761977
except Exception as e:
19771978
# A raise AFTER the loop reached its own normal-exit `break` (from
19781979
# the profiler's or hang detector's __exit__, or from the enclosing

‎tensorrt_llm/_utils.py‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1422,6 +1422,72 @@ def wrapper(*args: P.args, **kwargs: P.kwargs) -> None:
14221422
return wrapper
14231423

14241424

1425+
# 64Ki slots (512 KiB) make the reserving frame land in a fresh 1 MiB chunk,
1426+
# leaving about 512 KiB for the frames it calls.
1427+
_FRAME_STACK_RESERVE_SLOTS = 1 << 16
1428+
_FRAME_STACK_RESERVE_DISABLE_ENV = "TLLM_DISABLE_FRAME_STACK_RESERVE"
1429+
1430+
1431+
def run_with_frame_stack_reserve(fn: Callable[P, _T], /, *args: P.args,
1432+
**kwargs: P.kwargs) -> _T:
1433+
"""Call ``fn`` from a frame that reserves a large CPython data-stack chunk.
1434+
1435+
CPython allocates frames in per-thread data-stack chunks. On versions that
1436+
release an emptied chunk immediately (CPython gh-142183), a frequently
1437+
called function whose frame lands on a chunk boundary maps and unmaps
1438+
memory on every call. On affected interpreters, this function's
1439+
``co_stacksize`` is raised to ``_FRAME_STACK_RESERVE_SLOTS`` at import, so
1440+
its frame alone needs a new, large chunk, and a long-running loop called
1441+
through it starts near the beginning of that chunk. Ordinary call depths
1442+
below it then never reach a boundary. The reserved slots are never
1443+
written, so they cost little resident memory, although a frame object
1444+
built for this frame (for a traceback or a profiler) allocates them.
1445+
1446+
Wrap a thread's long-running entry point once; every call maps and later
1447+
unmaps one chunk of its own. On interpreters that cache an emptied chunk,
1448+
and when ``TLLM_DISABLE_FRAME_STACK_RESERVE`` is set to ``1`` at import
1449+
time, this is a plain call.
1450+
1451+
Args:
1452+
fn: The callable to run.
1453+
*args: Positional arguments forwarded to ``fn``.
1454+
**kwargs: Keyword arguments forwarded to ``fn``.
1455+
1456+
Returns:
1457+
Whatever ``fn`` returns. Exceptions from ``fn`` propagate unchanged.
1458+
"""
1459+
return fn(*args, **kwargs)
1460+
1461+
1462+
def _frame_stack_reserve_needed(implementation: str,
1463+
version: Sequence[int]) -> bool:
1464+
"""Whether the given interpreter frees an emptied data-stack chunk.
1465+
1466+
Args:
1467+
implementation: ``sys.implementation.name``.
1468+
version: ``sys.version_info`` or a ``(major, minor, micro)`` tuple.
1469+
"""
1470+
if implementation != "cpython":
1471+
return False
1472+
# 3.11 introduced chunked data stacks; 3.13.13 and 3.14.4 keep one
1473+
# emptied chunk per thread for reuse.
1474+
version = tuple(version[:3])
1475+
return ((3, 11) <= version < (3, 13, 13) or (3, 14) <= version < (3, 14, 4))
1476+
1477+
1478+
def _needs_frame_stack_reserve() -> bool:
1479+
if os.environ.get(_FRAME_STACK_RESERVE_DISABLE_ENV) == "1":
1480+
return False
1481+
return _frame_stack_reserve_needed(sys.implementation.name,
1482+
sys.version_info)
1483+
1484+
1485+
if _needs_frame_stack_reserve():
1486+
# Enlarge the frame only; the bytecode and everything else are unchanged.
1487+
run_with_frame_stack_reserve.__code__ = (
1488+
run_with_frame_stack_reserve.__code__.replace(
1489+
co_stacksize=_FRAME_STACK_RESERVE_SLOTS))
1490+
14251491
TORCH_PYBIND11_ABI = None
14261492

14271493

‎tests/integration/test_lists/test-db/l0_cpu.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -436,4 +436,5 @@ l0_cpu:
436436
- unittest/utils/test_logger.py
437437
- unittest/utils/test_confidential_compute.py
438438
- unittest/utils/test_llm_data.py
439+
- unittest/utils/test_frame_stack_reserve.py
439440
- unittest/visual_gen/test_output.py
Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
"""PyExecutor._event_loop_wrapper runs the event loop under the frame-stack reserve (no GPU)."""
16+
17+
import contextlib
18+
import sys
19+
import threading
20+
import types
21+
22+
import pytest
23+
24+
from tensorrt_llm import _utils
25+
from tensorrt_llm._torch.pyexecutor import py_executor as pe
26+
27+
pytestmark = pytest.mark.cpu_only
28+
29+
30+
def test_event_loop_runs_directly_under_the_reserve_frame(monkeypatch):
31+
# Same bypass-__init__ pattern as test_hang_detector_kill.py: only the
32+
# attributes _event_loop_wrapper reads are set.
33+
monkeypatch.setattr(pe, "host_profiler_context", lambda enable: contextlib.nullcontext())
34+
monkeypatch.setattr(pe, "customized_gc_thresholds", lambda threshold: contextlib.nullcontext())
35+
ex = pe.PyExecutor.__new__(pe.PyExecutor)
36+
ex.dist = types.SimpleNamespace(world_size=1)
37+
ex.garbage_collection_gen0_threshold = None
38+
ex.is_shutdown = False
39+
ex._event_loop_completed = False
40+
ex._event_loop_error_delivered = threading.Event()
41+
ex._executor_loop_cleanup = lambda: None
42+
callers = []
43+
ex.event_loop = lambda: callers.append(sys._getframe(1).f_code)
44+
45+
ex._event_loop_wrapper()
46+
47+
# The real reserve function, with its rebound code object, is the loop's caller.
48+
assert callers == [_utils.run_with_frame_stack_reserve.__code__]
Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
"""run_with_frame_stack_reserve: call semantics, version gate and data-stack churn (no GPU)."""
16+
17+
import os
18+
import subprocess
19+
import sys
20+
import threading
21+
22+
import pytest
23+
24+
from tensorrt_llm._utils import (
25+
_FRAME_STACK_RESERVE_DISABLE_ENV,
26+
_FRAME_STACK_RESERVE_SLOTS,
27+
_frame_stack_reserve_needed,
28+
_needs_frame_stack_reserve,
29+
run_with_frame_stack_reserve,
30+
)
31+
32+
pytestmark = pytest.mark.cpu_only
33+
34+
_CALLS_PER_DEPTH = 2000
35+
# Enough depths to cross several ordinary data-stack chunks.
36+
_DEPTHS = 300
37+
38+
39+
def _leaf():
40+
return None
41+
42+
43+
# A larger leaf frame makes it cross a chunk boundary during the sweep.
44+
_leaf.__code__ = _leaf.__code__.replace(co_stacksize=64)
45+
46+
47+
def _call_leaf_at_depth(depth, calls):
48+
if depth:
49+
return _call_leaf_at_depth(depth - 1, calls)
50+
for _ in range(calls):
51+
_leaf()
52+
53+
54+
def _sweep_depths():
55+
for depth in range(_DEPTHS):
56+
_call_leaf_at_depth(depth, _CALLS_PER_DEPTH)
57+
58+
59+
def _minor_faults_in_new_thread(fn):
60+
# Each freshly mapped data-stack chunk shows up as a minor page fault.
61+
resource = pytest.importorskip("resource")
62+
result = {}
63+
64+
def target():
65+
before = resource.getrusage(resource.RUSAGE_THREAD).ru_minflt
66+
fn()
67+
result["faults"] = resource.getrusage(resource.RUSAGE_THREAD).ru_minflt - before
68+
69+
thread = threading.Thread(target=target)
70+
thread.start()
71+
thread.join()
72+
return result["faults"]
73+
74+
75+
def test_forwards_arguments_and_result():
76+
def echo(*args, **kwargs):
77+
return args, kwargs
78+
79+
# ``fn`` is positional-only, so a keyword argument of the same name reaches the callee.
80+
assert run_with_frame_stack_reserve(echo, 1, 2, fn="x") == ((1, 2), {"fn": "x"})
81+
82+
83+
def test_exception_propagates_unchanged():
84+
error = ValueError("boom")
85+
86+
def crash():
87+
raise error
88+
89+
with pytest.raises(ValueError) as excinfo:
90+
run_with_frame_stack_reserve(crash)
91+
assert excinfo.value is error
92+
assert excinfo.value.__context__ is None
93+
94+
95+
@pytest.mark.parametrize(
96+
"implementation, version, expected",
97+
[
98+
("cpython", (3, 10, 16), False),
99+
("cpython", (3, 11, 0), True),
100+
("cpython", (3, 12, 3, "final", 0), True),
101+
("cpython", (3, 13, 12), True),
102+
("cpython", (3, 13, 13), False),
103+
("cpython", (3, 14, 3), True),
104+
("cpython", (3, 14, 4), False),
105+
("cpython", (3, 15, 0), False),
106+
("pypy", (3, 12, 3), False),
107+
],
108+
)
109+
def test_gate(implementation, version, expected):
110+
assert _frame_stack_reserve_needed(implementation, version) is expected
111+
112+
113+
def test_reserve_applied_at_import_iff_gate_holds():
114+
reserved = run_with_frame_stack_reserve.__code__.co_stacksize >= _FRAME_STACK_RESERVE_SLOTS
115+
assert reserved is _needs_frame_stack_reserve()
116+
117+
118+
def test_env_var_disables_reserve_at_import():
119+
env = dict(os.environ, **{_FRAME_STACK_RESERVE_DISABLE_ENV: "1"})
120+
code = (
121+
"from tensorrt_llm import _utils as u\n"
122+
"reserved = u.run_with_frame_stack_reserve.__code__.co_stacksize"
123+
" >= u._FRAME_STACK_RESERVE_SLOTS\n"
124+
"print('RESULT', int(reserved))\n"
125+
)
126+
proc = subprocess.run(
127+
[sys.executable, "-c", code],
128+
env=env,
129+
capture_output=True,
130+
text=True,
131+
timeout=600,
132+
check=True,
133+
)
134+
assert proc.stdout.strip().splitlines()[-1] == "RESULT 0"
135+
136+
137+
@pytest.mark.skipif(
138+
sys.platform != "linux" or not _needs_frame_stack_reserve(),
139+
reason="needs an affected CPython and per-thread rusage",
140+
)
141+
def test_reserve_removes_chunk_churn():
142+
# Without the reserve, the sweep must hit a chunk boundary, or the second check proves nothing.
143+
assert _minor_faults_in_new_thread(_sweep_depths) > _CALLS_PER_DEPTH
144+
faults = _minor_faults_in_new_thread(lambda: run_with_frame_stack_reserve(_sweep_depths))
145+
assert faults < _CALLS_PER_DEPTH

0 commit comments

Comments
 (0)