Skip to content

Commit 8de20b5

Browse files
committed
[TPU/Compile] Fix warmup slowdown from CPython 3.12 frame-stack churn
Issue ----- Warmup compile time depended on Python stack depth: on CPython 3.12, a hot call that lands on a frame-stack chunk boundary allocates and frees memory on every call (python/cpython#142183), so an unrelated extra wrapper frame, such as the one #814 added, could slow precompile. What ---- Run each forward-pass warmup (prefill and decode graphs) inside a reserved frame-stack chunk so its speed no longer depends on the caller's stack depth. Results: Qwen3-30B warmup is 5.7% faster with an unlucky stack layout.
1 parent d09629c commit 8de20b5

3 files changed

Lines changed: 109 additions & 2 deletions

File tree

‎python/sglang/srt/hardware_backend/tpu/compilation/decoder_compile_policy.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,9 @@
3131
_restore_forward_batch_inputs,
3232
build_bucketers,
3333
)
34+
from sglang.srt.hardware_backend.tpu.compilation.frame_stack import (
35+
call_with_reserved_frame_stack,
36+
)
3437
from sglang.srt.hardware_backend.tpu.compilation.pending_call import PendingCall
3538
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
3639
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
@@ -1214,9 +1217,11 @@ def _run_one(
12141217
and self._supports_async_warmup()
12151218
and async_warmup is not None
12161219
):
1217-
out = self.owner.forward.warmup_async(*args, **kwargs)
1220+
out = call_with_reserved_frame_stack(
1221+
self.owner.forward.warmup_async, *args, **kwargs
1222+
)
12181223
else:
1219-
out = entry.forward(*args, **kwargs)
1224+
out = call_with_reserved_frame_stack(entry.forward, *args, **kwargs)
12201225
if isinstance(out, PendingCall):
12211226
return out.then(
12221227
lambda result: _materialize_out(
Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
"""Reduce warmup tracing overhead from CPython 3.12 frame-stack allocation.
2+
3+
Frequent calls at a stack-chunk boundary repeatedly allocate and free memory
4+
(python/cpython#142183). Reserving a larger chunk gives tracing room without
5+
depending on the caller's stack depth.
6+
"""
7+
8+
import sys
9+
from collections.abc import Callable
10+
from typing import ParamSpec, TypeVar
11+
12+
_P = ParamSpec("_P")
13+
_T = TypeVar("_T")
14+
15+
# A 512 KiB frame gets a 1 MiB chunk, leaving room for tracing.
16+
# Nested reservations exceed that remainder and get their own chunk.
17+
_RESERVED_SLOTS = 1 << 16
18+
19+
20+
def call_with_reserved_frame_stack(
21+
fn: Callable[_P, _T], /, *args: _P.args, **kwargs: _P.kwargs
22+
) -> _T:
23+
"""Call ``fn`` with extra frame-stack space on CPython 3.12."""
24+
return fn(*args, **kwargs)
25+
26+
27+
# CPython sizes frames from co_stacksize; limit the workaround to 3.12.
28+
if sys.implementation.name == "cpython" and sys.version_info[:2] == (3, 12):
29+
call_with_reserved_frame_stack.__code__ = (
30+
call_with_reserved_frame_stack.__code__.replace(co_stacksize=_RESERVED_SLOTS)
31+
)
Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
"""Unit tests for the frame-stack reservation used around model tracing."""
2+
3+
import resource
4+
import sys
5+
import unittest
6+
7+
from sglang.srt.hardware_backend.tpu.compilation.frame_stack import (
8+
call_with_reserved_frame_stack,
9+
)
10+
from sglang.test.ci.ci_register import register_cpu_ci
11+
from sglang.test.test_utils import CustomTestCase
12+
13+
# Pure Python, no torch_tpu needed.
14+
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
15+
16+
_CALLS_PER_DEPTH = 2000
17+
# Sweep enough depths to cross several ordinary stack chunks.
18+
_DEPTHS = 300
19+
20+
21+
def _leaf():
22+
return None
23+
24+
25+
# Make the leaf large enough to cross a boundary during the depth sweep.
26+
_leaf.__code__ = _leaf.__code__.replace(co_stacksize=64)
27+
28+
29+
def _call_leaf_at_depth(depth, calls):
30+
if depth:
31+
return _call_leaf_at_depth(depth - 1, calls)
32+
for _ in range(calls):
33+
_leaf()
34+
35+
36+
def _sweep_depths():
37+
for depth in range(_DEPTHS):
38+
_call_leaf_at_depth(depth, _CALLS_PER_DEPTH)
39+
40+
41+
def _minor_faults(fn):
42+
# Repeated chunk allocations show up as minor page faults.
43+
before = resource.getrusage(resource.RUSAGE_THREAD).ru_minflt
44+
fn()
45+
return resource.getrusage(resource.RUSAGE_THREAD).ru_minflt - before
46+
47+
48+
@unittest.skipUnless(
49+
sys.platform == "linux" and sys.version_info[:2] == (3, 12),
50+
"the reservation is applied on CPython 3.12 only; needs per-thread rusage",
51+
)
52+
class TestCallWithReservedFrameStack(CustomTestCase):
53+
def test_no_chunk_churn_at_any_depth(self):
54+
"""A callee must not map and unmap a stack chunk on every call."""
55+
faults = _minor_faults(lambda: call_with_reserved_frame_stack(_sweep_depths))
56+
57+
self.assertLess(faults, _CALLS_PER_DEPTH)
58+
59+
def test_nested_reservation_keeps_its_own_room(self):
60+
"""An inner reservation must not use up the outer one's remainder."""
61+
62+
def nested():
63+
call_with_reserved_frame_stack(_sweep_depths)
64+
65+
faults = _minor_faults(lambda: call_with_reserved_frame_stack(nested))
66+
67+
self.assertLess(faults, _CALLS_PER_DEPTH)
68+
69+
70+
if __name__ == "__main__":
71+
unittest.main()

0 commit comments

Comments
 (0)