Skip to content

Commit dfd69e1

Browse files
committed
pythongh-160267: Follow interpreter list changes in the sampling profiler
When sampling all threads, re-read interpreters.head in the batched interpreter state read every 1024 samples and when the head interpreter is replaced. Make sys.remote_exec() rewrite only the event byte of eval_breaker so that it cannot restore a stale instrumentation version.
1 parent 868c555 commit dfd69e1

4 files changed

Lines changed: 90 additions & 28 deletions

File tree

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
11
Fix a crash when :mod:`profiling.sampling` profiles a process that creates and
2-
destroys threads quickly. Memory of deleted thread and interpreter states is
3-
now reused for new states instead of being freed.
2+
destroys threads or subinterpreters quickly. Memory of deleted thread and
3+
interpreter states is now reused for new states instead of being freed until
4+
the runtime is finalized. When sampling all threads, the profiler now also
5+
follows subinterpreters created or destroyed after it attached. Fix a possible
6+
hang of the target when :func:`sys.remote_exec` races with
7+
:mod:`sys.monitoring`.

‎Modules/_remote_debugging/_remote_debugging.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -373,6 +373,8 @@ typedef struct {
373373
int cache_frames;
374374
int collect_stats; // whether to collect statistics
375375
uint32_t stale_invalidation_counter; // counter for throttling frame_cache_invalidate_stale
376+
uint32_t head_refresh_counter; // counter for throttling interpreters.head reads
377+
int64_t head_interpreter_id; // id of the interpreter at interpreter_addr
376378
// L1 single-entry shortcut over cached_tstates[]: most workloads sample one
377379
// interpreter, so check these pairs before hashing into the table below.
378380
uintptr_t cached_tstate_interpreter_addr;

‎Modules/_remote_debugging/module.c‎

Lines changed: 72 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -419,6 +419,7 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self,
419419
self->async_debug_offsets_available = 0;
420420
}
421421

422+
self->head_interpreter_id = -1;
422423
if (populate_initial_state_data(all_threads, self, self->runtime_start_address,
423424
&self->interpreter_addr ,&self->tstate_addr) < 0)
424425
{
@@ -597,27 +598,57 @@ refresh_generation_caches_for_interpreter(
597598
return 0;
598599
}
599600

601+
// If read_head is set, *interpreter_addr is the list head from an earlier
602+
// sample. The head is read again in the same batch, and the interpreter state
603+
// is reread if it moved.
600604
static int
601605
read_interp_state_and_maybe_thread_frame(
602606
RemoteUnwinderObject *unwinder,
603-
uintptr_t interpreter_addr,
607+
uintptr_t *interpreter_addr,
608+
int read_head,
604609
char *interp_state_buffer,
605610
char *tstate_buffer,
606611
char *frame_buffer,
607612
RemoteReadPrefetch *prefetch)
608613
{
609614
prefetch->tstate = NULL;
610615
prefetch->frame = NULL;
616+
uintptr_t head = *interpreter_addr;
617+
size_t tstate_size = (size_t)unwinder->debug_offsets.thread_state.size;
618+
_Py_RemoteReadSegment segments[4] = {
619+
{unwinder->runtime_start_address
620+
+ (uintptr_t)unwinder->debug_offsets.runtime_state.interpreters_head,
621+
&head, sizeof(head)},
622+
{*interpreter_addr, interp_state_buffer, INTERP_STATE_BUFFER_SIZE},
623+
{prefetch->tstate_addr, tstate_buffer, tstate_size},
624+
{prefetch->frame_addr, frame_buffer, SIZEOF_INTERP_FRAME},
625+
};
626+
int nsegs = prefetch->tstate_addr == 0 ? 1 : prefetch->frame_addr == 0 ? 2 : 3;
627+
Py_ssize_t nread = -1;
628+
if (read_head || nsegs > 1) {
629+
nread = _Py_RemoteDebug_BatchedReadRemoteMemory(
630+
&unwinder->handle, read_head ? segments : segments + 1,
631+
nsegs + read_head);
632+
}
633+
if (read_head) {
634+
if (nread >= (Py_ssize_t)sizeof(head)) {
635+
nread -= (Py_ssize_t)sizeof(head);
636+
}
637+
else if (_Py_RemoteDebug_ReadRemoteMemory(
638+
&unwinder->handle, segments[0].remote_addr,
639+
sizeof(head), &head) < 0) {
640+
return -1;
641+
}
642+
if (head == 0) {
643+
PyErr_SetString(PyExc_RuntimeError, "No interpreter state found");
644+
return -1;
645+
}
646+
if (head != *interpreter_addr) {
647+
*interpreter_addr = head;
648+
nread = -1;
649+
}
650+
}
611651
if (prefetch->tstate_addr != 0) {
612-
size_t tstate_size = (size_t)unwinder->debug_offsets.thread_state.size;
613-
_Py_RemoteReadSegment segments[3] = {
614-
{interpreter_addr, interp_state_buffer, INTERP_STATE_BUFFER_SIZE},
615-
{prefetch->tstate_addr, tstate_buffer, tstate_size},
616-
{prefetch->frame_addr, frame_buffer, SIZEOF_INTERP_FRAME},
617-
};
618-
int nsegs = prefetch->frame_addr != 0 ? 3 : 2;
619-
Py_ssize_t nread = _Py_RemoteDebug_BatchedReadRemoteMemory(
620-
&unwinder->handle, segments, nsegs);
621652
int completed = 0;
622653
if (nread >= (Py_ssize_t)INTERP_STATE_BUFFER_SIZE) {
623654
completed = 1;
@@ -642,9 +673,12 @@ read_interp_state_and_maybe_thread_frame(
642673
return 0;
643674
}
644675
}
676+
else if (nread >= (Py_ssize_t)INTERP_STATE_BUFFER_SIZE) {
677+
return 0;
678+
}
645679
return _Py_RemoteDebug_ReadRemoteMemory(
646680
&unwinder->handle,
647-
interpreter_addr,
681+
*interpreter_addr,
648682
INTERP_STATE_BUFFER_SIZE,
649683
interp_state_buffer);
650684
}
@@ -733,17 +767,41 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self
733767
}
734768
}
735769

736-
if (read_interp_state_and_maybe_thread_frame(
770+
// When sampling all threads, follow interpreters created and destroyed
771+
// since attaching: re-read the list head now and then, and right away
772+
// if the head interpreter was replaced.
773+
int follow_head = self->tstate_addr == 0
774+
&& current_interpreter == self->interpreter_addr;
775+
int read_head = follow_head
776+
&& ++self->head_refresh_counter % 1024 == 0;
777+
int rc = read_interp_state_and_maybe_thread_frame(
737778
self,
738-
current_interpreter,
779+
&current_interpreter,
780+
read_head,
739781
interp_state_buffer,
740782
prefetched_tstate,
741783
prefetched_frame,
742-
&prefetch) < 0) {
784+
&prefetch);
785+
if (rc == 0 && follow_head && !read_head
786+
&& GET_MEMBER(int64_t, interp_state_buffer,
787+
self->debug_offsets.interpreter_state.id)
788+
!= self->head_interpreter_id)
789+
{
790+
prefetch = (RemoteReadPrefetch){0};
791+
rc = read_interp_state_and_maybe_thread_frame(
792+
self, &current_interpreter, 1, interp_state_buffer,
793+
prefetched_tstate, prefetched_frame, &prefetch);
794+
}
795+
if (rc < 0) {
743796
set_exception_cause(self, PyExc_RuntimeError, "Failed to read interpreter state buffer");
744797
Py_CLEAR(result);
745798
goto exit;
746799
}
800+
if (follow_head) {
801+
self->interpreter_addr = current_interpreter;
802+
self->head_interpreter_id = GET_MEMBER(int64_t, interp_state_buffer,
803+
self->debug_offsets.interpreter_state.id);
804+
}
747805
refresh_generation_caches_from_interp_state(self, current_interpreter, interp_state_buffer);
748806

749807
uintptr_t gc_frame = 0;

‎Python/remote_debugging.c‎

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -250,23 +250,21 @@ send_exec_to_proc_handle(proc_handle_t *handle, int tid, const char *debugger_sc
250250
return -1;
251251
}
252252

253-
uintptr_t eval_breaker;
254-
if (0 != read_memory(
255-
handle,
256-
thread_state_addr + (uintptr_t)debug_offsets.debugger_support.eval_breaker,
257-
sizeof(uintptr_t),
258-
&eval_breaker))
259-
{
253+
// Only rewrite the byte holding the event bits: the rest of eval_breaker
254+
// is the instrumentation version, which the target can change meanwhile.
255+
uintptr_t eval_breaker_addr = thread_state_addr +
256+
(uintptr_t)debug_offsets.debugger_support.eval_breaker;
257+
#if PY_BIG_ENDIAN
258+
eval_breaker_addr += sizeof(uintptr_t) - 1;
259+
#endif
260+
uint8_t eval_breaker;
261+
if (0 != read_memory(handle, eval_breaker_addr, 1, &eval_breaker)) {
260262
return -1;
261263
}
262264

263265
eval_breaker |= _PY_EVAL_PLEASE_STOP_BIT;
264266

265-
if (0 != write_memory(
266-
handle,
267-
thread_state_addr + (uintptr_t)debug_offsets.debugger_support.eval_breaker,
268-
sizeof(uintptr_t),
269-
&eval_breaker))
267+
if (0 != write_memory(handle, eval_breaker_addr, 1, &eval_breaker))
270268

271269
{
272270
return -1;

0 commit comments

Comments
 (0)