Skip to content

Commit 70b3c2d

Browse files
committed
pythongh-146527, pythongh-155811: Backport out-of-process GC statistics to 3.12
Add _remote_debugging.get_gc_stats and get_child_pids, and record per-collection garbage collector statistics into per-generation ring buffers discoverable through a cookie-tagged data section. An external process can read exact GC pause times, candidate counts and heap sizes from a running interpreter without pausing it or loading anything into it. gc.get_stats() gains two per-generation keys, candidates and duration, alongside the existing ones. The addition is backwards compatible for code that reads keys by name. A seqcount (pythongh-155811) guards the out-of-process read against torn snapshots: the writer bumps gc_stats.update_seq odd before an update and even after (with a seq_cst fence and a release store), and the reader brackets its whole-struct read with the counter, retrying once and then failing cleanly rather than returning a half-updated record. 3.12 is GIL-only, so a single writer updates the naturally aligned counter. This is the canonical form of the ts_stop-last publish ordering the earlier revision hand-rolled, so that trick and its zero-duration clamp are gone. update_seq is appended to the heap-allocated gc_stats, so no interpreter struct offset moves (abi-check IDENTICAL). Backported from 3.15.
1 parent 3bb231a commit 70b3c2d

11 files changed

Lines changed: 3270 additions & 40 deletions

File tree

‎Include/internal/pycore_gc.h‎

Lines changed: 51 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -144,14 +144,64 @@ struct gc_generation {
144144
generations */
145145
};
146146

147+
/* Stock 3.12's three counters. The generation_counters[] array below keeps
148+
this type and therefore its original 24 bytes at its original offset, so
149+
that _gc_runtime_state's layout is byte-identical to an unpatched build.
150+
It is still maintained, so an extension compiled against stock headers
151+
that reads it keeps reading correct values. */
152+
struct gc_generation_counters {
153+
/* total number of collections */
154+
Py_ssize_t collections;
155+
/* total number of collected objects */
156+
Py_ssize_t collected;
157+
/* total number of uncollectable objects (put into gc.garbage) */
158+
Py_ssize_t uncollectable;
159+
};
160+
147161
/* Running stats per generation */
148162
struct gc_generation_stats {
163+
_PyTime_t ts_start;
164+
_PyTime_t ts_stop;
149165
/* total number of collections */
150166
Py_ssize_t collections;
151167
/* total number of collected objects */
152168
Py_ssize_t collected;
153169
/* total number of uncollectable objects (put into gc.garbage) */
154170
Py_ssize_t uncollectable;
171+
// Total number of objects considered for collection and traversed:
172+
Py_ssize_t candidates;
173+
// Total duration of the collection in seconds:
174+
double duration;
175+
/* heap_size on the start of the collection */
176+
Py_ssize_t heap_size;
177+
};
178+
179+
#ifdef Py_GIL_DISABLED
180+
#define GC_YOUNG_STATS_SIZE 1
181+
#define GC_OLD_STATS_SIZE 1
182+
#else
183+
#define GC_YOUNG_STATS_SIZE 11
184+
#define GC_OLD_STATS_SIZE 3
185+
#endif
186+
struct gc_young_stats_buffer {
187+
struct gc_generation_stats items[GC_YOUNG_STATS_SIZE];
188+
int8_t index;
189+
};
190+
191+
struct gc_old_stats_buffer {
192+
struct gc_generation_stats items[GC_OLD_STATS_SIZE];
193+
int8_t index;
194+
};
195+
196+
/* Ring buffers of per-collection statistics, published to external readers.
197+
Upstream 3.15 keeps a struct gc_stats* here in _gc_runtime_state; this tree
198+
heap-allocates one too, but hangs the pointer off the tail of
199+
PyInterpreterState instead -- see the Deviations note in
200+
patches/gc-stats.patch for why _gc_runtime_state cannot carry it. */
201+
struct gc_stats {
202+
struct gc_young_stats_buffer young;
203+
struct gc_old_stats_buffer old[2];
204+
uint32_t update_seq;
155205
};
156206

157207
struct _gc_runtime_state {
@@ -169,7 +219,7 @@ struct _gc_runtime_state {
169219
PyGC_Head *generation0;
170220
/* a permanent generation which won't be collected */
171221
struct gc_generation permanent_generation;
172-
struct gc_generation_stats generation_stats[NUM_GENERATIONS];
222+
struct gc_generation_counters generation_counters[NUM_GENERATIONS];
173223
/* true if we are currently running the collector */
174224
int collecting;
175225
/* list of uncollectable objects */

‎Include/internal/pycore_gc_stats.h‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
#ifndef Py_INTERNAL_GC_STATS_H
2+
#define Py_INTERNAL_GC_STATS_H
3+
#ifdef __cplusplus
4+
extern "C" {
5+
#endif
6+
7+
#ifndef Py_BUILD_CORE
8+
# error "this header requires Py_BUILD_CORE define"
9+
#endif
10+
11+
#include <stdint.h>
12+
13+
/* Out-of-process discovery anchor for the GC statistics rings.
14+
*
15+
* The rings themselves live at the tail of PyInterpreterState (see struct
16+
* gc_stats in pycore_gc.h), mirroring upstream 3.15's gcmon. Upstream and the
17+
* 3.13/3.14 backports let an external reader reach them through
18+
* _Py_DebugOffsets, published at the front of _PyRuntime. 3.12 has neither
19+
* _Py_DebugOffsets nor a cookie'd _PyRuntime section, and adding
20+
* _Py_DebugOffsets to the front of _PyRuntimeState would move
21+
* gilstate/interpreters/_finalizing and break the ABI this patch is required
22+
* to preserve.
23+
*
24+
* So this branch publishes one self-describing anchor in its own section
25+
* instead: a cookie and version, the &_PyRuntime address, and the handful of
26+
* field offsets a reader needs to walk the interpreter list and find each
27+
* interpreter's ring. A new global in its own section changes no struct
28+
* layout, so it is ABI-safe. This is the single place the 3.12 backport
29+
* deviates from the 3.13 reader design; everything a consumer sees (the ring
30+
* shape and the GCStatsInfo record) is identical.
31+
*/
32+
33+
/* Bumped if the anchor changes shape. Distinct from the 1/2 the retired
34+
section-registry design used, so a stale reader rejects this format rather
35+
than misreading it. */
36+
#define _Py_GCStats_VERSION 3
37+
38+
/* Found by scanning the target's mappings for a section with this name, then
39+
validating this cookie at its start. Mirrors how _PyRuntime is located on
40+
3.14+. */
41+
#define _Py_GCStats_SECTION "gcmonstat"
42+
#define _Py_GCStats_COOKIE "gcmonstat\0\0\0\0\0\0"
43+
44+
/* Self-describing: a reader validates cookie and version, then trusts these
45+
fields rather than its own headers. runtime is (uintptr_t)&_PyRuntime; the
46+
offsets are offsetof() into _PyRuntimeState and PyInterpreterState, so the
47+
reader can walk the live interpreter list and read each ring without any
48+
layout knowledge of its own. */
49+
struct _Py_GCStatsAnchor {
50+
char cookie[16];
51+
uint64_t version;
52+
uint64_t flags; /* bit0 = free-threaded; always 0 on 3.12 */
53+
uint64_t gc_stats_size; /* sizeof(struct gc_stats) */
54+
uint64_t young_slots; /* GC_YOUNG_STATS_SIZE */
55+
uint64_t old_slots; /* GC_OLD_STATS_SIZE */
56+
uint64_t interpreters_head; /* offsetof(_PyRuntimeState, interpreters.head) */
57+
uint64_t interpreter_id; /* offsetof(PyInterpreterState, id) */
58+
uint64_t interpreter_next; /* offsetof(PyInterpreterState, next) */
59+
uint64_t generation_stats; /* offsetof(PyInterpreterState, generation_stats) */
60+
void *runtime; /* &_PyRuntime */
61+
};
62+
63+
PyAPI_DATA(struct _Py_GCStatsAnchor) _Py_gc_stats_anchor;
64+
65+
#ifdef __cplusplus
66+
}
67+
#endif
68+
#endif /* !Py_INTERNAL_GC_STATS_H */

‎Include/internal/pycore_interp.h‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -230,6 +230,16 @@ struct _is {
230230

231231
/* the initial PyInterpreterState.threads.head */
232232
PyThreadState _initial_thread;
233+
234+
/* Heap-allocated ring of per-collection GC statistics, published to
235+
external readers. Upstream 3.15 keeps this pointer in _gc_runtime_state
236+
(interp->gc); this tree cannot, because _gc_runtime_state sits mid-struct
237+
with many members after it, so growing it would move every one of their
238+
offsets. Appended at the very tail of PyInterpreterState instead, after
239+
_initial_thread (the last stock member), which grows
240+
sizeof(PyInterpreterState) by one pointer and moves nothing.
241+
See the Deviations note in patches/gc-stats.patch. */
242+
struct gc_stats *generation_stats;
233243
};
234244

235245

‎Lib/test/test_gc.py‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -787,11 +787,15 @@ def test_get_stats(self):
787787
self.assertEqual(len(stats), 3)
788788
for st in stats:
789789
self.assertIsInstance(st, dict)
790-
self.assertEqual(set(st),
791-
{"collected", "collections", "uncollectable"})
790+
self.assertEqual(
791+
set(st),
792+
{"collected", "collections", "uncollectable", "candidates", "duration"}
793+
)
792794
self.assertGreaterEqual(st["collected"], 0)
793795
self.assertGreaterEqual(st["collections"], 0)
794796
self.assertGreaterEqual(st["uncollectable"], 0)
797+
self.assertGreaterEqual(st["candidates"], 0)
798+
self.assertGreaterEqual(st["duration"], 0)
795799
# Check that collection counts are incremented correctly
796800
if gc.isenabled():
797801
self.addCleanup(gc.enable)
@@ -802,11 +806,25 @@ def test_get_stats(self):
802806
self.assertEqual(new[0]["collections"], old[0]["collections"] + 1)
803807
self.assertEqual(new[1]["collections"], old[1]["collections"])
804808
self.assertEqual(new[2]["collections"], old[2]["collections"])
809+
self.assertGreater(new[0]["duration"], old[0]["duration"])
810+
self.assertEqual(new[1]["duration"], old[1]["duration"])
811+
self.assertEqual(new[2]["duration"], old[2]["duration"])
812+
for stat in ["collected", "uncollectable", "candidates"]:
813+
self.assertGreaterEqual(new[0][stat], old[0][stat])
814+
self.assertEqual(new[1][stat], old[1][stat])
815+
self.assertEqual(new[2][stat], old[2][stat])
805816
gc.collect(2)
806-
new = gc.get_stats()
807-
self.assertEqual(new[0]["collections"], old[0]["collections"] + 1)
817+
old, new = new, gc.get_stats()
818+
self.assertEqual(new[0]["collections"], old[0]["collections"])
808819
self.assertEqual(new[1]["collections"], old[1]["collections"])
809820
self.assertEqual(new[2]["collections"], old[2]["collections"] + 1)
821+
self.assertEqual(new[0]["duration"], old[0]["duration"])
822+
self.assertEqual(new[1]["duration"], old[1]["duration"])
823+
self.assertGreater(new[2]["duration"], old[2]["duration"])
824+
for stat in ["collected", "uncollectable", "candidates"]:
825+
self.assertEqual(new[0][stat], old[0][stat])
826+
self.assertEqual(new[1][stat], old[1][stat])
827+
self.assertGreaterEqual(new[2][stat], old[2][stat])
810828

811829
def test_freeze(self):
812830
gc.freeze()

0 commit comments

Comments
 (0)