Skip to content

Commit 0871f66

Browse files
committed
pythongh-120317: Lock around tokenizer calls in the tokenizer module
1 parent c3b6dbf commit 0871f66

1 file changed

Lines changed: 19 additions & 0 deletions

File tree

‎Python/Python-tokenize.c‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,10 @@
55
#include "../Parser/tokenizer/tokenizer.h"
66
#include "../Parser/pegen.h" // _PyPegen_byte_offset_to_character_offset()
77

8+
#ifdef Py_GIL_DISABLED
9+
#include "internal/pycore_lock.h"
10+
#endif
11+
812
static struct PyModuleDef _tokenizemodule;
913

1014
typedef struct {
@@ -37,6 +41,10 @@ typedef struct
3741
PyObject *last_line;
3842
Py_ssize_t last_lineno;
3943
Py_ssize_t byte_col_offset_diff;
44+
45+
#ifdef Py_GIL_DISABLED
46+
PyMutex mutex;
47+
#endif
4048
} tokenizeriterobject;
4149

4250
/*[clinic input]
@@ -74,6 +82,10 @@ tokenizeriter_new_impl(PyTypeObject *type, PyObject *readline,
7482
}
7583
self->done = 0;
7684

85+
#ifdef Py_GIL_DISABLED
86+
self->mutex = (PyMutex) {0};
87+
#endif
88+
7789
self->last_line = NULL;
7890
self->byte_col_offset_diff = 0;
7991
self->last_lineno = 0;
@@ -182,7 +194,14 @@ tokenizeriter_next(tokenizeriterobject *it)
182194
struct token token;
183195
_PyToken_Init(&token);
184196

197+
#ifdef Py_GIL_DISABLED
198+
PyMutex_Lock(&it->mutex);
199+
#endif
185200
int type = _PyTokenizer_Get(it->tok, &token);
201+
#ifdef Py_GIL_DISABLED
202+
PyMutex_Unlock(&it->mutex);
203+
#endif
204+
186205
if (type == ERRORTOKEN) {
187206
if(!PyErr_Occurred()) {
188207
_tokenizer_error(it->tok);

0 commit comments

Comments
 (0)