Skip to content

Commit 74172a7

Browse files
committed
pythongh-120317: Lock around tokenizer calls in the tokenizer module
1 parent c3b6dbf commit 74172a7

1 file changed

Lines changed: 15 additions & 0 deletions

File tree

‎Python/Python-tokenize.c‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,10 @@
55
#include "../Parser/tokenizer/tokenizer.h"
66
#include "../Parser/pegen.h" // _PyPegen_byte_offset_to_character_offset()
77

8+
#ifdef Py_GIL_DISABLED
9+
#include "internal/pycore_lock.h"
10+
#endif
11+
812
static struct PyModuleDef _tokenizemodule;
913

1014
typedef struct {
@@ -37,6 +41,10 @@ typedef struct
3741
PyObject *last_line;
3842
Py_ssize_t last_lineno;
3943
Py_ssize_t byte_col_offset_diff;
44+
45+
#ifdef Py_GIL_DISABLED
46+
PyMutex mutex;
47+
#endif
4048
} tokenizeriterobject;
4149

4250
/*[clinic input]
@@ -182,7 +190,14 @@ tokenizeriter_next(tokenizeriterobject *it)
182190
struct token token;
183191
_PyToken_Init(&token);
184192

193+
#ifdef Py_GIL_DISABLED
194+
PyMutex_Lock(&it->mutex);
195+
#endif
185196
int type = _PyTokenizer_Get(it->tok, &token);
197+
#ifdef Py_GIL_DISABLED
198+
PyMutex_Unlock(&it->mutex);
199+
#endif
200+
186201
if (type == ERRORTOKEN) {
187202
if(!PyErr_Occurred()) {
188203
_tokenizer_error(it->tok);

0 commit comments

Comments
 (0)