Skip to content

Commit 43ca27d

Browse files
authored
[mypyc] Fix missing module during concurrent circular import (python#22027)
The test `testConcurrentCircularNativeImports` occasionally fails by raising a `KeyError` exception when looking up a module in `sys.modules`. This appears to be caused by looking up the module in the middle of a reinsert operation done by importlib where the module is popped from `sys.modules` and inserted into it right after to reorder it - [Example](https://github.com/python/cpython/blob/3.14/Lib/importlib/_bootstrap.py#L949). During a circular import a thread participating in the cycle may not obtain the module lock because of deadlock detection. On a free-threaded build the thread holding the lock may concurrently perform the importlib reinsert and a thread without the lock may perform the look-up. To fix this, skip the look-up when possible after a native import and use the module object returned from the native import function. This also has the benefit of speeding up imports using `from native import x` statements by removing one module look-up. The module is also looked up after failing to acquire the module lock due to deadlock to obtain the partial module. If the module is missing here due to the race, for a module that has already done executing we can retry because its top-level statements cannot block other threads. The importlib reinsert happens only after a module has done executing, so otherwise the race shouldn't happen and the partial module missing from `sys.modules` is some other error. Also report an error when the `sys.modules` entry of a given native module is replaced/removed inside its own top-level function as this was inconsistent before.
1 parent 3b28c29 commit 43ca27d

6 files changed

Lines changed: 178 additions & 39 deletions

File tree

‎mypyc/irbuild/builder.py‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -523,7 +523,8 @@ def add_to_non_ext_dict(
523523
# doesn't cause contention.
524524
self.builder.set_immortal_if_free_threaded(val, line)
525525

526-
def gen_import(self, module: str, line: int) -> None:
526+
def gen_import(self, module: str, line: int) -> Value | None:
527+
"""Import a module and return it for a same-group native import."""
527528
self.imports[module] = None
528529

529530
needs_import, out = BasicBlock(), BasicBlock()
@@ -547,7 +548,20 @@ def gen_import(self, module: str, line: int) -> None:
547548
module_lock_api = Integer(0, c_pointer_rprimitive)
548549
else:
549550
module_id = self.load_str(module, line)
550-
self.check_if_module_loaded(module_id, module_cache, line, needs_import, out, import_state)
551+
already_loaded = BasicBlock() if is_same_group_native else out
552+
self.check_if_module_loaded(
553+
module_id, module_cache, line, needs_import, already_loaded, import_state
554+
)
555+
556+
result: Register | None = None
557+
if is_same_group_native:
558+
# Preserve the module returned by the native import. In particular,
559+
# don't look it up again in sys.modules after a circular import:
560+
# importlib temporarily removes a module there during finalization.
561+
result = Register(object_rprimitive, line=line)
562+
self.activate_block(already_loaded)
563+
self.add(Assign(result, self.get_module(module, line), line))
564+
self.goto(out)
551565

552566
self.activate_block(needs_import)
553567
if is_same_group_native:
@@ -593,13 +607,16 @@ def gen_import(self, module: str, line: int) -> None:
593607
],
594608
line,
595609
)
610+
assert result is not None
611+
self.add(Assign(result, value, line))
596612
else:
597613
# Import using generic Python C API
598614
assert module_id is not None
599615
value = self.call_c(import_op, [module_id], line)
600616
if not is_same_group_native:
601617
self.call_c(import_cache_replace_for_import_op, [module_cache, value], line)
602618
self.goto_and_activate(out)
619+
return result
603620

604621
def check_if_module_loaded(
605622
self,
@@ -643,8 +660,8 @@ def get_module_attr(self, module: str, attr: str, line: int) -> Value:
643660
644661
Import the module if needed.
645662
"""
646-
self.gen_import(module, line)
647-
module_obj = self.get_module(module, line)
663+
imported = self.gen_import(module, line)
664+
module_obj = imported if imported is not None else self.get_module(module, line)
648665
return self.py_get_attr(module_obj, attr, line)
649666

650667
def assign_if_null(self, target: Register, get_val: Callable[[], Value], line: int) -> None:

‎mypyc/irbuild/statement.py‎

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -419,14 +419,16 @@ def transform_imports_without_grouping(
419419
) -> None:
420420
globals = builder.load_globals_dict()
421421
for mod_id, as_name, line in group:
422-
builder.gen_import(mod_id, line)
422+
imported = builder.gen_import(mod_id, line)
423423
globals_id, globals_name = import_globals_id_and_name(mod_id, as_name)
424+
globals_name_value = builder.load_str(globals_name)
425+
module = (
426+
imported
427+
if imported is not None and globals_id == mod_id
428+
else builder.get_module(globals_id, line)
429+
)
424430
builder.gen_method_call(
425-
globals,
426-
"__setitem__",
427-
[builder.load_str(globals_name), builder.get_module(globals_id, line)],
428-
result_type=None,
429-
line=line,
431+
globals, "__setitem__", [globals_name_value, module], result_type=None, line=line
430432
)
431433

432434

@@ -558,7 +560,8 @@ def transform_import_from_buckets(
558560
]
559561
transform_imports_without_grouping(builder, group)
560562
elif bucket.kind == IMPORT_NATIVE_ATTR:
561-
builder.gen_import(module_id, line)
563+
native_module = builder.gen_import(module_id, line)
564+
assert native_module is not None
562565
names_literal = builder.add(LoadLiteral(tuple(bucket.names), object_rprimitive))
563566
if bucket.as_names == bucket.names:
564567
as_names_literal = names_literal
@@ -568,12 +571,7 @@ def transform_import_from_buckets(
568571
)
569572
builder.call_c(
570573
get_native_attrs_op,
571-
[
572-
builder.load_str(module_id),
573-
names_literal,
574-
as_names_literal,
575-
builder.load_globals_dict(),
576-
],
574+
[native_module, names_literal, as_names_literal, builder.load_globals_dict()],
577575
line,
578576
)
579577
else:

‎mypyc/lib-rt/CPy.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1085,7 +1085,7 @@ bool CPyImport_ImportMany(PyObject *modules, CPyModuleCache *statics[], PyObject
10851085
PyObject *tb_path, PyObject *tb_function, Py_ssize_t *tb_lines);
10861086
PyObject *CPyImport_ImportFromMany(PyObject *mod_id, PyObject *names, PyObject *as_names,
10871087
PyObject *globals);
1088-
PyObject *CPyImport_GetNativeAttrs(PyObject *mod_id, PyObject *names, PyObject *as_names,
1088+
PyObject *CPyImport_GetNativeAttrs(PyObject *mod, PyObject *names, PyObject *as_names,
10891089
PyObject *globals);
10901090
PyObject *CPyImport_ImportNative(PyObject *module_name,
10911091
PyObject *(*init_only_fn)(void),

‎mypyc/lib-rt/misc_ops.c‎

Lines changed: 45 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -906,33 +906,24 @@ PyObject *CPyImport_ImportFromMany(PyObject *mod_id, PyObject *names, PyObject *
906906
return mod;
907907
}
908908

909-
// Import attributes from an already-imported native module and store them
909+
// Import attributes from an already-imported native module object and store them
910910
// in the globals dict. Returns the module on success, NULL on error.
911-
PyObject *CPyImport_GetNativeAttrs(PyObject *mod_id, PyObject *names,
911+
PyObject *CPyImport_GetNativeAttrs(PyObject *mod, PyObject *names,
912912
PyObject *as_names, PyObject *globals) {
913-
PyObject *mod = PyImport_GetModule(mod_id);
914-
if (mod == NULL) {
915-
if (!PyErr_Occurred()) {
916-
PyErr_Format(PyExc_ImportError, "module '%U' is not in sys.modules", mod_id);
917-
}
918-
return NULL;
919-
}
920913
for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(names); i++) {
921914
PyObject *name = PyTuple_GET_ITEM(names, i);
922915
PyObject *as_name = PyTuple_GET_ITEM(as_names, i);
923916
PyObject *obj = PyObject_GetAttr(mod, name);
924917
if (obj == NULL) {
925-
Py_DECREF(mod);
926918
return NULL;
927919
}
928920
int ret = CPyDict_SetItem(globals, as_name, obj);
929921
Py_DECREF(obj);
930922
if (ret < 0) {
931-
Py_DECREF(mod);
932923
return NULL;
933924
}
934925
}
935-
return mod;
926+
return Py_NewRef(mod);
936927
}
937928

938929
// From CPython
@@ -1667,6 +1658,18 @@ static int CPyImport_ReleaseLockPreservingException(PyObject *module_lock) {
16671658
return result;
16681659
}
16691660

1661+
static void CPyImport_SetModuleReplacedError(PyObject *module_name) {
1662+
PyErr_Format(PyExc_ImportError,
1663+
"native module '%U' in sys.modules was replaced after initialization",
1664+
module_name);
1665+
}
1666+
1667+
static void CPyImport_SetModuleMissingError(PyObject *module_name) {
1668+
PyErr_Format(PyExc_ImportError,
1669+
"initialized native module '%U' is missing from sys.modules",
1670+
module_name);
1671+
}
1672+
16701673
// Execute a module once; caller holds the module lock.
16711674
int CPyImport_Exec(PyObject *module, int (*exec_fn)(PyObject *), CPyImportState *state,
16721675
CPyModuleCache *module_cache) {
@@ -1721,15 +1724,27 @@ PyObject *CPyImport_ImportNative(PyObject *module_name,
17211724
// We then check sys.modules to determine whether the module body
17221725
// has already been executed (or is being executed in a circular import).
17231726
PyObject *module_lock;
1724-
int lock_result = CPyImport_AcquireLock(lock_api, module_name, &module_lock);
1725-
if (lock_result == CPY_LOCK_DEADLOCK) {
1727+
int lock_result;
1728+
bool retried_after_execution = false;
1729+
while ((lock_result = CPyImport_AcquireLock(lock_api, module_name, &module_lock)) ==
1730+
CPY_LOCK_DEADLOCK) {
17261731
PyObject *partial = PyDict_GetItemWithError(PyImport_GetModuleDict(), module_name);
17271732
if (partial != NULL &&
17281733
(*module_static == NULL || partial == (PyObject *)*module_static)) {
17291734
CPyImport_ReplaceModuleCacheUnverified(module_cache, partial);
17301735
Py_INCREF(partial);
17311736
return partial;
17321737
}
1738+
// Multiple threads can detect the same cycle concurrently on a
1739+
// free-threaded build. If the target body has finished but the module
1740+
// is temporarily absent during importlib's pop/reinsert, retry once to
1741+
// wait for finalization. If it's still missing quit and report an error
1742+
// to not spin infinitely.
1743+
if (partial == NULL && !PyErr_Occurred() && CPyImport_IsExecuted(state) &&
1744+
!retried_after_execution) {
1745+
retried_after_execution = true;
1746+
continue;
1747+
}
17331748
if (!PyErr_Occurred()) {
17341749
PyErr_Format(PyExc_ImportError,
17351750
"import deadlock for native module '%U' without a partial module",
@@ -1779,9 +1794,7 @@ PyObject *CPyImport_ImportNative(PyObject *module_name,
17791794
}
17801795
return existing;
17811796
}
1782-
PyErr_Format(PyExc_ImportError,
1783-
"native module '%U' in sys.modules was replaced after initialization",
1784-
module_name);
1797+
CPyImport_SetModuleReplacedError(module_name);
17851798
CPyImport_ReleaseLockPreservingException(module_lock);
17861799
return NULL;
17871800
}
@@ -1793,9 +1806,7 @@ PyObject *CPyImport_ImportNative(PyObject *module_name,
17931806

17941807
if (CPyImport_IsInitialized(state) ||
17951808
CPyImport_GetModuleCache(module_cache) == (PyObject *)*module_static) {
1796-
PyErr_Format(PyExc_ImportError,
1797-
"initialized native module '%U' is missing from sys.modules",
1798-
module_name);
1809+
CPyImport_SetModuleMissingError(module_name);
17991810
CPyImport_ReleaseLockPreservingException(module_lock);
18001811
return NULL;
18011812
}
@@ -1831,6 +1842,20 @@ PyObject *CPyImport_ImportNative(PyObject *module_name,
18311842
goto fail;
18321843
}
18331844

1845+
// Direct imports bypass importlib's final sys.modules lookup. Reject changes
1846+
// to the entry instead of returning an object different from future imports.
1847+
PyObject *current = PyDict_GetItemWithError(module_dict, module_name);
1848+
if (current == NULL) {
1849+
if (!PyErr_Occurred()) {
1850+
CPyImport_SetModuleMissingError(module_name);
1851+
}
1852+
goto fail;
1853+
}
1854+
if (current != modobj) {
1855+
CPyImport_SetModuleReplacedError(module_name);
1856+
goto fail;
1857+
}
1858+
18341859
end_result = CPyImport_EndInitializing(initializing_spec);
18351860
initializing_spec = NULL;
18361861
if (end_result < 0) {

‎mypyc/primitives/misc_ops.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -207,7 +207,7 @@
207207
error_kind=ERR_MAGIC,
208208
)
209209

210-
# Get attributes from an already-imported native module and store them in globals.
210+
# Get attributes from an already-imported native module object and store them in globals.
211211
get_native_attrs_op = custom_op(
212212
arg_types=[object_rprimitive, object_rprimitive, object_rprimitive, object_rprimitive],
213213
return_type=object_rprimitive,

‎mypyc/test-data/run-multimodule.test‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -438,6 +438,63 @@ else:
438438

439439
assert sys.modules["other_top"] is fake
440440

441+
[case testNativeImportRejectsSysModulesReplacementDuringInitialization]
442+
# separate: [(["native.py", "other_target.py"], "testgroup")]
443+
def load_target() -> None:
444+
import other_target
445+
446+
[file other_target.py]
447+
import sys
448+
449+
replacement = object()
450+
sys.modules[__name__] = replacement
451+
452+
[file sys/__init__.pyi]
453+
modules: dict[str, object]
454+
455+
[file driver.py]
456+
import sys
457+
458+
import native
459+
460+
from testutil import assertRaises
461+
462+
with assertRaises(
463+
ImportError,
464+
"native module 'other_target' in sys.modules was replaced after initialization",
465+
):
466+
native.load_target()
467+
468+
assert "other_target" not in sys.modules
469+
470+
[case testNativeImportRejectsSysModulesRemovalDuringInitialization]
471+
# separate: [(["native.py", "other_target.py"], "testgroup")]
472+
def load_target() -> None:
473+
import other_target
474+
475+
[file other_target.py]
476+
import sys
477+
478+
del sys.modules[__name__]
479+
480+
[file sys/__init__.pyi]
481+
modules: dict[str, object]
482+
483+
[file driver.py]
484+
import sys
485+
486+
import native
487+
488+
from testutil import assertRaises
489+
490+
with assertRaises(
491+
ImportError,
492+
"initialized native module 'other_target' is missing from sys.modules",
493+
):
494+
native.load_target()
495+
496+
assert "other_target" not in sys.modules
497+
441498
[case testMultiModuleFastpaths]
442499
[file other_main.py]
443500

@@ -2633,6 +2690,48 @@ with ThreadPoolExecutor(max_workers=2) as executor:
26332690
assert other_a.other_value == "b"
26342691
assert other_b.other_value == "a"
26352692

2693+
[case testConcurrentCircularNativeFromImports]
2694+
# separate: [(["other_a.py", "other_b.py"], "testgroup")]
2695+
pass
2696+
2697+
[file import_sync.py]
2698+
from threading import Barrier
2699+
2700+
barrier = Barrier(2)
2701+
2702+
[file other_a.py]
2703+
from import_sync import barrier
2704+
2705+
def value() -> str:
2706+
return "a"
2707+
2708+
barrier.wait(timeout=15)
2709+
from other_b import value as other_value_fn
2710+
other_value = other_value_fn()
2711+
2712+
[file other_b.py]
2713+
from import_sync import barrier
2714+
2715+
def value() -> str:
2716+
return "b"
2717+
2718+
barrier.wait(timeout=15)
2719+
from other_a import value as other_value_fn
2720+
other_value = other_value_fn()
2721+
2722+
[file driver.py]
2723+
from concurrent.futures import ThreadPoolExecutor
2724+
import importlib
2725+
2726+
with ThreadPoolExecutor(max_workers=2) as executor:
2727+
future_a = executor.submit(importlib.import_module, "other_a")
2728+
future_b = executor.submit(importlib.import_module, "other_b")
2729+
other_a = future_a.result(timeout=15)
2730+
other_b = future_b.result(timeout=15)
2731+
2732+
assert other_a.other_value == "b"
2733+
assert other_b.other_value == "a"
2734+
26362735
[case testNativeModuleReimportBehavior]
26372736
value = 42
26382737

0 commit comments

Comments
 (0)