From ec42c281a8aa4e7e450e5146266ab9cc482e452e Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" <68491+gpshead@users.noreply.github.com> Date: Sat, 24 Oct 2026 17:06:37 -0700 Subject: [PATCH 1/2] gh-159963: Don't hold parent package import locks while a submodule executes (GH-159965) Since gh-83065 (3.15.0), the first import of pkg.mod kept pkg's module lock until pkg/mod.py had finished executing. Threads that needed only the finished pkg stayed blocked for that long, and concurrent first imports could raise _DeadlockError or a spurious "partially initialized module" ImportError, or hang, with no import cycle involved. Parent packages are now imported, or waited for if another thread is initializing them, before the submodule's module lock is taken, so only that lock is held while the submodule executes. The lock ordering deadlock fixed by gh-83065 stays fixed. importlib.import_module() now returns the partially initialized module in a circular import split across threads, as the import statement already did, instead of raising _DeadlockError. That was long-standing behavior rather than part of the 3.15.0 regression. (cherry picked from commit 7159b152a0b6205941ad7f14ced3f7d9ec20f68b) Co-authored-by: Gregory P. Smith <68491+gpshead@users.noreply.github.com> --- Lib/importlib/_bootstrap.py | 169 +++--- .../test_importlib/import_/test_packages.py | 16 + .../test_importlib/test_threaded_import.py | 566 ++++++++++++++++++ ...-10-20-22-00-00.gh-issue-159963.pLkHld.rst | 6 + 4 files changed, 672 insertions(+), 85 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-22-00-00.gh-issue-159963.pLkHld.rst diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 75bc901e2382932..82f3c200ac8710e 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -424,64 +424,6 @@ def __exit__(self, *args, **kwargs): self._lock.release() -def _get_module_chain(name): - """Return the chain of dotted-name prefixes from root to leaf. - - For example: 'a.b.c' -> ['a', 'a.b', 'a.b.c'] - """ - parts = name.split('.') - return ['.'.join(parts[:i+1]) for i in range(len(parts))] - - -class _HierarchicalLockManager: - """Manages acquisition of multiple module locks in hierarchical order. - - This prevents deadlocks by ensuring all threads acquire locks in the - same order (parent modules before child modules). - """ - - def __init__(self, name): - self._name = name - self._module_chain = _get_module_chain(name) - self._locks = [] - - def __enter__(self): - try: - for module_name in self._module_chain: - # Only acquire lock if module is not already fully loaded - module = sys.modules.get(module_name) - if (module is None or - getattr(getattr(module, "__spec__", None), - "_initializing", False)): - lock = _get_module_lock(module_name) - try: - lock.acquire() - except _DeadlockError: - if module_name == self._name: - raise - # The parent is being initialised by a thread that - # is (transitively) waiting on a lock we hold. - # Apply the same policy as _lock_unlock_module(): - # accept a partially-initialised parent for circular - # imports rather than failing the whole chain. - continue - self._locks.append((module_name, lock)) - except: - # __exit__ is not called when __enter__ raises (e.g. _DeadlockError - # on the leaf lock, or KeyboardInterrupt), so release whatever we - # already hold to avoid permanently leaking held module locks. - for module_name, lock in reversed(self._locks): - lock.release() - self._locks.clear() - raise - return self - - def __exit__(self, *args, **kwargs): - for module_name, lock in reversed(self._locks): - lock.release() - self._locks.clear() - - # The following two functions are for consumption by Python/import.c. def _get_module_lock(name): @@ -1262,17 +1204,15 @@ def _sanity_check(name, package, level): def _find_and_load_unlocked(name, import_, *, lazy_submodule=False): path = None - sys.audit( - "import", - name, - path, - getattr(sys, "path", None), - getattr(sys, "meta_path", None), - getattr(sys, "path_hooks", None) - ) parent = name.rpartition('.')[0] parent_spec = None if parent: + # _resolve_parents() has normally imported the parent already. It can + # be missing again by now if its import failed in another thread or + # something removed it from sys.modules; it is then imported afresh, + # like any other module that is not in sys.modules. That import runs + # while holding name's lock, the lock order _resolve_parents() exists + # to avoid; it is tolerated here because the case is so rare. if parent not in sys.modules: _call_with_frames_removed(import_, parent) # Crazy side-effects! @@ -1330,31 +1270,90 @@ def _find_and_load_unlocked(name, import_, *, lazy_submodule=False): _NEEDS_LOADING = object() +# Return whether module (a sys.modules value) is still executing. +def _is_initializing(module): + return getattr(getattr(module, '__spec__', None), '_initializing', False) + + +# Import, or wait for, the parent packages of name. +# +# _find_and_load() calls this before taking name's module lock, so that no +# thread holds a submodule's lock while waiting for its parent. This avoids +# the deadlocks and import errors of gh-83065 and gh-159963. +def _resolve_parents(name, import_): + # Wait for each ancestor package that another thread is still + # initializing, outermost first. Importing beneath a package while its + # __init__ is part way through its own imports could give one of the two + # threads a partially initialized module. Missing ancestors are not + # imported: only the immediate parent has to exist. + parts = name.split('.') + for i in range(1, len(parts) - 1): + ancestor = '.'.join(parts[:i]) + if _is_initializing(sys.modules.get(ancestor)): + _lock_unlock_module(ancestor) + # Import the immediate parent, or wait for it likewise. + parent = name.rpartition('.')[0] + if parent: + module = sys.modules.get(parent, _NEEDS_LOADING) + if module is _NEEDS_LOADING or _is_initializing(module): + _call_with_frames_removed(import_, parent) + + def _find_and_load(name, import_, *, lazy_submodule=False): """Find and load the module.""" # Optimization: we avoid unneeded module locking if the module # already exists in sys.modules and is fully initialized. module = sys.modules.get(name, _NEEDS_LOADING) - if (module is _NEEDS_LOADING or - getattr(getattr(module, "__spec__", None), "_initializing", False)): - - if '.' in name: - lock_manager = _HierarchicalLockManager(name) - else: - lock_manager = _ModuleLockManager(name) - - with lock_manager: - module = sys.modules.get(name, _NEEDS_LOADING) - if module is _NEEDS_LOADING: - return _find_and_load_unlocked( - name, import_, lazy_submodule=lazy_submodule) - - # Optimization: only call _bootstrap._lock_unlock_module() if - # module.__spec__._initializing is True. - # NOTE: because of this, initializing must be set *before* - # putting the new module in sys.modules. - _lock_unlock_module(name) + if module is _NEEDS_LOADING or _is_initializing(module): + if module is not _NEEDS_LOADING: + # Wait for the thread executing the module. + # NOTE: this relies on __spec__._initializing being set *before* + # the module is put in sys.modules. + lock = _get_module_lock(name) + try: + lock.acquire() + except _DeadlockError: + # A circular import split across threads: accept the + # partially initialized module. + pass + else: + # The module is gone if the thread loading it failed. Look + # while still holding the lock, so that another thread cannot + # have begun loading it again. + try: + module = sys.modules.get(name, _NEEDS_LOADING) + finally: + lock.release() + + if module is _NEEDS_LOADING: + sys.audit( + "import", + name, + None, + getattr(sys, "path", None), + getattr(sys, "meta_path", None), + getattr(sys, "path_hooks", None) + ) + _resolve_parents(name, import_) + lock = _get_module_lock(name) + try: + lock.acquire() + except _DeadlockError: + # A circular import split across threads. As in + # _lock_unlock_module(), accept a partially initialized + # module if there is one. + module = sys.modules.get(name, _NEEDS_LOADING) + if module is _NEEDS_LOADING: + raise + else: + try: + module = sys.modules.get(name, _NEEDS_LOADING) + if module is _NEEDS_LOADING: + return _find_and_load_unlocked( + name, import_, lazy_submodule=lazy_submodule) + finally: + lock.release() else: # Verify the module is still in sys.modules. Another thread may have # removed it (due to import failure) between our sys.modules.get() diff --git a/Lib/test/test_importlib/import_/test_packages.py b/Lib/test/test_importlib/import_/test_packages.py index 0c29d6083265fa1..10ca5ce2d5a0e1d 100644 --- a/Lib/test/test_importlib/import_/test_packages.py +++ b/Lib/test/test_importlib/import_/test_packages.py @@ -1,5 +1,6 @@ from test.test_importlib import util import sys +import types import unittest from test.support import import_helper @@ -100,6 +101,21 @@ def module_injection(): finally: import_helper.unload(subname) + def test_missing_grandparent_not_imported(self): + # Only the immediate parent has to be present. With a hand-made + # 'pkg.sub' package in sys.modules and no 'pkg' at all, importing + # pkg.sub.mod must neither fail nor try to import 'pkg'. + parent = types.ModuleType('pkg.sub') + parent.__path__ = [] + with util.mock_spec('pkg.sub.mod') as mock: + with util.import_state(meta_path=[mock]): + with util.uncache('pkg', 'pkg.sub', 'pkg.sub.mod'): + sys.modules['pkg.sub'] = parent + module = self.__import__('pkg.sub.mod', fromlist=['*']) + self.assertEqual(module.__name__, 'pkg.sub.mod') + self.assertIs(parent.mod, module) + self.assertNotIn('pkg', sys.modules) + (Frozen_ParentTests, Source_ParentTests diff --git a/Lib/test/test_importlib/test_threaded_import.py b/Lib/test/test_importlib/test_threaded_import.py index 6875fdca9c8528d..99f1480c8d72122 100644 --- a/Lib/test/test_importlib/test_threaded_import.py +++ b/Lib/test/test_importlib/test_threaded_import.py @@ -6,12 +6,16 @@ # randrange, and then Python hangs. import _imp as imp +import collections +import faulthandler import os import importlib +import importlib.machinery import sys import time import shutil import threading +import types import unittest from test import support from test.support import verbose @@ -461,6 +465,568 @@ def do_import(name): errors, [], f"Import(s) failed on iteration {i}: {errors}") + # Helpers for the deterministic two-thread tests below. Rather than + # relying on sleeps, a package __init__ calls back into the test and is + # held there until the second thread is observed waiting on a specific + # module lock, so the problematic interleaving happens on every run. + + def _setup_package_tree(self, files): + # These tests are about what happens when one module's body imports + # another. Under -X lazy_imports=all those imports would never run. + self.addCleanup(sys.set_lazy_imports, sys.get_lazy_imports()) + sys.set_lazy_imports("normal") + os.makedirs(TESTFN, exist_ok=True) + self.addCleanup(rmtree, TESTFN) + for relpath, source in files.items(): + path = os.path.join(TESTFN, *relpath.split("/")) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + f.write(source) + sys.path.insert(0, TESTFN) + self.addCleanup(sys.path.remove, TESTFN) + # Forget by top-level name, not by file name: a submodule may come + # from a directory added to __path__ or have no file at all. + tops = {relpath.partition("/")[0].removesuffix(".py") + for relpath in files} + self.addCleanup(self._forget_trees, tops) + importlib.invalidate_caches() + + @staticmethod + def _forget_trees(tops): + for name in list(sys.modules): + if name.partition(".")[0] in tops: + forget(name) + + def _wait_until_blocked_on(self, thread, modname): + # Return once *thread* is waiting to acquire *modname*'s module lock + # (any module lock if *modname* is None) and another thread owns it. + # The owner check matters: a lock is listed in _blocking_on for a + # moment during every acquire(), contended or not. + blocking_on = importlib._bootstrap._blocking_on + for _ in support.sleeping_retry( + support.SHORT_TIMEOUT, + f"thread never waited for the module lock of {modname!r}"): + for lock in list(blocking_on.get(thread.ident, ())): + if (modname in (None, lock.name) + and lock.owner not in (None, thread.ident)): + return + if not thread.is_alive(): + self.fail("thread finished without ever waiting for the " + f"module lock of {modname!r}") + + def _join_import_threads(self, threads): + # Every wait made by the test's own code inside these threads is + # bounded by SHORT_TIMEOUT and reports what it was waiting for, so + # wait longer than their sum here: only a real import deadlock gets + # this far. The threads are daemons so that one cannot also block + # interpreter shutdown. + deadline = time.monotonic() + support.LONG_TIMEOUT + for t in threads: + if t.ident is not None: + t.join(max(deadline - time.monotonic(), 0)) + hung = [t for t in threads if t.is_alive()] + if hung: + faulthandler.dump_traceback(sys.__stderr__) + self.fail(f"import deadlock: unable to join {hung}") + + def _run_two_first_imports(self, files, stmt1, stmt2, *, + sync_name, t2_waits_for): + # Thread 1 runs stmt1. Something it executes calls + # .hold(), which starts thread 2 on stmt2 and returns + # only once thread 2 is blocked on the module lock of t2_waits_for + # (any module lock if None). Module code may also call + # .join_t2() to wait for thread 2 to finish. + # Returns a list of (what, problem) pairs; empty means both imports + # succeeded and no module body ran more than once. + # list.append() is atomic, on free-threaded builds too. + count = f"import {sync_name}; {sync_name}.executed.append(__name__)\n" + self._setup_package_tree({relpath: count + source + for relpath, source in files.items()}) + sync = types.ModuleType(sync_name) + sync.executed = [] + sys.modules[sync_name] = sync + self.addCleanup(sys.modules.pop, sync_name, None) + errors = [] + + def run(stmt): + try: + exec(stmt, {}) + except Exception as e: + errors.append((stmt, f"{type(e).__name__}: {e}")) + + t1 = threading.Thread(target=run, args=(stmt1,), daemon=True) + t2 = threading.Thread(target=run, args=(stmt2,), daemon=True) + + def hold(): + # One-shot: if thread 1's import fails, thread 2 may re-run the + # same code; it must not wait on itself. + if t2.ident is not None: + return + t2.start() + self._wait_until_blocked_on(t2, t2_waits_for) + sync.hold = hold + + def join_t2(): + t2.join(support.SHORT_TIMEOUT) + if t2.is_alive(): + errors.append((stmt2, "still blocked while thread 1's module " + "code waited for it")) + sync.join_t2 = join_t2 + + t1.start() + self._join_import_threads([t1, t2]) + if t2.ident is None: + errors.append((stmt2, "was never started")) + for name, n in collections.Counter(sync.executed).items(): + if n != 1: + errors.append((name, f"executed {n} times")) + return errors + + def _hold_in_create_module(self, modname, sync_name): + # Have the import of *modname* call .hold() from its + # loader's create_module(): the importing thread then owns modname's + # module lock but modname is not in sys.modules yet. (A finder's + # find_spec() would be too early a place to stop: it runs under the + # global import lock, which every other import needs briefly.) + class Loader(importlib.machinery.SourceFileLoader): + def create_module(self, spec): + sys.modules[sync_name].hold() + return super().create_module(spec) + + class Finder: + def find_spec(self, name, path=None, target=None): + if name != modname: + return None + spec = importlib.machinery.PathFinder.find_spec( + name, path, target) + spec.loader = Loader(spec.loader.name, spec.loader.path) + return spec + + finder = Finder() + sys.meta_path.insert(0, finder) + self.addCleanup(sys.meta_path.remove, finder) + + # A first import of {pkg}.mod. The statements call back into C + # (import_ensure_initialized()) for a parent that is in sys.modules; + # import_module() stays in _find_and_load(). + SUBMODULE_IMPORTS = ( + "from {pkg}.mod import X", + "import {pkg}.mod", + "import importlib; importlib.import_module('{pkg}.mod')", + ) + + def test_parent_lock_not_held_while_submodule_executes(self): + # gh-159963: the first `import pkg.mod` in a process also imports pkg. + # Once pkg/__init__.py has finished, pkg's module lock must not stay + # held while mod's body runs; otherwise every thread that began + # waiting for pkg is blocked for the whole of mod's body as well. + self._setup_package_tree({ + "unheld_pkg/__init__.py": "", + "unheld_pkg/mod.py": ( + "import sys, importlib._bootstrap as _b\n" + "def held(name):\n" + " ref = _b._module_locks.get(name)\n" + " lock = ref() if ref is not None else None\n" + " return lock is not None and lock.locked()\n" + "own_lock_held = held(__name__)\n" + "parent_lock_held = held('unheld_pkg')\n" + "parent_initializing = " + "sys.modules['unheld_pkg'].__spec__._initializing\n"), + }) + import unheld_pkg.mod + self.assertTrue(unheld_pkg.mod.own_lock_held) + self.assertFalse(unheld_pkg.mod.parent_initializing) + self.assertFalse(unheld_pkg.mod.parent_lock_held) + + def test_package_init_imports_submodule_deterministic(self): + # gh-83065 (bpo-38884) with the losing interleaving forced: thread 1 + # is inside pkg/__init__.py, about to import pkg.mod, at the moment + # thread 2's first import of pkg.mod has got as far as waiting for + # pkg. Thread 2 must be waiting *without* already owning pkg.mod's + # lock, or thread 1's import of pkg.mod deadlocks against it. + for i, stmt in enumerate(self.SUBMODULE_IMPORTS): + pkg = f"detinit{i}_pkg" + stmt = stmt.format(pkg=pkg) + with self.subTest(stmt=stmt): + errors = self._run_two_first_imports( + { + f"{pkg}/__init__.py": ( + "import detinit_sync; detinit_sync.hold()\n" + f"import {pkg}.mod\n"), + f"{pkg}/mod.py": "X = 1\n", + }, + f"import {pkg}", + stmt, + sync_name="detinit_sync", + t2_waits_for=pkg, + ) + self.assertEqual(errors, []) + self.assertEqual(sys.modules[f"{pkg}.mod"].X, 1) + + def test_submodule_import_while_package_not_yet_in_sys_modules(self): + # gh-83065, the earlier window: thread 1 owns pkg's lock but has not + # put pkg in sys.modules yet when thread 2 first imports pkg.mod. + for i, stmt in enumerate(self.SUBMODULE_IMPORTS): + pkg = f"early{i}_pkg" + stmt = stmt.format(pkg=pkg) + with self.subTest(stmt=stmt): + self._hold_in_create_module(pkg, "early_sync") + errors = self._run_two_first_imports( + { + f"{pkg}/__init__.py": "from . import mod\n", + f"{pkg}/mod.py": "X = 1\n", + }, + f"import {pkg}", + stmt, + sync_name="early_sync", + t2_waits_for=pkg, + ) + self.assertEqual(errors, []) + self.assertEqual(sys.modules[f"{pkg}.mod"].X, 1) + + def test_submodule_found_via_path_set_late_in_package_init(self): + # pkg.mod can only be found once pkg/__init__.py has replaced + # __path__, pkgutil.extend_path() style. Thread 2 must look at + # pkg.__path__ after it has waited for pkg, not before. + extra = os.path.abspath(os.path.join(TESTFN, "latepath_extra")) + errors = self._run_two_first_imports( + { + "latepath_pkg/__init__.py": ( + "import latepath_sync; latepath_sync.hold()\n" + f"__path__ = [*__path__, {extra!r}]\n"), + "latepath_extra/mod.py": "X = 1\n", + }, + "import latepath_pkg", + "from latepath_pkg.mod import X", + sync_name="latepath_sync", + t2_waits_for="latepath_pkg", + ) + self.assertEqual(errors, []) + self.assertEqual(sys.modules["latepath_pkg.mod"].X, 1) + + def test_submodule_put_in_sys_modules_late_in_package_init(self): + # gh-130094: there is no pkg/mod.py; pkg/__init__.py puts pkg.mod in + # sys.modules itself, as collections does for collections.abc. + errors = self._run_two_first_imports( + { + "inject_pkg/__init__.py": ( + "import inject_sync; inject_sync.hold()\n" + "import sys, types\n" + "mod = types.ModuleType('inject_pkg.mod'); mod.X = 1\n" + "sys.modules['inject_pkg.mod'] = mod\n"), + }, + "import inject_pkg", + "from inject_pkg.mod import X", + sync_name="inject_sync", + t2_waits_for="inject_pkg", + ) + self.assertEqual(errors, []) + + def test_concurrent_first_imports_of_sibling_packages(self): + # gh-159963. Thread 1: `import a.leaf`, where a/leaf.py needs b.leaf. + # Thread 2: `import b.leaf`, where b/__init__.py needs a, started + # while thread 1 is still inside a/__init__.py. + # Dependencies are a.leaf -> b.leaf -> b -> a; `a` needs nothing, so + # there is no cycle and either order works in one thread. If the + # importer of a.leaf kept a's lock for the duration of a/leaf.py, + # this would raise _DeadlockError. + errors = self._run_two_first_imports( + { + "sib_a/__init__.py": "import sib_sync; sib_sync.hold()\n", + "sib_a/leaf.py": "from sib_b.leaf import X\n", + "sib_b/__init__.py": "import sib_a\n", + "sib_b/leaf.py": "X = 1\n", + }, + "import sib_a.leaf", + "import sib_b.leaf", + sync_name="sib_sync", + t2_waits_for="sib_a", + ) + self.assertEqual(errors, []) + self.assertEqual(sys.modules["sib_a.leaf"].X, 1) + self.assertEqual(sys.modules["sib_b.leaf"].X, 1) + + def test_finished_package_import_not_blocked_by_submodule_body(self): + # Thread 2 starts waiting for pkg while pkg/__init__.py runs. It + # needs only the finished pkg, so it must be released when that + # __init__ returns and not when thread 1's pkg/main.py does. Here + # main.py waits for thread 2, which no deadlock detector can see. + for i, stmt in enumerate(( + "import {pkg}", + "import {pkg}.other", + "from {pkg}.other import X")): + pkg = f"body{i}_pkg" + stmt = stmt.format(pkg=pkg) + with self.subTest(stmt=stmt): + errors = self._run_two_first_imports( + { + f"{pkg}/__init__.py": ( + "import body_sync; body_sync.hold()\n"), + f"{pkg}/main.py": ( + "import body_sync; body_sync.join_t2()\n"), + f"{pkg}/other.py": "X = 1\n", + }, + f"import {pkg}.main", + stmt, + sync_name="body_sync", + t2_waits_for=pkg, + ) + self.assertEqual(errors, []) + + def test_concurrent_first_imports_of_nested_packages(self): + # Thread 1 does `import g.sub.leaf`; g/sub/__init__.py needs + # module c; c needs g. Thread 2 imports c + # while thread 1 is still inside g/__init__.py. No cycle. If + # thread 1 still held g's lock while it ran g/sub/__init__.py, + # thread 2 (holding c, waiting for g) would look deadlocked to it + # and it would get "cannot import name 'X' from partially + # initialized module". So g's lock must be released before any + # package beneath it executes, not only before the leaf's body. + errors = self._run_two_first_imports( + { + "nest_g/__init__.py": "import nest_sync; nest_sync.hold()\n", + "nest_g/sub/__init__.py": "from nest_c import X\n", + "nest_g/sub/leaf.py": "Y = 2\n", + "nest_c.py": "import nest_g\nX = 1\n", + }, + "import nest_g.sub.leaf", + "import nest_c", + sync_name="nest_sync", + t2_waits_for="nest_g", + ) + self.assertEqual(errors, []) + self.assertEqual(sys.modules["nest_g.sub"].X, 1) + self.assertEqual(sys.modules["nest_g.sub.leaf"].Y, 2) + + def test_submodule_import_waits_for_initializing_top_package(self): + # Thread 1 runs g/__init__.py, which has imported subpackage g.a and + # will next need g.a.other. Thread 2's first import of g.a.other + # starts at that point; other.py needs an attribute of g that is + # already set. Thread 2 must wait for g *before* owning g.a.other: + # if it loaded g.a.other first and only then blocked on g (inside + # other.py), thread 1 would find g.a.other held by a thread waiting + # on itself and be handed the half-executed module ("cannot import + # name 'X'"). Nothing here can fail in a single thread. + errors = self._run_two_first_imports( + { + "topw_g/__init__.py": ( + "Z = 1\n" + "import topw_g.a\n" + "import topw_sync; topw_sync.hold()\n" + "from topw_g.a.other import X\n"), + "topw_g/a/__init__.py": "", + "topw_g/a/other.py": "from topw_g import Z\nX = 1\n", + }, + "import topw_g", + "from topw_g.a.other import X", + sync_name="topw_sync", + t2_waits_for="topw_g", + ) + self.assertEqual(errors, []) + self.assertEqual(sys.modules["topw_g"].X, 1) + + def test_import_module_of_initializing_submodule_waits_for_it_only(self): + # The converse of waiting for an initializing ancestor before + # loading a submodule: pkg/__init__.py has *already* + # imported pkg.leaf and then blocks on thread 2 (any non-import + # wait). Thread 2, started while pkg/leaf.py is still executing, + # asks for pkg.leaf via importlib. It must get it as soon as leaf.py + # finishes, exactly as the import statement would, and not be made + # to wait for pkg/__init__.py -- which is waiting for thread 2. + errors = self._run_two_first_imports( + { + "lfo_pkg/__init__.py": ( + "import lfo_pkg.leaf\n" + "import lfo_sync; lfo_sync.join_t2()\n"), + "lfo_pkg/leaf.py": "import lfo_sync; lfo_sync.hold()\nX = 1\n", + }, + "import lfo_pkg", + "import importlib; importlib.import_module('lfo_pkg.leaf')", + sync_name="lfo_sync", + t2_waits_for=None, + ) + self.assertEqual(errors, []) + + def test_import_module_in_cross_thread_cycle_returns_partial_module(self): + # ax/x.py sets X, then imports by.y; by/y.py does `from ax.x import + # X`. Thread 1 is inside x.py (holding ax.x's lock) when thread 2's + # `import by.y` reaches y.py and blocks on ax.x. Thread 1 then asks + # for by.y and is the one to detect the cycle. The import statement + # resolves this by handing thread 1 the partially + # initialized by.y (import.c -> _lock_unlock_module()); + # importlib.import_module() must do the same rather than raise + # _DeadlockError into x.py. + for i, form in enumerate(( + "import by{i}_cyc.y", + "import importlib; importlib.import_module('by{i}_cyc.y')")): + form = form.format(i=i) + with self.subTest(form=form): + errors = self._run_two_first_imports( + { + f"ax{i}_cyc/__init__.py": "", + f"ax{i}_cyc/x.py": ( + "X = 1\n" + "import cyc_sync; cyc_sync.hold()\n" + f"{form}\n"), + f"by{i}_cyc/__init__.py": "", + f"by{i}_cyc/y.py": ( + f"from ax{i}_cyc.x import X\nassert X == 1\n"), + }, + f"import ax{i}_cyc.x", + f"import by{i}_cyc.y", + sync_name="cyc_sync", + t2_waits_for=f"ax{i}_cyc.x", + ) + self.assertEqual(errors, []) + + def test_submodule_enters_cross_thread_cycle_after_parents_resolved(self): + # Thread 1 is inside m.py and has found pkg.leaf missing, but only + # asks for its lock once thread 2 is inside leaf.py, waiting for m. + # Thread 1 is the one to detect the cycle, and pkg.leaf is in + # sys.modules by now, so it must get the partial module, as if it had + # found it there in the first place. + resolve_parents = importlib._bootstrap._resolve_parents + + def stalling_resolve_parents(name, import_): + resolve_parents(name, import_) + if name == "stall_pkg.leaf": + sys.modules["stall_sync"].hold() + + with support.swap_attr(importlib._bootstrap, "_resolve_parents", + stalling_resolve_parents): + errors = self._run_two_first_imports( + { + "stall_pkg/__init__.py": "", + "stall_pkg/leaf.py": "import stall_m\nX = 1\n", + "stall_m.py": "import stall_pkg.leaf\n", + }, + "import stall_m", + "import stall_pkg.leaf", + sync_name="stall_sync", + t2_waits_for="stall_m", + ) + self.assertEqual(errors, []) + self.assertEqual(sys.modules["stall_pkg.leaf"].X, 1) + + def test_parent_removed_before_submodule_is_loaded(self): + # The parent package leaves sys.modules after it was imported but + # before the submodule is loaded, as when its import failed in + # another thread. The parent must be imported again rather than + # the submodule's import failing with KeyError. + self._setup_package_tree({ + "gone_pkg/__init__.py": "", + "gone_pkg/mod.py": "X = 1\n", + }) + resolve_parents = importlib._bootstrap._resolve_parents + removed = [] + + def removing_resolve_parents(name, import_): + resolve_parents(name, import_) + if name == "gone_pkg.mod": + removed.append(sys.modules.pop("gone_pkg")) + + with support.swap_attr(importlib._bootstrap, "_resolve_parents", + removing_resolve_parents): + import gone_pkg.mod + self.assertEqual(len(removed), 1) + self.assertIsNot(sys.modules["gone_pkg"], removed[0]) + self.assertEqual(sys.modules["gone_pkg"].mod.X, 1) + + def test_waiter_retries_after_failed_submodule_import(self): + # Thread 2 waits for thread 1's import of pkg.mod, whose first + # execution then fails. Thread 2 must load pkg.mod itself and not + # return the discarded module. + for i, stmt in enumerate(self.SUBMODULE_IMPORTS): + pkg = f"retry{i}_pkg" + stmt = stmt.format(pkg=pkg) + with self.subTest(stmt=stmt): + errors = self._run_two_first_imports( + { + f"{pkg}/__init__.py": "", + f"{pkg}/mod.py": ( + "import retry_sync; retry_sync.hold()\n" + "if retry_sync.executed.count(__name__) == 1:\n" + " raise RuntimeError('first attempt fails')\n" + "X = 1\n"), + }, + f"import {pkg}.mod", + stmt, + sync_name="retry_sync", + t2_waits_for=f"{pkg}.mod", + ) + self.assertEqual(errors, [ + (f"import {pkg}.mod", + "RuntimeError: first attempt fails"), + (f"{pkg}.mod", "executed 2 times"), + ]) + self.assertEqual(sys.modules[f"{pkg}.mod"].X, 1) + + def test_waiter_not_given_module_being_retried(self): + # Thread B waits for thread A's import of a module, which fails. + # A retries, so sys.modules may hold A's new, still executing + # module by the time B looks there again. B must wait for that + # attempt too rather than return a half initialized module. + self._setup_package_tree({ + "again_mod.py": "import again_sync; again_sync.body()\nX = 1\n", + }) + sync = types.ModuleType("again_sync") + sys.modules["again_sync"] = sync + self.addCleanup(sys.modules.pop, "again_sync", None) + released = threading.Event() # B stopped waiting for the 1st attempt + retrying = threading.Event() # the 2nd attempt is in sys.modules + errors = [] + results = [] + + def a(): + try: + try: + import again_mod + except RuntimeError: + pass + released.wait(support.SHORT_TIMEOUT) + import again_mod + except Exception as e: + errors.append(f"A: {type(e).__name__}: {e}") + + def b(): + try: + module = importlib.import_module("again_mod") + results.append(hasattr(module, "X")) + except Exception as e: + errors.append(f"B: {type(e).__name__}: {e}") + + ta = threading.Thread(target=a, daemon=True) + tb = threading.Thread(target=b, daemon=True) + + def body(): + if tb.ident is None: + tb.start() + self._wait_until_blocked_on(tb, "again_mod") + raise RuntimeError("first attempt fails") + retrying.set() + if threading.current_thread() is not tb: + # Finish only once B is waiting for this attempt. + self._wait_until_blocked_on(tb, "again_mod") + sync.body = body + + # Hold B back, once it has stopped waiting for the first attempt, + # until the second attempt is under way. + release = importlib._bootstrap._ModuleLock.release + + def stalling_release(lock): + release(lock) + if (lock.name == "again_mod" and not released.is_set() + and threading.current_thread() is tb): + released.set() + retrying.wait(support.SHORT_TIMEOUT) + + with support.swap_attr(importlib._bootstrap._ModuleLock, "release", + stalling_release): + ta.start() + self._join_import_threads([ta, tb]) + self.assertEqual(results, [True]) + self.assertEqual(errors, []) + def setUpModule(): thread_info = threading_helper.threading_setup() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-22-00-00.gh-issue-159963.pLkHld.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-22-00-00.gh-issue-159963.pLkHld.rst new file mode 100644 index 000000000000000..5c3bd8f63f1d4fb --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-22-00-00.gh-issue-159963.pLkHld.rst @@ -0,0 +1,6 @@ +Fix a regression in Python 3.15.0 where threads importing submodules of +packages for the first time could raise ``_DeadlockError`` or a spurious +"partially initialized module" :exc:`ImportError`, or block one another, with +no circular import involved. :func:`importlib.import_module` now returns the +partially initialized module in a cross-thread circular import, as the +:keyword:`import` statement does, instead of raising ``_DeadlockError``. From 533de4b326804b413ee177483ac9b21018ecadc4 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Sun, 25 Oct 2026 00:26:45 +0000 Subject: [PATCH 2/2] Add a What's New in 3.15.2 entry --- Doc/whatsnew/3.15.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 91bd562daf25b13..7d63dce4c611a9b 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -2912,6 +2912,18 @@ that may require changes to your code. Notable changes in 3.15.2 ========================= +importlib +--------- + +* Threads that simultaneously perform first-time ``import package.submodule`` + imports no longer risk a deadlock or spurious import errors, a regression + introduced in 3.15.0 by the fix for :gh:`83065`. A package's import lock is + now released as soon as its ``__init__`` returns. + :func:`importlib.import_module` now returns the partially initialized + module in a circular import split across threads, as the :keyword:`import` + statement already did, instead of raising ``_DeadlockError``. + (Contributed by Gregory P. Smith in :gh:`159963`.) + ssl ---