From a1a16c32d95bae71a0c380ed038295912b9c9dbe Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 20 Jul 2026 21:34:42 +0200 Subject: [PATCH 1/3] fix(android): make subinterpreters able to import relocated native modules PEP 734 subinterpreters (Python 3.14 concurrent.interpreters / InterpreterPoolExecutor) were unusable in Android apps: the main interpreter works, but every subinterpreter failed to import any C extension (ModuleNotFoundError: _struct / _interpqueues / ...), which breaks the whole feature (its cross-interpreter transport pickles -> _struct, and its queues need _interpqueues). iOS and desktop are unaffected. Root cause: Android relocates C extensions (native-mmap packaging) and resolves them through the custom _SorefFinder installed on sys.meta_path. sys.meta_path is per-interpreter, and install() only ran in the main interpreter, so a freshly created subinterpreter had a meta_path without the finder and could import no relocated .so. Verified on-device: main meta_path has _SorefFinder, a subinterpreter's does not. Fix: teach every new subinterpreter to install the finder. Split the finder insertion into _install_finder(), and make install() also call a new _patch_subinterpreters() that wraps concurrent.interpreters.create() so each new interpreter runs `import _sp_bootstrap; _sp_bootstrap.install()`. The setup runs via Interpreter.exec() -- a source string, which is pickle-free (unlike Interpreter.call()), so it works BEFORE _struct is importable in the child. (A fresh subinterpreter inherits the process-wide sys.path, so no sys.path seeding is needed -- verified on-device.) Notes: - No dart-bridge / C change: the existing Android bootstrap already calls install(), which now triggers the patch. The wrapper is tagged `_sp_patched` for idempotency. - create() is the factory InterpreterPoolExecutor uses (WorkerContext.initialize -> interpreters.create()), so patching it fixes the pool transparently -- with no user code change. - A user-supplied InterpreterPoolExecutor(initializer=...) can NOT fix this: the initializer is delivered over the same broken pickle path, so the finder must be installed at interpreter-creation time. - No-op before 3.14 (module absent) and on iOS/desktop (this file is Android-only). Idempotent; supports nested subinterpreters. Verified end-to-end on an Android 15 / arm64-v8a emulator (Python 3.14.6): the clean probe app -- with no app-level workaround -- runs a low-level create+queue round-trip, an InterpreterPoolExecutor across 4 interpreters in one process, and imports a C extension inside a subinterpreter. --- .../python/_sp_bootstrap.py | 53 ++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/src/serious_python_android/python/_sp_bootstrap.py b/src/serious_python_android/python/_sp_bootstrap.py index 0c19ca2b..a1647149 100644 --- a/src/serious_python_android/python/_sp_bootstrap.py +++ b/src/serious_python_android/python/_sp_bootstrap.py @@ -136,7 +136,7 @@ def find_spec(self, fullname, path=None, target=None): return spec -def install(): +def _install_finder(): """Insert the finder at the front of `sys.meta_path` (idempotent).""" global _installed if _installed: @@ -158,3 +158,54 @@ def install(): return sys.meta_path.insert(0, _SorefFinder()) _installed = True + + +def _patch_subinterpreters(): + """Make PEP 734 subinterpreters (Python 3.14+) able to import relocated + native modules. + + `sys.meta_path` is *per-interpreter*: a subinterpreter created by + `concurrent.interpreters` / `InterpreterPoolExecutor` starts with a fresh + meta_path that does NOT contain the `_SorefFinder` installed by `_install_finder`, + so it can import no relocated C extension (`_struct`, `_interpqueues`, ...). Since + that machinery ships work between interpreters via pickle (`_struct`) and + cross-interpreter queues (`_interpqueues`), the whole feature is unusable. + + Wrap `concurrent.interpreters.create()` so every new interpreter installs + the finder before it is used. The install runs via `Interpreter.exec()` — a + source string, which is pickle-free (unlike `Interpreter.call()`), so it + works *before* `_struct` is importable in the child. Idempotent; a no-op + before 3.14 (module absent) and cannot fix + `InterpreterPoolExecutor(initializer=...)` because the initializer is + delivered over the same broken pickle path — so it must run here, at + creation time. + """ + try: + from concurrent import interpreters + except Exception: + return # < 3.14, or subinterpreters unavailable + if getattr(interpreters.create, "_sp_patched", False): + return + _orig_create = interpreters.create + + def create(*args, **kwargs): + interp = _orig_create(*args, **kwargs) + try: + interp.exec("import _sp_bootstrap\n_sp_bootstrap.install()\n") + except Exception: + pass + return interp + + create._sp_patched = True + interpreters.create = create + + +def install(): + """Entry point called from the dart-bridge Android bootstrap. + + Installs the native-module finder in the current interpreter and — on + 3.14+ — teaches every future subinterpreter to install it too. + Safe to call in any interpreter; idempotent. + """ + _install_finder() + _patch_subinterpreters() From bcfcade21188169855363392af1cb40789665c10 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 20 Jul 2026 21:54:41 +0200 Subject: [PATCH 2/3] docs: note the Android subinterpreter fix under 4.3.6 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fold the subinterpreter fix into the existing 4.3.6 section (main jumped 4.3.4 -> 4.3.6 with the Windows UTF-8 release; our 4.3.5 draft was rebased away): a detailed bullet in serious_python_android and a summary bullet in serious_python (umbrella). The Apple/Windows/Linux/ interface packages already carry their 4.3.6 entries from the Windows release and are unaffected by this Android-only change. No pubspec bump needed — this rides in the same 4.3.6 that main is already on. --- src/serious_python/CHANGELOG.md | 1 + src/serious_python_android/CHANGELOG.md | 1 + 2 files changed, 2 insertions(+) diff --git a/src/serious_python/CHANGELOG.md b/src/serious_python/CHANGELOG.md index 3d282b5c..92538239 100644 --- a/src/serious_python/CHANGELOG.md +++ b/src/serious_python/CHANGELOG.md @@ -1,5 +1,6 @@ ## 4.3.6 +* **Android:** PEP 734 subinterpreters (Python 3.14's `concurrent.interpreters` / `InterpreterPoolExecutor`) now work in built apps. Previously the main interpreter could import them, but every *subinterpreter* failed to import any relocated C extension (`ModuleNotFoundError: _struct` / `_interpqueues` / ...) — which broke the whole feature (its cross-interpreter transport pickles → `_struct`, and its queues need `_interpqueues`). The native-module finder lives on `sys.meta_path`, which is per-interpreter, and was installed only in the main interpreter. See `serious_python_android` 4.3.6. * **Windows:** fix startup with non-ASCII app paths or environment values — Dart FFI strings (UTF-8) are now converted to UTF-16 before being passed to the Windows CRT, so paths/env values are no longer corrupted through the process ANSI code page, and Python UTF-8 mode is enabled before `Py_Initialize()`. See `serious_python_windows` 4.3.6 and flet-dev/flet#6641. * Bundled python-build snapshot re-pinned to **20260720** (`dart_bridge` **1.5.0 → 1.5.1**, which carries the Windows fix above). Python versions (**3.12.13 / 3.13.14 / 3.14.6**) are unchanged. diff --git a/src/serious_python_android/CHANGELOG.md b/src/serious_python_android/CHANGELOG.md index ec4c989f..e8760e7c 100644 --- a/src/serious_python_android/CHANGELOG.md +++ b/src/serious_python_android/CHANGELOG.md @@ -1,5 +1,6 @@ ## 4.3.6 +* **PEP 734 subinterpreters now work** (Python 3.14 `concurrent.interpreters` / `InterpreterPoolExecutor`) — enabling true multi-core CPU parallelism inside a single process, which `multiprocessing` cannot provide on Android. Previously the main interpreter imported the machinery fine, but every subinterpreter raised `ModuleNotFoundError` for `_struct` / `_interpqueues` / any other C extension, so `InterpreterPoolExecutor` and the low-level API were unusable. Cause: Android relocates C extensions (native-mmap packaging) and resolves them through `_SorefFinder` on `sys.meta_path`; `sys.meta_path` is per-interpreter, and `install()` only ran in the main interpreter, so a freshly created subinterpreter had a `meta_path` without the finder. Fix: `_sp_bootstrap.install()` now also wraps `concurrent.interpreters.create()` so every new interpreter installs the finder before use. The install runs via `Interpreter.exec()` — a source string, which is pickle-free (unlike `Interpreter.call()`), so it works *before* `_struct` is importable in the child. Because `InterpreterPoolExecutor` builds its workers via `concurrent.interpreters.create()`, the pool is fixed transparently with no user-code change. A no-op before 3.14; iOS/desktop were unaffected. * Re-pins the bundled python-build snapshot to **20260720** (`dart_bridge` **1.5.0 → 1.5.1**). 1.5.1 is a Windows-only UTF-8 startup fix (see `serious_python_windows` 4.3.6); the Android runtime is functionally unchanged from 20260719. ## 4.3.4 From e522e41cdc3673716fccf56b52f6a481002e57b5 Mon Sep 17 00:00:00 2001 From: Feodor Fitsner Date: Mon, 20 Jul 2026 14:06:58 -0700 Subject: [PATCH 3/3] docs+debug: note create()-attribute assumption; stderr breadcrumb on child finder-install failure --- src/serious_python_android/python/_sp_bootstrap.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/serious_python_android/python/_sp_bootstrap.py b/src/serious_python_android/python/_sp_bootstrap.py index a1647149..01a6aafa 100644 --- a/src/serious_python_android/python/_sp_bootstrap.py +++ b/src/serious_python_android/python/_sp_bootstrap.py @@ -179,6 +179,12 @@ def _patch_subinterpreters(): `InterpreterPoolExecutor(initializer=...)` because the initializer is delivered over the same broken pickle path — so it must run here, at creation time. + + NOTE: this relies on callers resolving `interpreters.create` as a module + attribute at call time (as `InterpreterPoolExecutor` does on 3.14) rather + than binding it via `from concurrent.interpreters import create`. If a + future CPython changes that, the patch silently stops applying to the pool + and subinterpreter imports regress — verified working on 3.14.6. """ try: from concurrent import interpreters @@ -192,8 +198,12 @@ def create(*args, **kwargs): interp = _orig_create(*args, **kwargs) try: interp.exec("import _sp_bootstrap\n_sp_bootstrap.install()\n") - except Exception: - pass + except Exception as e: + # Finder not installed in the child: it will hit the original + # ModuleNotFoundError on its first relocated import. Leave a + # breadcrumb (cf. the SP_BOOTSTRAP audit in `install`) instead of + # failing `create` outright. + sys.stderr.write("SP_BOOTSTRAP subinterpreter finder install failed: %r\n" % (e,)) return interp create._sp_patched = True