From 5bc7cb073bab91709bfc21585677fe1c720169fc Mon Sep 17 00:00:00 2001 From: Brad Barnett <127794626+bdbarnett@users.noreply.github.com> Date: Wed, 7 Oct 2026 12:37:06 -0500 Subject: [PATCH 1/2] Build the windows port natively on Windows, from PowerShell, with MSYS2 python build_mp.py --port windows --variant pydevices --modules all now builds on Windows itself, with MSYS2's make and MinGW gcc, and nothing else to set up. Linux builds are unchanged. build_mp.py: on Windows it finds MSYS2 (C:\msys64, or MSYS2_ROOT), puts its tools first on PATH, sets MSYSTEM=MINGW64 unless it's set (so libffi configures for MinGW, not MSYS), and runs make's shell as MSYS2's bash by full path: a bare "bash" from PowerShell is WSL's launcher. Without make or gcc it stops with the MSYS2 install lines instead of a traceback. It also takes the build lock with msvcrt where there is no fcntl, gives make this Python (Windows has no python3), passes make forward-slash paths (MSYS sh drops backslashes), drops CROSS_COMPILE on a Windows host (there MinGW gcc is the native compiler, and windres has no prefix), finds mpy-cross.exe, sets PYTHONUTF8 (upstream's tools read sources in the ANSI code page), and clones modules with core.autocrlf off. modules/manifest.py: the "not a board port" checks match a Windows path. .gitattributes: LF everywhere. Git for Windows checks out CRLF by default, and then not one patch applies. Overlay: patch 0003's libffi rules chain with && after cd, so a cd that fails can't run configure in ports/windows (it overwrote the port's Makefile). New 0019-0022: C-module paths with forward slashes, and response files on Windows for the three commands that pass its 32767-character limit (the qstr scan, freezing .mpy files, the link). modules.lock: lvgl-micropython and pydevices at their Windows fixes (lvgl-micropython#22, pydevices#128). Docs: newcomers.md "On Windows", and a pointer from the README. --- .gitattributes | 9 ++ README.md | 9 +- build_mp.py | 120 +++++++++++++++--- docs/newcomers.md | 49 ++++++- modules.lock | 4 +- modules/manifest.py | 4 +- patches/apply_patches.py | 10 +- ...hon-windows-enable-ffi-modffi-libffi.patch | 16 ++- ...kemanifest-c-modules-forward-slashes.patch | 37 ++++++ ...krules-qstr-response-file-on-windows.patch | 84 ++++++++++++ ...st-mpy-tool-response-file-on-windows.patch | 72 +++++++++++ ...krules-link-response-file-on-windows.patch | 57 +++++++++ provenance.json | 14 +- 13 files changed, 447 insertions(+), 38 deletions(-) create mode 100644 .gitattributes create mode 100755 patches/micropython/0019-micropython-tools-makemanifest-c-modules-forward-slashes.patch create mode 100755 patches/micropython/0020-micropython-py-mkrules-qstr-response-file-on-windows.patch create mode 100755 patches/micropython/0021-micropython-tools-makemanifest-mpy-tool-response-file-on-windows.patch create mode 100755 patches/micropython/0022-micropython-py-mkrules-link-response-file-on-windows.patch diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..6074f1aa --- /dev/null +++ b/.gitattributes @@ -0,0 +1,9 @@ +# LF in the index and in every checkout, Windows included. Git for Windows +# defaults to core.autocrlf=true, which would check the patches out CRLF, and +# then not one of them applies to MicroPython's LF tree (and the overlay's +# series id, a hash of the patch bytes, would differ from Linux's). +* text=auto eol=lf +*.patch text eol=lf +*.sh text eol=lf +*.jpg binary +*.h264 binary diff --git a/README.md b/README.md index a37902e8..42f3f4db 100644 --- a/README.md +++ b/README.md @@ -17,12 +17,17 @@ asks you for port, board and variant. Output lands in commit. Don't have it yet? This clones it into `micropython-pydevices` in the current -directory and checks your tools (Linux or WSL): +directory and checks your tools (Linux, WSL, or Git Bash on Windows): ```bash curl -fsSL https://pydevices.github.io/install.sh | sh ``` +On Windows you can build the windows port natively from PowerShell, with +`python build_mp.py --port windows --variant pydevices --modules all`, once +MSYS2 is installed: [On Windows](docs/newcomers.md#on-windows) has the +steps. Everything else builds from WSL. + New here? The [newcomer's guide](docs/newcomers.md) explains the model; the [build plan](docs/build-plan.md) has the layout, the rules and the traps. @@ -41,7 +46,7 @@ New here? The [newcomer's guide](docs/newcomers.md) explains the model; the `tflite/` and `vision/` (esp-vision's modules, from the pin in `.esp-vision/`; see [docs/esp-vision.md](docs/esp-vision.md)), and the modules themselves, linked or cloned. -- `patches/micropython/` — our patch series (`0001-…` to `0018-…`), each +- `patches/micropython/` — our patch series (`0001-…` to `0022-…`), each justified in its own header; `patches/apply_patches.py` applies it, plus each module's own MicroPython patches, once, as one local commit. - `variants/` — our variants: `unix/` and `windows/` (`pydevices`, diff --git a/build_mp.py b/build_mp.py index 200bb88b..b358752a 100755 --- a/build_mp.py +++ b/build_mp.py @@ -15,11 +15,12 @@ convention. See docs/build-plan.md. Environment: OUT_DIR (default builds/), VARIANTS_DIR (default variants/), -MODULES_DIR (default modules/), JOBS (default: every core). +MODULES_DIR (default modules/), JOBS (default: every core), and on Windows +MSYS2_ROOT (default C:\\msys64), the MSYS2 whose make and MinGW gcc build the +windows port. """ import argparse -import fcntl import json import os import re @@ -32,6 +33,12 @@ import urllib.request from pathlib import Path +try: + import fcntl +except ImportError: # Windows: msvcrt's byte-range lock stands in for flock + fcntl = None + import msvcrt + REPO = Path(__file__).resolve().parent OUT_DIR = Path(os.environ.get("OUT_DIR", REPO / "builds")).resolve() VARIANTS_DIR = Path(os.environ.get("VARIANTS_DIR", REPO / "variants")).resolve() @@ -65,6 +72,50 @@ def git_out(path, *args): return r.stdout.strip() if r.returncode == 0 else "" +# ---- native Windows: MSYS2's make, gcc and bash --------------------------- + +# The shell make's recipes run under. On Windows it is resolved by +# windows_toolchain(), never left to PATH: from PowerShell or cmd, a bare +# "bash" is WSL's launcher in WindowsApps, which would run the build inside +# Linux against Windows paths. +BASH = "bash" + +MSYS2_STEPS = """install MSYS2 and its MinGW toolchain, from PowerShell: + winget install --id MSYS2.MSYS2 -e + C:\\msys64\\usr\\bin\\bash.exe -lc "pacman -Syu --noconfirm" + C:\\msys64\\usr\\bin\\bash.exe -lc "pacman -S --needed --noconfirm make mingw-w64-x86_64-gcc autoconf automake libtool" +then run this again. MSYS2 somewhere other than C:\\msys64? Set MSYS2_ROOT.""" + + +def windows_toolchain(): + """On native Windows, put MSYS2's MinGW and tools first on PATH (for this + process and every child) and pick its bash, so a plain PowerShell needs no + setup. Returns the MSYS2 root, or None if there isn't one.""" + global BASH + root = Path(os.environ.get("MSYS2_ROOT", r"C:\msys64")) + if (root / "usr" / "bin" / "bash.exe").is_file(): + os.environ["PATH"] = os.pathsep.join([str(root / "mingw64" / "bin"), str(root / "usr" / "bin"), os.environ.get("PATH", "")]) + BASH = str(root / "usr" / "bin" / "bash.exe") + else: + root = None + found = shutil.which("bash") + # WindowsApps' and System32's bash.exe are WSL's. + if found and not any(part.lower() in ("windowsapps", "system32") for part in Path(found).parts): + BASH = found + else: + BASH = None + # libffi's configure guesses its host from uname, which says MSYS (POSIX + # emulation) unless MSYSTEM names the MinGW environment. + os.environ.setdefault("MSYSTEM", "MINGW64") + return root + + +def check_windows_tools(): + missing = [t for t in ("make", "gcc") if not shutil.which(t)] + ([] if BASH else ["bash (not WSL's)"]) + if missing: + die(f"no {', '.join(missing)} for the windows port; {MSYS2_STEPS}") + + # ---- the workspace and what fills micropython/, modules/ and deps/ --------- def workspace(): @@ -96,11 +147,13 @@ def link(dest, target): def clone_at(url, ref, dest, recursive=False): say(f"cloning {url} at {ref} into {dest.relative_to(REPO)}") run(["git", "init", "-q", str(dest)]) + # Git for Windows defaults core.autocrlf=true: patches checked out CRLF don't apply. + run(["git", "-C", str(dest), "config", "core.autocrlf", "false"]) run(["git", "-C", str(dest), "remote", "add", "origin", url]) # A tag is fetched as a tag, so the checkout can still say which one it is. spec = ref if re.fullmatch(r"[0-9a-f]{40}", ref) else f"refs/tags/{ref}:refs/tags/{ref}" run(["git", "-C", str(dest), "fetch", "-q", "--depth", "1", "origin", spec]) - run(["git", "-C", str(dest), "checkout", "-q", ref if spec != ref else "FETCH_HEAD"]) + run(["git", "-C", str(dest), "-c", "advice.detachedHead=false", "checkout", "-q", ref if spec != ref else "FETCH_HEAD"]) if recursive: run(["git", "-C", str(dest), "submodule", "update", "--init", "--recursive", "--depth", "1", "-q"]) @@ -113,7 +166,7 @@ def ensure_micropython(ws): else: upstream = (REPO / "UPSTREAM").read_text().strip() say(f"cloning MicroPython {upstream}") - run(["git", "clone", "-q", "--depth", "1", "--branch", upstream, "https://github.com/micropython/micropython", str(MP)]) + run(["git", "-c", "advice.detachedHead=false", "clone", "-q", "--depth", "1", "--branch", upstream, "https://github.com/micropython/micropython", str(MP)]) def ensure_modules(ws): @@ -158,10 +211,10 @@ def ensure_dep(name, ws): link(dest, ws / name) elif name == "esp-idf": clone_at(url, version, dest, recursive=True) - run(["bash", "-c", f"cd {dest} && ./install.sh all"]) + run([BASH, "-c", f"cd {dest} && ./install.sh all"]) elif name == "emsdk": run(["git", "clone", "-q", url, str(dest)]) - run(["bash", "-c", f"cd {dest} && ./emsdk install {version} && ./emsdk activate {version}"]) + run([BASH, "-c", f"cd {dest} && ./emsdk install {version} && ./emsdk activate {version}"]) elif name == "SDL2": DEPS.mkdir(exist_ok=True) with tempfile.TemporaryDirectory() as tmp: @@ -459,6 +512,9 @@ def main(): ap.add_argument("--clean", action="store_true", help="delete this target's build dir first") args, make_extra = ap.parse_known_args() + msys2 = windows_toolchain() if os.name == "nt" else None + if msys2: + say(f"using MSYS2 at {msys2}") ws = workspace() ensure_micropython(ws) ensure_modules(ws) @@ -469,17 +525,34 @@ def main(): # other build tools there take (the anchor's .micropython-build.lock). # Held until this process exits. lock = open(mp.parent / ".micropython-build.lock", "w") - try: - fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) - except BlockingIOError: - say(f"waiting for another build to release {lock.name}") - fcntl.flock(lock, fcntl.LOCK_EX) + if fcntl: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + say(f"waiting for another build to release {lock.name}") + fcntl.flock(lock, fcntl.LOCK_EX) + else: + try: + msvcrt.locking(lock.fileno(), msvcrt.LK_NBLCK, 1) + except OSError: + say(f"waiting for another build to release {lock.name}") + while True: + try: # LK_LOCK gives up after ten seconds, so ask again + msvcrt.locking(lock.fileno(), msvcrt.LK_LOCK, 1) + break + except OSError: + pass upstream = (REPO / "UPSTREAM").read_text().strip() apply_patches.prepare(mp, upstream, apply_patches.series(), refresh=True) port = args.port or choose("Port:", ports()) if port not in ports(): die(f"no port '{port}' (have: {', '.join(ports())})") + if os.name == "nt": + if port == "windows": + check_windows_tools() + else: + say(f"note: on native Windows only the windows port is tested; build {port} from WSL") board = args.board if is_board_port(port): board = board or choose("Board:", boards(port)) @@ -541,24 +614,30 @@ def main(): for leak in ("USER_C_MODULES", "FROZEN_MANIFEST", "BUILD", "BOARD", "VARIANT", "BOARD_DIR", "VARIANT_DIR"): env.pop(leak, None) env["PYDEVICES_MODULES"] = spec + if os.name == "nt": + # Upstream's tools open sources without encoding=; Windows' default is cp1252. + env.setdefault("PYTHONUTF8", "1") # A prebuilt mpy-cross, named in the environment: neither make nor CMake # then runs the mpy-cross sub-make that would inherit BUILD= (micropython#19667). - run(["make", "-C", str(mp / "mpy-cross"), "-j", JOBS], env={k: v for k, v in env.items() if k != "PYDEVICES_MODULES"}, + # Windows has no python3: give make this Python, in a form MSYS sh keeps. + winpy = [f"PYTHON={Path(sys.executable).as_posix()}"] if os.name == "nt" else [] + run(["make", "-C", str(mp / "mpy-cross"), "-j", JOBS, *winpy], env={k: v for k, v in env.items() if k != "PYDEVICES_MODULES"}, stdout=subprocess.DEVNULL) - env["MICROPY_MPYCROSS"] = str(mp / "mpy-cross" / "build" / "mpy-cross") + exe = ".exe" if os.name == "nt" else "" + env["MICROPY_MPYCROSS"] = (mp / "mpy-cross" / "build" / f"mpy-cross{exe}").as_posix() port_dir = mp / "ports" / port - make = ["make", "-C", str(port_dir), "-j", JOBS, f"BUILD={build}"] + make = ["make", "-C", str(port_dir), "-j", JOBS, f"BUILD={build.as_posix()}", *winpy] prefix = "" ours = our_variant_dir(port, variant) if not board else None manifest = ours / "manifest.py" if ours and (ours / "manifest.py").exists() else MODULES_DIR / "manifest.py" - make.append(f"FROZEN_MANIFEST={manifest}") + make.append(f"FROZEN_MANIFEST={manifest.as_posix()}") if board: make.append(f"BOARD={board}") if variant: make.append(f"BOARD_VARIANT={variant}") elif ours: - make.append(f"VARIANT_DIR={ours}") + make.append(f"VARIANT_DIR={ours.as_posix()}") elif variant: make.append(f"VARIANT={variant}") @@ -569,16 +648,17 @@ def main(): emsdk = ensure_dep("emsdk", ws) prefix = f'. "{emsdk}/emsdk_env.sh" >/dev/null 2>&1 && ' elif port == "windows": - make.append("CROSS_COMPILE=x86_64-w64-mingw32-") + if os.name != "nt": # on Windows, MSYS2's MinGW gcc is the native compiler + make.append("CROSS_COMPILE=x86_64-w64-mingw32-") if any(d.resolve().name == "displayif" for d in module_dirs): - make.append(f"SDL2_DEV={ensure_dep('SDL2', ws)}") + make.append(f"SDL2_DEV={ensure_dep('SDL2', ws).as_posix()}") make += make_extra def shell(cmd, log=None): line = prefix + shlex.join(cmd) if log is None: - return subprocess.run(["bash", "-c", line], env=env).returncode - p = subprocess.Popen(["bash", "-c", line], env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + return subprocess.run([BASH, "-c", line], env=env).returncode + p = subprocess.Popen([BASH, "-c", line], env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) out = [] for chunk in p.stdout: sys.stdout.write(chunk) diff --git a/docs/newcomers.md b/docs/newcomers.md index 1d58335c..2efcdde0 100644 --- a/docs/newcomers.md +++ b/docs/newcomers.md @@ -8,7 +8,7 @@ build command. It is not a fork, and it publishes no Python package. Get it, if you haven't: `curl -fsSL https://pydevices.github.io/install.sh | sh` clones this repository into `micropython-pydevices` in the current directory -and checks your tools. +and checks your tools. On Windows, see [On Windows](#on-windows). Then, from the clone: ```bash @@ -25,6 +25,53 @@ The desktop `pydevices` variants freeze a board config that imports repository whose `lib/` holds it) or `all`; without it `import board_config` fails. +## On Windows + +You can build the windows port natively, from PowerShell, with MSYS2's +MinGW toolchain. You need Python from python.org and Git for Windows; the +rest is a one-time install of MSYS2 and four of its packages: + +```powershell +winget install --id MSYS2.MSYS2 -e +C:\msys64\usr\bin\bash.exe -lc "pacman -Syu --noconfirm" +C:\msys64\usr\bin\bash.exe -lc "pacman -S --needed --noconfirm make mingw-w64-x86_64-gcc autoconf automake libtool" +``` + +If the first `pacman` line says it updated MSYS2's core and must close, run +it once more. Then, in any PowerShell: + +```powershell +git clone https://github.com/PyDevices/micropython-pydevices.git +cd micropython-pydevices +python build_mp.py --port windows --variant pydevices --modules all +.\builds\windows\pydevices\micropython.exe +``` + +That's all you type. `build_mp.py` finds MSYS2 in `C:\msys64` (set +`MSYS2_ROOT` if it's elsewhere), puts its tools first on the build's `PATH`, +and uses its `bash`. Left to `PATH`, `bash` from PowerShell is WSL's, which +would run the build inside Linux. Without MSYS2 it stops and prints the +install lines above. + +A few things differ from Linux: + +- Type `python build_mp.py`, not `./build_mp.py`: Windows doesn't read the + `#!` line, and python.org's Python is `python` or `py`, never `python3`. + If `python` opens the Microsoft Store, turn off the `python.exe` and + `python3.exe` App Execution Aliases (Settings, Apps, Advanced app + settings, App execution aliases), or type `py` instead. +- Use `--variant pydevices` for the Windows display and audio backends + (`displaydev.windisplay`, `audiodev.win_audio`). They need `uwin32`, which + only that variant freezes, so on `dev` or the default they fail to import. +- Only the windows port builds natively. Build esp32, rp2, webassembly and + unix from WSL, as on Linux. +- The installer, `curl -fsSL https://pydevices.github.io/install.sh | sh`, + runs in Git Bash (Start menu, Git Bash), not PowerShell. A plain + `git clone`, as above, does the same job. + +The `micropython.exe` it builds needs only Windows' own DLLs, so you can run +it from any shell, MSYS2 or not. + ## Boards we build for Every board here is one of upstream's, with at most a small variant of ours diff --git a/modules.lock b/modules.lock index 536379a4..2706738b 100644 --- a/modules.lock +++ b/modules.lock @@ -8,10 +8,10 @@ audiodsp https://github.com/PyDevices/audiodsp.git eaec6810c4 audioif https://github.com/PyDevices/audioif.git b6b35a48a0cd23ddc549f8c0643d77ffe778da28 cameraif https://github.com/PyDevices/cameraif.git fdb65caaa587cf46677d275fd0e4aab2fb0a6491 displayif https://github.com/PyDevices/displayif.git 17b3750828d5bfc7dc33a6f068e98a17ec3389d6 -lvgl-micropython https://github.com/PyDevices/lvgl-micropython.git d42aeff1514987f2fd9d71c79e9737ebda7fa088 +lvgl-micropython https://github.com/PyDevices/lvgl-micropython.git e43d7ea7f9c638822b1434f73c25f366d64a51c7 palettes https://github.com/PyDevices/palettes.git 6f0870f0c8a0b48c56d2150b3f800bbbee7761e1 pdwidgets https://github.com/PyDevices/pdwidgets.git 306c1e20e0c3c63066dab6d10155fcf165a6ba13 -pydevices https://github.com/PyDevices/pydevices.git ae30bce479daa62da37daca65df2e7d3a4ea0d91 +pydevices https://github.com/PyDevices/pydevices.git 48aae1eb2ffaad6b3788bd2850573c8c7212d10d pygraphics https://github.com/PyDevices/pygraphics.git b4d4c49cc2a22cdd2632f2eea2593b4cc4df4db1 ulab https://github.com/v923z/micropython-ulab.git 1d3ddd8f5228e2c7335dac71f4846e6abb0ec9f9 usbif https://github.com/PyDevices/usbif.git 95bcb3b29933009299abdf59d6b81cf742e95778 diff --git a/modules/manifest.py b/modules/manifest.py index fae627f6..f1c95459 100644 --- a/modules/manifest.py +++ b/modules/manifest.py @@ -33,7 +33,7 @@ try: include("$(PORT_DIR)/boards/manifest.py") except Exception as _e: - if "/boards/manifest.py" not in str(_e): + if "/boards/manifest.py" not in str(_e).replace("\\", "/"): raise # Upstream's default variant for the port, which includes the port's # own manifest and adds what the default build carries (asyncio on @@ -46,7 +46,7 @@ _included = True break except Exception as _e2: - if "/variants/" + _default + "/manifest.py" not in str(_e2): + if "/variants/" + _default + "/manifest.py" not in str(_e2).replace("\\", "/"): raise if not _included: include("$(PORT_DIR)/variants/manifest.py") diff --git a/patches/apply_patches.py b/patches/apply_patches.py index a4e9e88a..a25f5e13 100755 --- a/patches/apply_patches.py +++ b/patches/apply_patches.py @@ -63,7 +63,7 @@ def apply(mp, patches): for patch in patches: if git(mp, "apply", str(patch), check=False) != 0: sys.exit(f"apply_patches: {patch.relative_to(REPO)} does not apply") - print(f"applied {patch.relative_to(REPO)}") + print(f"applied {patch.relative_to(REPO)}", flush=True) def check(mp, upstream, patches): @@ -77,7 +77,7 @@ def check(mp, upstream, patches): apply(tree, patches) finally: git(mp, "worktree", "remove", "--force", str(tree), check=False) - print(f"the series applies clean to {upstream}") + print(f"the series applies clean to {upstream}", flush=True) def prepare(mp, upstream, patches, refresh=False): @@ -88,7 +88,7 @@ def prepare(mp, upstream, patches, refresh=False): if git(mp, "log", "-1", "--format=%s", capture=True) == mark and refresh and sid not in git( mp, "log", "-1", "--format=%b", capture=True ).splitlines(): - print(f"{mp} carries an older overlay; moving it back to {upstream} and preparing again") + print(f"{mp} carries an older overlay; moving it back to {upstream} and preparing again", flush=True) git(mp, "checkout", "--quiet", "--", "ports/esp32/lockfiles", check=False) git(mp, "checkout", "--quiet", upstream) if git(mp, "log", "-1", "--format=%s", capture=True) == mark: @@ -98,7 +98,7 @@ def prepare(mp, upstream, patches, refresh=False): f"and prepare again:\n git -C {mp} checkout {upstream} && {sys.argv[0]} {mp}" ) head = git(mp, "log", "-1", "--format=%h", capture=True) - print(f"overlay already applied: {head} on {upstream}") + print(f"overlay already applied: {head} on {upstream}", flush=True) else: dirty = [ line @@ -116,7 +116,7 @@ def prepare(mp, upstream, patches, refresh=False): "commit", "--quiet", "-m", mark, "-m", sid, ) head = git(mp, "log", "-1", "--format=%h", capture=True) - print(f"overlay applied: {head} on {upstream}") + print(f"overlay applied: {head} on {upstream}", flush=True) # Every manifest require()s from micropython-lib, so a clone without it # cannot even list its C modules. A port's other submodules come from # `make submodules`, the upstream way. diff --git a/patches/micropython/0003-micropython-windows-enable-ffi-modffi-libffi.patch b/patches/micropython/0003-micropython-windows-enable-ffi-modffi-libffi.patch index 036a3608..2799f506 100644 --- a/patches/micropython/0003-micropython-windows-enable-ffi-modffi-libffi.patch +++ b/patches/micropython/0003-micropython-windows-enable-ffi-modffi-libffi.patch @@ -5,6 +5,12 @@ Subject: [PATCH] windows: enable ffi (modffi, libffi) on Windows (MinGW) Build libffi for the windows port and enable modffi, matching the unix port. +The libffi rules chain their steps with && rather than the unix port's ;, +so a step that fails stops the rule. With ;, a cd into the build dir that +failed (a Windows path whose backslashes the shell dropped) ran libffi's +configure in ports/windows instead, which overwrote the port's tracked +Makefile and left configure's output in the tree. + Based-on: micropython 0fd6c573e (tag v1.29.0) --- ports/unix/modffi.c | 76 +++++++++++++++++++++++++++++++---- @@ -160,7 +166,7 @@ index c16d40ad3..c60c681e6 100644 mp_obj_ffimod_t *o = mp_obj_malloc(mp_obj_ffimod_t, type); o->handle = mod; diff --git a/ports/windows/Makefile b/ports/windows/Makefile -index e22ea225d..4b6b18270 100644 +index e22ea225d..c6a8ec8b3 100644 --- a/ports/windows/Makefile +++ b/ports/windows/Makefile @@ -96,6 +96,21 @@ endif @@ -206,12 +212,12 @@ index e22ea225d..4b6b18270 100644 +libffi: $(BUILD)/lib/libffi/include/ffi.h + +$(TOP)/lib/libffi/configure: $(TOP)/lib/libffi/autogen.sh -+ cd $(TOP)/lib/libffi; ./autogen.sh ++ cd $(TOP)/lib/libffi && ./autogen.sh + +$(BUILD)/lib/libffi/include/ffi.h: $(TOP)/lib/libffi/configure -+ mkdir -p $(BUILD)/lib/libffi; cd $(BUILD)/lib/libffi; \ -+ $(abspath $(TOP))/lib/libffi/configure $(CROSS_COMPILE_HOST) --prefix=$$PWD/out --disable-shared --disable-structs CC="$(CC)" CXX="$(CXX)" LD="$(LD)" CFLAGS="-Os -fomit-frame-pointer -fstrict-aliasing -ffast-math -fno-exceptions"; \ -+ $(MAKE) install-exec-recursive; $(MAKE) -C include install-data-am ++ mkdir -p $(BUILD)/lib/libffi && cd $(BUILD)/lib/libffi && \ ++ $(abspath $(TOP))/lib/libffi/configure $(CROSS_COMPILE_HOST) --prefix=$$PWD/out --disable-shared --disable-structs CC="$(CC)" CXX="$(CXX)" LD="$(LD)" CFLAGS="-Os -fomit-frame-pointer -fstrict-aliasing -ffast-math -fno-exceptions" && \ ++ $(MAKE) install-exec-recursive && $(MAKE) -C include install-data-am + # Note for recent gcc versions like 13.2: # - mingw64-x86_64 gcc builds will pass the math_domain_special test diff --git a/patches/micropython/0019-micropython-tools-makemanifest-c-modules-forward-slashes.patch b/patches/micropython/0019-micropython-tools-makemanifest-c-modules-forward-slashes.patch new file mode 100755 index 00000000..734f39d5 --- /dev/null +++ b/patches/micropython/0019-micropython-tools-makemanifest-c-modules-forward-slashes.patch @@ -0,0 +1,37 @@ +From 12478b29ebf389a7a913c505f9c5dbea17c19911 Mon Sep 17 00:00:00 2001 +From: Brad Barnett <127794626+bdbarnett@users.noreply.github.com> +Date: Tue, 6 Oct 2026 22:53:01 -0500 +Subject: [PATCH] tools/makemanifest.py: list C modules with forward slashes. + +--list-c-modules prints each c_module() path as manifestfile resolved it, +with os.path.abspath, so on Windows it carries backslashes. py/manifest.mk +reads that list into USER_C_MODULES, and a module's micropython.mk that +runs a script from $(USERMOD_DIR) through the shell gets the path with its +backslashes dropped (C:\Users\... becomes C:Users...). Python and make both +take forward slashes on Windows, and the list is unchanged elsewhere. + +Seen on a native Windows (MSYS2 MinGW) build of ports/windows with +audiodsp, whose micropython.mk runs $(USERMOD_DIR)/scripts/fetch_deps.sh. +Unchanged on upstream v1.29.0. +--- + tools/makemanifest.py | 4 +++- + 1 file changed, 3 insertions(+), 1 deletion(-) + +diff --git a/tools/makemanifest.py b/tools/makemanifest.py +index a149755..5f7903d 100644 +--- a/tools/makemanifest.py ++++ b/tools/makemanifest.py +@@ -176,7 +176,9 @@ def main(): + if c_modules: + # Output one path per line to handle paths with spaces + for module in c_modules: +- print(module) ++ # Forward slashes: make hands these to sh, which drops a ++ # Windows path's backslashes. ++ print(module.replace(os.sep, "/")) + sys.exit(0) + + # Get paths to tools +-- +2.54.0.windows.1 + diff --git a/patches/micropython/0020-micropython-py-mkrules-qstr-response-file-on-windows.patch b/patches/micropython/0020-micropython-py-mkrules-qstr-response-file-on-windows.patch new file mode 100755 index 00000000..9ef4107f --- /dev/null +++ b/patches/micropython/0020-micropython-py-mkrules-qstr-response-file-on-windows.patch @@ -0,0 +1,84 @@ +From 3807e8cf45e32c0bf7bd79030afcabdd8b7ed3c3 Mon Sep 17 00:00:00 2001 +From: Brad Barnett <127794626+bdbarnett@users.noreply.github.com> +Date: Tue, 6 Oct 2026 23:01:11 -0500 +Subject: [PATCH] py/mkrules.mk: pass makeqstrdefs.py its file lists in a + response file on Windows. + +Windows caps a command line at 32767 characters. The qstr pp rule names +every QSTR source on it, twice on a fresh build ($^ as sources and $? as +changed_sources), as the absolute paths user C modules give. A windows +port build with --modules all made a 52058-character command, and make +failed with "python.exe: Argument list too long". + +On Windows ($(OS) is Windows_NT) the rule now writes the sources, +dependencies and changed_sources lists to genhdr/qstr.i.args with +$(file) and passes @; makeqstrdefs.py expands an @file +argument into its words. The flags stay on the command line, since +some carry shell quoting a file would not. $(file) needs GNU make 4.0, +which MSYS2 has; every other host keeps the command line it had, so +make 3.81 (macOS) is unaffected. + +Seen on a native Windows (MSYS2 MinGW) build of ports/windows with +audiodsp, ulab, displayif and the rest of micropython-pydevices' +modules. Unchanged on upstream v1.29.0. +--- + py/makeqstrdefs.py | 10 ++++++++++ + py/mkrules.mk | 13 ++++++++++++- + 2 files changed, 22 insertions(+), 1 deletion(-) + +diff --git a/py/makeqstrdefs.py b/py/makeqstrdefs.py +index 628fbe6..f34eaaa 100644 +--- a/py/makeqstrdefs.py ++++ b/py/makeqstrdefs.py +@@ -222,7 +222,17 @@ if __name__ == "__main__": + ] + } + ++ # An @file argument is a response file: its words stand in its place ++ # (py/mkrules.mk passes the source lists that way on Windows). ++ argv = [] + for arg in sys.argv[1:]: ++ if arg.startswith("@"): ++ with open(arg[1:]) as f: ++ argv.extend(f.read().split()) ++ else: ++ argv.append(arg) ++ ++ for arg in argv: + if arg in named_args: + current_tok = arg + else: +diff --git a/py/mkrules.mk b/py/mkrules.mk +index a04cd14..788f1f9 100644 +--- a/py/mkrules.mk ++++ b/py/mkrules.mk +@@ -139,6 +139,17 @@ $(BUILD)/%.o: $(BUILD)/%.c + # to get built before we try to compile any of them. + $(OBJ): | $(HEADER_BUILD)/qstrdefs.generated.h $(HEADER_BUILD)/mpversion.h $(OBJ_EXTRA_ORDER_DEPS) + ++# The file lists for makeqstrdefs.py pp. Windows caps a command line at 32767 ++# characters, which a build with large user C modules passes (every source is ++# named twice on a fresh build: $^ and $?), so there they go in a response file ++# that makeqstrdefs.py expands. $(file) needs GNU make 4.0; Windows' (MSYS2) has ++# it, and other hosts keep the command line as it was. ++ifeq ($(OS),Windows_NT) ++QSTR_PP_FILES = $(file >$(HEADER_BUILD)/qstr.i.args,sources $^ dependencies $(QSTR_GLOBAL_DEPENDENCIES) changed_sources $?)@$(HEADER_BUILD)/qstr.i.args ++else ++QSTR_PP_FILES = sources $^ dependencies $(QSTR_GLOBAL_DEPENDENCIES) changed_sources $? ++endif ++ + # The logic for qstr regeneration (applied by makeqstrdefs.py) is: + # - if anything in QSTR_GLOBAL_DEPENDENCIES is newer, then process all source files ($^) + # - else, if list of newer prerequisites ($?) is not empty, then process just these ($?) +@@ -146,7 +157,7 @@ $(OBJ): | $(HEADER_BUILD)/qstrdefs.generated.h $(HEADER_BUILD)/mpversion.h $(OBJ + # See more information about this process in docs/develop/qstr.rst. + $(HEADER_BUILD)/qstr.i.last: $(SRC_QSTR) $(QSTR_GLOBAL_DEPENDENCIES) | $(QSTR_GLOBAL_REQUIREMENTS) + $(ECHO) "GEN $@" +- $(Q)$(PYTHON) $(PY_SRC)/makeqstrdefs.py pp $(CPP) output $(HEADER_BUILD)/qstr.i.last cflags $(QSTR_GEN_CFLAGS) cxxflags $(QSTR_GEN_CXXFLAGS) sources $^ dependencies $(QSTR_GLOBAL_DEPENDENCIES) changed_sources $? ++ $(Q)$(PYTHON) $(PY_SRC)/makeqstrdefs.py pp $(CPP) output $(HEADER_BUILD)/qstr.i.last cflags $(QSTR_GEN_CFLAGS) cxxflags $(QSTR_GEN_CXXFLAGS) $(QSTR_PP_FILES) + + $(HEADER_BUILD)/qstr.split: $(HEADER_BUILD)/qstr.i.last + $(ECHO) "GEN $@" +-- +2.54.0.windows.1 + diff --git a/patches/micropython/0021-micropython-tools-makemanifest-mpy-tool-response-file-on-windows.patch b/patches/micropython/0021-micropython-tools-makemanifest-mpy-tool-response-file-on-windows.patch new file mode 100755 index 00000000..30ab7901 --- /dev/null +++ b/patches/micropython/0021-micropython-tools-makemanifest-mpy-tool-response-file-on-windows.patch @@ -0,0 +1,72 @@ +From add79c24a6d16b42ada7251c98310b124aeea2b1 Mon Sep 17 00:00:00 2001 +From: Brad Barnett <127794626+bdbarnett@users.noreply.github.com> +Date: Tue, 6 Oct 2026 23:18:38 -0500 +Subject: [PATCH] tools/makemanifest.py: pass mpy-tool.py its .mpy files in a + response file on Windows. + +Windows caps a command line at 32767 characters. makemanifest.py hands +mpy-tool.py every frozen .mpy by path on one command line, and a windows +port build with micropython-pydevices' --modules all froze 361 of them +(about 41000 characters of paths), so CreateProcess failed with +"[WinError 206] The filename or extension is too long". + +mpy-tool.py now takes argparse's @file (one argument per line), and on +Windows makemanifest.py writes the .mpy list to +/frozen_mpy.args and passes @. Every other host keeps +the command line it had. + +Seen on a native Windows (MSYS2 MinGW) build of ports/windows. +Unchanged on upstream v1.29.0. +--- + tools/makemanifest.py | 10 +++++++++- + tools/mpy-tool.py | 6 +++++- + 2 files changed, 14 insertions(+), 2 deletions(-) + +diff --git a/tools/makemanifest.py b/tools/makemanifest.py +index 5f7903d..3f05a30 100644 +--- a/tools/makemanifest.py ++++ b/tools/makemanifest.py +@@ -245,6 +245,14 @@ def main(): + + # Freeze .mpy files + if mpy_files: ++ mpy_args = mpy_files ++ if os.name == "nt": ++ # Windows caps a command line at 32767 characters, which a few hundred ++ # frozen files' paths pass; mpy-tool.py reads them from a file instead. ++ list_file = os.path.join(args.build_dir, "frozen_mpy.args") ++ with open(list_file, "w") as f: ++ f.write("\n".join(mpy_files) + "\n") ++ mpy_args = ["@" + list_file] + res, output_mpy = system( + [ + sys.executable, +@@ -254,7 +262,7 @@ def main(): + args.build_dir + "/genhdr/qstrdefs.preprocessed.h", + ] + + args.mpy_tool_flags.split() +- + mpy_files ++ + mpy_args + ) + if res != 0: + print("error freezing mpy {}:".format(mpy_files)) +diff --git a/tools/mpy-tool.py b/tools/mpy-tool.py +index efe9bfe..811d76d 100755 +--- a/tools/mpy-tool.py ++++ b/tools/mpy-tool.py +@@ -2088,7 +2088,11 @@ def main(args=None): + + import argparse + +- cmd_parser = argparse.ArgumentParser(description="A tool to work with MicroPython .mpy files.") ++ cmd_parser = argparse.ArgumentParser( ++ description="A tool to work with MicroPython .mpy files.", ++ # @file names a file of arguments, one per line (makemanifest.py on Windows). ++ fromfile_prefix_chars="@", ++ ) + cmd_parser.add_argument( + "-x", "--hexdump", action="store_true", help="output an annotated hex dump of files" + ) +-- +2.54.0.windows.1 + diff --git a/patches/micropython/0022-micropython-py-mkrules-link-response-file-on-windows.patch b/patches/micropython/0022-micropython-py-mkrules-link-response-file-on-windows.patch new file mode 100755 index 00000000..4ae2a492 --- /dev/null +++ b/patches/micropython/0022-micropython-py-mkrules-link-response-file-on-windows.patch @@ -0,0 +1,57 @@ +From 3f4147195ff73471352363a8eb650f0ba6af3681 Mon Sep 17 00:00:00 2001 +From: Brad Barnett <127794626+bdbarnett@users.noreply.github.com> +Date: Tue, 6 Oct 2026 23:23:42 -0500 +Subject: [PATCH] py/mkrules.mk: pass the linker its objects in a response file + on Windows. + +Windows caps a command line at 32767 characters. The program's link rule +names every object on it, and a windows port build with +micropython-pydevices' --modules all links 948 objects by absolute path +(about 100000 characters; LVGL alone is 463), so make failed with +"gcc: Argument list too long". + +On Windows ($(OS) is Windows_NT) the rule now writes the objects to +$(BUILD)/$(PROG).objs with $(file) and passes @, which gcc +reads natively. $(LIBS) and $(LDFLAGS) stay on the command line. $(file) +needs GNU make 4.0, which MSYS2 has; every other host keeps the command +line it had, so make 3.81 (macOS) is unaffected. + +The same limit as the qstr pp rule's and makemanifest.py's mpy-tool.py +call, met the same way. + +Seen on a native Windows (MSYS2 MinGW) build of ports/windows. +Unchanged on upstream v1.29.0. +--- + py/mkrules.mk | 12 +++++++++++- + 1 file changed, 11 insertions(+), 1 deletion(-) + +diff --git a/py/mkrules.mk b/py/mkrules.mk +index 788f1f9..19acea6 100644 +--- a/py/mkrules.mk ++++ b/py/mkrules.mk +@@ -253,11 +253,21 @@ endif + + all: $(BUILD)/$(PROG) + ++# The objects for the link. Windows caps a command line at 32767 characters, ++# which the objects of large user C modules pass, so there they go in a ++# response file, which gcc reads as @file. $(file) needs GNU make 4.0; ++# Windows' (MSYS2) has it, and other hosts keep the command line as it was. ++ifeq ($(OS),Windows_NT) ++PROG_OBJ = $(file >$(BUILD)/$(PROG).objs,$^)@$(BUILD)/$(PROG).objs ++else ++PROG_OBJ = $^ ++endif ++ + $(BUILD)/$(PROG): $(OBJ) + $(ECHO) "LINK $@" + # Do not pass COPT here - it's *C* compiler optimizations. For example, + # we may want to compile using Thumb, but link with non-Thumb libc. +- $(Q)$(CC) -o $@ $^ $(LIBS) $(LDFLAGS) ++ $(Q)$(CC) -o $@ $(PROG_OBJ) $(LIBS) $(LDFLAGS) + ifndef DEBUG + ifdef STRIP + $(Q)$(STRIP) $(STRIPFLAGS_EXTRA) $@ +-- +2.54.0.windows.1 + diff --git a/provenance.json b/provenance.json index 9d98f236..d90e6725 100644 --- a/provenance.json +++ b/provenance.json @@ -9,7 +9,7 @@ "sha256": "b3827e1dea6b189d401c58d9addbda62389c94edc2043a38d8347abcbcaa0818" }, "0003-micropython-windows-enable-ffi-modffi-libffi.patch": { - "sha256": "b3e3c34aa84dc24a622dc1448d7ff08ec5f387693308a0412542f41f6d988d1f" + "sha256": "7fe6e6bedf604b86161473bc98eb5feeb899803ab0211fae1ad93ad278779118" }, "0004-micropython-webassembly-allow-Asyncify-in-runPythonAsync.patch": { "sha256": "563e85fcd9f2c6762f8d78a6d2caae091ad3c3a130d041f5854629a4e99320f0" @@ -55,6 +55,18 @@ }, "0018-micropython-esp32-drop-missing-usermod-include-dirs.patch": { "sha256": "ed968c8c24147056cad48642612fe9eed9a8fff051b64705ee587c548a8144b9" + }, + "0019-micropython-tools-makemanifest-c-modules-forward-slashes.patch": { + "sha256": "0d0cc9dfad8d3d07b762337cd7eb95ef43eee12d8c380f7cb405d480b2723956" + }, + "0020-micropython-py-mkrules-qstr-response-file-on-windows.patch": { + "sha256": "04d2f0f01cd14ed67481066681c08002ec6baa8d7f40508bec0b27bf4c1d928f" + }, + "0021-micropython-tools-makemanifest-mpy-tool-response-file-on-windows.patch": { + "sha256": "72d976eaeac044a053139f5a41c06b4e7f75f5defbc738635e3599e9bbbab207" + }, + "0022-micropython-py-mkrules-link-response-file-on-windows.patch": { + "sha256": "b4d522a23779986dae282a4ad54f9e7f2984c49727c84dfbae484fb1d4fe4816" } } } From e70406cb908835cdcda213972930a4133449fc83 Mon Sep 17 00:00:00 2001 From: Brad Barnett <127794626+bdbarnett@users.noreply.github.com> Date: Wed, 7 Oct 2026 13:04:48 -0500 Subject: [PATCH 2/2] modules.lock: lvgl-micropython and pydevices at the merge commits of lvgl-micropython#22 and pydevices#128 --- modules.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules.lock b/modules.lock index 2706738b..f442123d 100644 --- a/modules.lock +++ b/modules.lock @@ -8,10 +8,10 @@ audiodsp https://github.com/PyDevices/audiodsp.git eaec6810c4 audioif https://github.com/PyDevices/audioif.git b6b35a48a0cd23ddc549f8c0643d77ffe778da28 cameraif https://github.com/PyDevices/cameraif.git fdb65caaa587cf46677d275fd0e4aab2fb0a6491 displayif https://github.com/PyDevices/displayif.git 17b3750828d5bfc7dc33a6f068e98a17ec3389d6 -lvgl-micropython https://github.com/PyDevices/lvgl-micropython.git e43d7ea7f9c638822b1434f73c25f366d64a51c7 +lvgl-micropython https://github.com/PyDevices/lvgl-micropython.git 2c74ef56a90939206cab840ea720e5026c50e162 palettes https://github.com/PyDevices/palettes.git 6f0870f0c8a0b48c56d2150b3f800bbbee7761e1 pdwidgets https://github.com/PyDevices/pdwidgets.git 306c1e20e0c3c63066dab6d10155fcf165a6ba13 -pydevices https://github.com/PyDevices/pydevices.git 48aae1eb2ffaad6b3788bd2850573c8c7212d10d +pydevices https://github.com/PyDevices/pydevices.git ab3f7743aa266bd12a758e1780f66ce1948f6851 pygraphics https://github.com/PyDevices/pygraphics.git b4d4c49cc2a22cdd2632f2eea2593b4cc4df4db1 ulab https://github.com/v923z/micropython-ulab.git 1d3ddd8f5228e2c7335dac71f4846e6abb0ec9f9 usbif https://github.com/PyDevices/usbif.git 95bcb3b29933009299abdf59d6b81cf742e95778