Build the windows port natively on Windows, from PowerShell, with MSYS2

.gitattributes

@@ -0,0 +1,9 @@
+# LF in the index and in every checkout, Windows included. Git for Windows
+# defaults to core.autocrlf=true, which would check the patches out CRLF, and
+# then not one of them applies to MicroPython's LF tree (and the overlay's
+# series id, a hash of the patch bytes, would differ from Linux's).
+* text=auto eol=lf
+*.patch text eol=lf
+*.sh text eol=lf
+*.jpg binary
+*.h264 binary

README.md

@@ -17,12 +17,17 @@ asks you for port, board and variant. Output lands in
 commit.
 
 Don't have it yet? This clones it into `micropython-pydevices` in the current
-directory and checks your tools (Linux or WSL):
+directory and checks your tools (Linux, WSL, or Git Bash on Windows):
 
 ```bash
 curl -fsSL https://pydevices.github.io/install.sh | sh
 ```
 
+On Windows you can build the windows port natively from PowerShell, with
+`python build_mp.py --port windows --variant pydevices --modules all`, once
+MSYS2 is installed: [On Windows](docs/newcomers.md#on-windows) has the
+steps. Everything else builds from WSL.
+
 New here? The [newcomer's guide](docs/newcomers.md) explains the model; the
 [build plan](docs/build-plan.md) has the layout, the rules and the traps.
 
@@ -41,7 +46,7 @@ New here? The [newcomer's guide](docs/newcomers.md) explains the model; the
   `tflite/` and `vision/` (esp-vision's modules, from the pin in `.esp-vision/`;
   see [docs/esp-vision.md](docs/esp-vision.md)), and the modules themselves,
   linked or cloned.
-- `patches/micropython/` — our patch series (`0001-…` to `0018-…`), each
+- `patches/micropython/` — our patch series (`0001-…` to `0022-…`), each
   justified in its own header; `patches/apply_patches.py` applies it, plus
   each module's own MicroPython patches, once, as one local commit.
 - `variants/` — our variants: `unix/` and `windows/` (`pydevices`,

build_mp.py

@@ -15,11 +15,12 @@
 convention. See docs/build-plan.md.
 
 Environment: OUT_DIR (default builds/), VARIANTS_DIR (default variants/),
-MODULES_DIR (default modules/), JOBS (default: every core).
+MODULES_DIR (default modules/), JOBS (default: every core), and on Windows
+MSYS2_ROOT (default C:\\msys64), the MSYS2 whose make and MinGW gcc build the
+windows port.
 """
 
 import argparse
-import fcntl
 import json
 import os
 import re
@@ -32,6 +33,12 @@
 import urllib.request
 from pathlib import Path
 
+try:
+    import fcntl
+except ImportError:  # Windows: msvcrt's byte-range lock stands in for flock
+    fcntl = None
+    import msvcrt
+
 REPO = Path(__file__).resolve().parent
 OUT_DIR = Path(os.environ.get("OUT_DIR", REPO / "builds")).resolve()
 VARIANTS_DIR = Path(os.environ.get("VARIANTS_DIR", REPO / "variants")).resolve()
@@ -65,6 +72,50 @@ def git_out(path, *args):
     return r.stdout.strip() if r.returncode == 0 else ""
 
 
+# ---- native Windows: MSYS2's make, gcc and bash ---------------------------
+
+# The shell make's recipes run under. On Windows it is resolved by
+# windows_toolchain(), never left to PATH: from PowerShell or cmd, a bare
+# "bash" is WSL's launcher in WindowsApps, which would run the build inside
+# Linux against Windows paths.
+BASH = "bash"
+
+MSYS2_STEPS = """install MSYS2 and its MinGW toolchain, from PowerShell:
+    winget install --id MSYS2.MSYS2 -e
+    C:\\msys64\\usr\\bin\\bash.exe -lc "pacman -Syu --noconfirm"
+    C:\\msys64\\usr\\bin\\bash.exe -lc "pacman -S --needed --noconfirm make mingw-w64-x86_64-gcc autoconf automake libtool"
+then run this again. MSYS2 somewhere other than C:\\msys64? Set MSYS2_ROOT."""
+
+
+def windows_toolchain():
+    """On native Windows, put MSYS2's MinGW and tools first on PATH (for this
+    process and every child) and pick its bash, so a plain PowerShell needs no
+    setup. Returns the MSYS2 root, or None if there isn't one."""
+    global BASH
+    root = Path(os.environ.get("MSYS2_ROOT", r"C:\msys64"))
+    if (root / "usr" / "bin" / "bash.exe").is_file():
+        os.environ["PATH"] = os.pathsep.join([str(root / "mingw64" / "bin"), str(root / "usr" / "bin"), os.environ.get("PATH", "")])
+        BASH = str(root / "usr" / "bin" / "bash.exe")
+    else:
+        root = None
+        found = shutil.which("bash")
+        # WindowsApps' and System32's bash.exe are WSL's.
+        if found and not any(part.lower() in ("windowsapps", "system32") for part in Path(found).parts):
+            BASH = found
+        else:
+            BASH = None
+    # libffi's configure guesses its host from uname, which says MSYS (POSIX
+    # emulation) unless MSYSTEM names the MinGW environment.
+    os.environ.setdefault("MSYSTEM", "MINGW64")
+    return root
+
+
+def check_windows_tools():
+    missing = [t for t in ("make", "gcc") if not shutil.which(t)] + ([] if BASH else ["bash (not WSL's)"])
+    if missing:
+        die(f"no {', '.join(missing)} for the windows port; {MSYS2_STEPS}")
+
+
 # ---- the workspace and what fills micropython/, modules/ and deps/ ---------
 
 def workspace():
@@ -96,11 +147,13 @@ def link(dest, target):
 def clone_at(url, ref, dest, recursive=False):
     say(f"cloning {url} at {ref} into {dest.relative_to(REPO)}")
     run(["git", "init", "-q", str(dest)])
+    # Git for Windows defaults core.autocrlf=true: patches checked out CRLF don't apply.
+    run(["git", "-C", str(dest), "config", "core.autocrlf", "false"])
     run(["git", "-C", str(dest), "remote", "add", "origin", url])
     # A tag is fetched as a tag, so the checkout can still say which one it is.
     spec = ref if re.fullmatch(r"[0-9a-f]{40}", ref) else f"refs/tags/{ref}:refs/tags/{ref}"
     run(["git", "-C", str(dest), "fetch", "-q", "--depth", "1", "origin", spec])
-    run(["git", "-C", str(dest), "checkout", "-q", ref if spec != ref else "FETCH_HEAD"])
+    run(["git", "-C", str(dest), "-c", "advice.detachedHead=false", "checkout", "-q", ref if spec != ref else "FETCH_HEAD"])
     if recursive:
         run(["git", "-C", str(dest), "submodule", "update", "--init", "--recursive", "--depth", "1", "-q"])
 
@@ -113,7 +166,7 @@ def ensure_micropython(ws):
     else:
         upstream = (REPO / "UPSTREAM").read_text().strip()
         say(f"cloning MicroPython {upstream}")
-        run(["git", "clone", "-q", "--depth", "1", "--branch", upstream, "https://github.com/micropython/micropython", str(MP)])
+        run(["git", "-c", "advice.detachedHead=false", "clone", "-q", "--depth", "1", "--branch", upstream, "https://github.com/micropython/micropython", str(MP)])
 
 
 def ensure_modules(ws):
@@ -158,10 +211,10 @@ def ensure_dep(name, ws):
             link(dest, ws / name)
         elif name == "esp-idf":
             clone_at(url, version, dest, recursive=True)
-            run(["bash", "-c", f"cd {dest} && ./install.sh all"])
+            run([BASH, "-c", f"cd {dest} && ./install.sh all"])
         elif name == "emsdk":
             run(["git", "clone", "-q", url, str(dest)])
-            run(["bash", "-c", f"cd {dest} && ./emsdk install {version} && ./emsdk activate {version}"])
+            run([BASH, "-c", f"cd {dest} && ./emsdk install {version} && ./emsdk activate {version}"])
         elif name == "SDL2":
             DEPS.mkdir(exist_ok=True)
             with tempfile.TemporaryDirectory() as tmp:
@@ -459,6 +512,9 @@ def main():
     ap.add_argument("--clean", action="store_true", help="delete this target's build dir first")
     args, make_extra = ap.parse_known_args()
 
+    msys2 = windows_toolchain() if os.name == "nt" else None
+    if msys2:
+        say(f"using MSYS2 at {msys2}")
     ws = workspace()
     ensure_micropython(ws)
     ensure_modules(ws)
@@ -469,17 +525,34 @@ def main():
     # other build tools there take (the anchor's .micropython-build.lock).
     # Held until this process exits.
     lock = open(mp.parent / ".micropython-build.lock", "w")
-    try:
-        fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
-    except BlockingIOError:
-        say(f"waiting for another build to release {lock.name}")
-        fcntl.flock(lock, fcntl.LOCK_EX)
+    if fcntl:
+        try:
+            fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError:
+            say(f"waiting for another build to release {lock.name}")
+            fcntl.flock(lock, fcntl.LOCK_EX)
+    else:
+        try:
+            msvcrt.locking(lock.fileno(), msvcrt.LK_NBLCK, 1)
+        except OSError:
+            say(f"waiting for another build to release {lock.name}")
+            while True:
+                try:  # LK_LOCK gives up after ten seconds, so ask again
+                    msvcrt.locking(lock.fileno(), msvcrt.LK_LOCK, 1)
+                    break
+                except OSError:
+                    pass
     upstream = (REPO / "UPSTREAM").read_text().strip()
     apply_patches.prepare(mp, upstream, apply_patches.series(), refresh=True)
 
     port = args.port or choose("Port:", ports())
     if port not in ports():
         die(f"no port '{port}' (have: {', '.join(ports())})")
+    if os.name == "nt":
+        if port == "windows":
+            check_windows_tools()
+        else:
+            say(f"note: on native Windows only the windows port is tested; build {port} from WSL")
     board = args.board
     if is_board_port(port):
         board = board or choose("Board:", boards(port))
@@ -541,24 +614,30 @@ def main():
     for leak in ("USER_C_MODULES", "FROZEN_MANIFEST", "BUILD", "BOARD", "VARIANT", "BOARD_DIR", "VARIANT_DIR"):
         env.pop(leak, None)
     env["PYDEVICES_MODULES"] = spec
+    if os.name == "nt":
+        # Upstream's tools open sources without encoding=; Windows' default is cp1252.
+        env.setdefault("PYTHONUTF8", "1")
     # A prebuilt mpy-cross, named in the environment: neither make nor CMake
     # then runs the mpy-cross sub-make that would inherit BUILD= (micropython#19667).
-    run(["make", "-C", str(mp / "mpy-cross"), "-j", JOBS], env={k: v for k, v in env.items() if k != "PYDEVICES_MODULES"},
+    # Windows has no python3: give make this Python, in a form MSYS sh keeps.
+    winpy = [f"PYTHON={Path(sys.executable).as_posix()}"] if os.name == "nt" else []
+    run(["make", "-C", str(mp / "mpy-cross"), "-j", JOBS, *winpy], env={k: v for k, v in env.items() if k != "PYDEVICES_MODULES"},
         stdout=subprocess.DEVNULL)
-    env["MICROPY_MPYCROSS"] = str(mp / "mpy-cross" / "build" / "mpy-cross")
+    exe = ".exe" if os.name == "nt" else ""
+    env["MICROPY_MPYCROSS"] = (mp / "mpy-cross" / "build" / f"mpy-cross{exe}").as_posix()
 
     port_dir = mp / "ports" / port
-    make = ["make", "-C", str(port_dir), "-j", JOBS, f"BUILD={build}"]
+    make = ["make", "-C", str(port_dir), "-j", JOBS, f"BUILD={build.as_posix()}", *winpy]
     prefix = ""
     ours = our_variant_dir(port, variant) if not board else None
     manifest = ours / "manifest.py" if ours and (ours / "manifest.py").exists() else MODULES_DIR / "manifest.py"
-    make.append(f"FROZEN_MANIFEST={manifest}")
+    make.append(f"FROZEN_MANIFEST={manifest.as_posix()}")
     if board:
         make.append(f"BOARD={board}")
         if variant:
             make.append(f"BOARD_VARIANT={variant}")
     elif ours:
-        make.append(f"VARIANT_DIR={ours}")
+        make.append(f"VARIANT_DIR={ours.as_posix()}")
     elif variant:
         make.append(f"VARIANT={variant}")
 
@@ -569,16 +648,17 @@ def main():
         emsdk = ensure_dep("emsdk", ws)
         prefix = f'. "{emsdk}/emsdk_env.sh" >/dev/null 2>&1 && '
     elif port == "windows":
-        make.append("CROSS_COMPILE=x86_64-w64-mingw32-")
+        if os.name != "nt":  # on Windows, MSYS2's MinGW gcc is the native compiler
+            make.append("CROSS_COMPILE=x86_64-w64-mingw32-")
         if any(d.resolve().name == "displayif" for d in module_dirs):
-            make.append(f"SDL2_DEV={ensure_dep('SDL2', ws)}")
+            make.append(f"SDL2_DEV={ensure_dep('SDL2', ws).as_posix()}")
     make += make_extra
 
     def shell(cmd, log=None):
         line = prefix + shlex.join(cmd)
         if log is None:
-            return subprocess.run(["bash", "-c", line], env=env).returncode
-        p = subprocess.Popen(["bash", "-c", line], env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
+            return subprocess.run([BASH, "-c", line], env=env).returncode
+        p = subprocess.Popen([BASH, "-c", line], env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
         out = []
         for chunk in p.stdout:
             sys.stdout.write(chunk)

docs/newcomers.md

@@ -8,7 +8,7 @@ build command. It is not a fork, and it publishes no Python package.
 
 Get it, if you haven't: `curl -fsSL https://pydevices.github.io/install.sh | sh`
 clones this repository into `micropython-pydevices` in the current directory
-and checks your tools.
+and checks your tools. On Windows, see [On Windows](#on-windows).
 Then, from the clone:
 
 ```bash
@@ -25,6 +25,53 @@ The desktop `pydevices` variants freeze a board config that imports
 repository whose `lib/` holds it) or `all`; without it `import board_config`
 fails.
 
+## On Windows
+
+You can build the windows port natively, from PowerShell, with MSYS2's
+MinGW toolchain. You need Python from python.org and Git for Windows; the
+rest is a one-time install of MSYS2 and four of its packages:
+
+```powershell
+winget install --id MSYS2.MSYS2 -e
+C:\msys64\usr\bin\bash.exe -lc "pacman -Syu --noconfirm"
+C:\msys64\usr\bin\bash.exe -lc "pacman -S --needed --noconfirm make mingw-w64-x86_64-gcc autoconf automake libtool"
+```
+
+If the first `pacman` line says it updated MSYS2's core and must close, run
+it once more. Then, in any PowerShell:
+
+```powershell
+git clone https://github.com/PyDevices/micropython-pydevices.git
+cd micropython-pydevices
+python build_mp.py --port windows --variant pydevices --modules all
+.\builds\windows\pydevices\micropython.exe
+```
+
+That's all you type. `build_mp.py` finds MSYS2 in `C:\msys64` (set
+`MSYS2_ROOT` if it's elsewhere), puts its tools first on the build's `PATH`,
+and uses its `bash`. Left to `PATH`, `bash` from PowerShell is WSL's, which
+would run the build inside Linux. Without MSYS2 it stops and prints the
+install lines above.
+
+A few things differ from Linux:
+
+- Type `python build_mp.py`, not `./build_mp.py`: Windows doesn't read the
+  `#!` line, and python.org's Python is `python` or `py`, never `python3`.
+  If `python` opens the Microsoft Store, turn off the `python.exe` and
+  `python3.exe` App Execution Aliases (Settings, Apps, Advanced app
+  settings, App execution aliases), or type `py` instead.
+- Use `--variant pydevices` for the Windows display and audio backends
+  (`displaydev.windisplay`, `audiodev.win_audio`). They need `uwin32`, which
+  only that variant freezes, so on `dev` or the default they fail to import.
+- Only the windows port builds natively. Build esp32, rp2, webassembly and
+  unix from WSL, as on Linux.
+- The installer, `curl -fsSL https://pydevices.github.io/install.sh | sh`,
+  runs in Git Bash (Start menu, Git Bash), not PowerShell. A plain
+  `git clone`, as above, does the same job.
+
+The `micropython.exe` it builds needs only Windows' own DLLs, so you can run
+it from any shell, MSYS2 or not.
+
 ## Boards we build for
 
 Every board here is one of upstream's, with at most a small variant of ours

modules.lock

@@ -8,10 +8,10 @@ audiodsp          https://github.com/PyDevices/audiodsp.git           eaec6810c4
 audioif           https://github.com/PyDevices/audioif.git            b6b35a48a0cd23ddc549f8c0643d77ffe778da28
 cameraif          https://github.com/PyDevices/cameraif.git           fdb65caaa587cf46677d275fd0e4aab2fb0a6491
 displayif         https://github.com/PyDevices/displayif.git          17b3750828d5bfc7dc33a6f068e98a17ec3389d6
-lvgl-micropython  https://github.com/PyDevices/lvgl-micropython.git   d42aeff1514987f2fd9d71c79e9737ebda7fa088
+lvgl-micropython  https://github.com/PyDevices/lvgl-micropython.git   2c74ef56a90939206cab840ea720e5026c50e162
 palettes          https://github.com/PyDevices/palettes.git           6f0870f0c8a0b48c56d2150b3f800bbbee7761e1
 pdwidgets         https://github.com/PyDevices/pdwidgets.git          306c1e20e0c3c63066dab6d10155fcf165a6ba13
-pydevices         https://github.com/PyDevices/pydevices.git          ae30bce479daa62da37daca65df2e7d3a4ea0d91
+pydevices         https://github.com/PyDevices/pydevices.git          ab3f7743aa266bd12a758e1780f66ce1948f6851
 pygraphics        https://github.com/PyDevices/pygraphics.git         b4d4c49cc2a22cdd2632f2eea2593b4cc4df4db1
 ulab              https://github.com/v923z/micropython-ulab.git       1d3ddd8f5228e2c7335dac71f4846e6abb0ec9f9
 usbif             https://github.com/PyDevices/usbif.git              95bcb3b29933009299abdf59d6b81cf742e95778

modules/manifest.py

@@ -33,7 +33,7 @@
     try:
         include("$(PORT_DIR)/boards/manifest.py")
     except Exception as _e:
-        if "/boards/manifest.py" not in str(_e):
+        if "/boards/manifest.py" not in str(_e).replace("\\", "/"):
             raise
         # Upstream's default variant for the port, which includes the port's
         # own manifest and adds what the default build carries (asyncio on
@@ -46,7 +46,7 @@
                 _included = True
                 break
             except Exception as _e2:
-                if "/variants/" + _default + "/manifest.py" not in str(_e2):
+                if "/variants/" + _default + "/manifest.py" not in str(_e2).replace("\\", "/"):
                     raise
         if not _included:
             include("$(PORT_DIR)/variants/manifest.py")