diff --git a/.gitignore b/.gitignore index 2a8dd6a7c3eabd..657a149505a793 100644 --- a/.gitignore +++ b/.gitignore @@ -139,6 +139,7 @@ Tools/unicode/data/ /config.status.lineno /.ccache /cross-build*/ +/dist/ /jit_stencils*.h /jit_unwind_info*.h .jit-stamp diff --git a/Platforms/WASI/__main__.py b/Platforms/WASI/__main__.py index 880058bad660be..58cbc44834dd28 100644 --- a/Platforms/WASI/__main__.py +++ b/Platforms/WASI/__main__.py @@ -1,14 +1,19 @@ -#!/usr/bin/env python3 - -__lazy_modules__ = ["_build"] +__lazy_modules__ = [ + "argparse", + "os", + "pathlib", + "_build", + "_package", + "_shared", +] import argparse import os import pathlib import _build - -HERE = pathlib.Path(__file__).parent +import _package +import _shared def main(): @@ -27,6 +32,7 @@ def main(): # may want to use them. "--config {WASMTIME_CONFIG_PATH}" ) + context = _shared.Context() parser = argparse.ArgumentParser() subcommands = parser.add_subparsers(dest="subcommand") @@ -59,6 +65,9 @@ def main(): pythoninfo_host = subcommands.add_parser( "pythoninfo-host", help="Display build info of the host/WASI Python" ) + package = subcommands.add_parser( + "package", help="Package the host/WASI Python into an archive" + ) subcommands.add_parser( "clean", help="Delete files and directories created by this script" ) @@ -84,6 +93,8 @@ def main(): "--logdir", type=pathlib.Path, default=None, + dest="_log_path", + metavar="LOG-DIR", help="Directory to store log files", ) for subcommand in ( @@ -113,8 +124,9 @@ def main(): subcommand.add_argument( "--wasi-sdk", type=pathlib.Path, - dest="wasi_sdk_path", + dest="_wasi_sdk_path", default=None, + metavar="WASI-SDK-PATH", help="Path to the WASI SDK; defaults to WASI_SDK_PATH environment variable " "or the appropriate version found in /opt", ) @@ -132,16 +144,19 @@ def main(): make_host, build_host, pythoninfo_host, + package, ): subcommand.add_argument( "--host-triple", action="store", default=None, + dest="_host_triple", + metavar="WASI-TRIPLE", help="The target triple for the WASI host build; " - f"defaults to the value found in {os.fsdecode(HERE / 'config.toml')}", + f"defaults to the value found in {os.fsdecode(context.here / 'config.toml')}", ) - context = parser.parse_args() + parser.parse_args(namespace=context) match context.subcommand: case "configure-build-python": @@ -166,14 +181,19 @@ def main(): # Configure and build the build Python _build.configure_build_python(context) _build.make_build_python(context) - _build.pythoninfo_build_python(context) + if not context.quiet: + _build.pythoninfo_build_python(context) # Configure and build the host/WASI Python _build.configure_wasi_python(context) _build.make_wasi_python(context) - _build.pythoninfo_wasi_python(context) + if not context.quiet: + _build.pythoninfo_wasi_python(context) case "clean": _build.clean_contents(context) + case "package": + _package.gather(context) + _package.archive(context) case None: parser.print_help() case _: diff --git a/Platforms/WASI/_build.py b/Platforms/WASI/_build.py index 792f8edd994313..66e2ae64e181d7 100644 --- a/Platforms/WASI/_build.py +++ b/Platforms/WASI/_build.py @@ -1,6 +1,13 @@ -#!/usr/bin/env python3 - -__lazy_modules__ = ["shutil", "sys", "tempfile", "tomllib"] +__lazy_modules__ = [ + "contextlib", + "functools", + "os", + "pathlib", + "shutil", + "subprocess", + "sys", + "tempfile", +] import contextlib import functools @@ -9,32 +16,15 @@ import shutil import subprocess import sys -import sysconfig import tempfile -import tomllib try: from os import process_cpu_count as cpu_count except ImportError: from os import cpu_count +import _shared -CHECKOUT = HERE = pathlib.Path(__file__).parent - -while CHECKOUT != CHECKOUT.parent: - if (CHECKOUT / "configure").is_file(): - break - CHECKOUT = CHECKOUT.parent -else: - raise FileNotFoundError( - "Unable to find the root of the CPython checkout by looking for 'configure'" - ) - -CROSS_BUILD_DIR = CHECKOUT / "cross-build" -# Build platform can also be found via `config.guess`. -BUILD_DIR = CROSS_BUILD_DIR / sysconfig.get_config_var("BUILD_GNU_TYPE") - -LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" LOCAL_SETUP_MARKER = ( b"# Generated by Platforms/WASI .\n" b"# Required to statically build extension modules." @@ -57,17 +47,6 @@ def separator(): print("โŽฏ" * terminal_width) -def log(emoji, message, *, spacing=None): - """Print a notification with an emoji. - - If 'spacing' is None, calculate the spacing based on the number of code points - in the emoji as terminals "eat" a space when the emoji has multiple code points. - """ - if spacing is None: - spacing = " " if len(emoji) == 1 else " " - print("".join([emoji, spacing, message])) - - def updated_env(updates={}): """Create a new dict representing the environment to use. @@ -94,29 +73,24 @@ def updated_env(updates={}): env_vars = [ f"\n {key}={item}" for key, item in sorted(env_diff.items()) ] - log("๐ŸŒŽ", f"Environment changes:{''.join(env_vars)}") + _shared.log("๐ŸŒŽ", f"Environment changes:{''.join(env_vars)}") return environment -def subdir(working_dir, *, clean_ok=False): +def subdir(context_attr, *, clean_ok=False): """Decorator to change to a working directory.""" def decorator(func): @functools.wraps(func) def wrapper(context): - nonlocal working_dir + nonlocal context_attr - if callable(working_dir): - working_dir = working_dir(context) + working_dir = getattr(context, context_attr) separator() - log("๐Ÿ“", os.fsdecode(working_dir)) - if ( - clean_ok - and getattr(context, "clean", False) - and working_dir.exists() - ): - log("๐Ÿšฎ", "Deleting directory (--clean)...") + _shared.log("๐Ÿ“", os.fsdecode(working_dir)) + if clean_ok and context.clean and working_dir.exists(): + _shared.log("๐Ÿšฎ", "Deleting directory (--clean)...") shutil.rmtree(working_dir) working_dir.mkdir(parents=True, exist_ok=True) @@ -137,75 +111,55 @@ def call(command, *, context=None, quiet=False, **kwargs): if context is not None: quiet = context.quiet - log("โฏ", " ".join(map(str, command)), spacing=" ") + _shared.log("โฏ", " ".join(map(str, command)), spacing=" ") if not quiet: stdout = None stderr = None else: - if (logdir := getattr(context, "logdir", None)) is None: - logdir = pathlib.Path(tempfile.gettempdir()) + if (log_path := getattr(context, "log_path", None)) is None: + log_path = pathlib.Path(tempfile.gettempdir()) stdout = tempfile.NamedTemporaryFile( "w", encoding="utf-8", delete=False, - dir=logdir, + dir=log_path, prefix="cpython-wasi-", suffix=".log", ) stderr = subprocess.STDOUT - log("๐Ÿ“", f"Logging output to {stdout.name} (--quiet)...") + _shared.log("๐Ÿ“", f"Logging output to {stdout.name} (--quiet)...") subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr) -def build_python_path(): - """The path to the build Python binary.""" - binary = BUILD_DIR / "python" - if not binary.is_file(): - binary = binary.with_suffix(".exe") - if not binary.is_file(): - raise FileNotFoundError( - f"Unable to find `python(.exe)` in {BUILD_DIR}" - ) - - return binary - - -def build_python_is_pydebug(): - """Find out if the build Python is a pydebug build.""" - test = "import sys, test.support; sys.exit(test.support.Py_DEBUG)" - result = subprocess.run( - [build_python_path(), "-c", test], - capture_output=True, - ) - return bool(result.returncode) - - -@subdir(BUILD_DIR, clean_ok=True) +@subdir("build_python_path", clean_ok=True) def configure_build_python(context, working_dir): """Configure the build/host Python.""" - if LOCAL_SETUP.exists(): - if LOCAL_SETUP.read_bytes() == LOCAL_SETUP_MARKER: - log("๐Ÿ‘", f"{LOCAL_SETUP} exists ...") + if context.setup_local_path.exists(): + if context.setup_local_path.read_bytes() == LOCAL_SETUP_MARKER: + _shared.log("๐Ÿ‘", f"{context.setup_local_path} exists ...") else: - log("โš ๏ธ", f"{LOCAL_SETUP} exists, but has unexpected contents") + _shared.log( + "โš ๏ธ", + f"{context.setup_local_path} exists, but has unexpected contents", + ) else: - log("๐Ÿ“", f"Creating {LOCAL_SETUP} ...") - LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER) + _shared.log("๐Ÿ“", f"Creating {context.setup_local_path} ...") + context.setup_local_path.write_bytes(LOCAL_SETUP_MARKER) - configure = [os.path.relpath(CHECKOUT / "configure", working_dir)] + configure = [os.path.relpath(context.checkout / "configure", working_dir)] if context.args: configure.extend(context.args) call(configure, context=context) -@subdir(BUILD_DIR) -def make_build_python(context, working_dir): +@subdir("build_python_path") +def make_build_python(context, _working_dir): """Make/build the build Python.""" call(["make", "--jobs", str(cpu_count()), "all"], context=context) - binary = build_python_path() + binary = context.build_python_interpreter cmd = [ binary, "-c", @@ -214,80 +168,12 @@ def make_build_python(context, working_dir): ] version = subprocess.check_output(cmd, encoding="utf-8").strip() - log("๐ŸŽ‰", f"{binary} {version}") - - -def wasi_sdk(context): - """Find the path to the WASI SDK.""" - if wasi_sdk_path := context.wasi_sdk_path: - if not wasi_sdk_path.exists(): - raise ValueError( - "WASI SDK not found at " - f"{os.fsdecode(wasi_sdk_path)!r} (via --wasi-sdk)" - ) - return wasi_sdk_path - - with (HERE / "config.toml").open("rb") as file: - config = tomllib.load(file) - wasi_sdk_version = config["targets"]["wasi-sdk"] - - if wasi_sdk_path_env_var := os.environ.get("WASI_SDK_PATH"): - wasi_sdk_path = pathlib.Path(wasi_sdk_path_env_var) - if not wasi_sdk_path.exists(): - raise ValueError( - f"WASI SDK not found at {os.fsdecode(wasi_sdk_path)!r} " - "(via $WASI_SDK_PATH)" - ) - else: - opt_path = pathlib.Path("/opt") - # WASI SDK versions have a ``.0`` suffix, but it's a constant; the WASI SDK team - # has said they don't plan to ever do a point release and all of their Git tags - # lack the ``.0`` suffix. - # Starting with WASI SDK 23, the tarballs went from containing a directory named - # ``wasi-sdk-{WASI_SDK_VERSION}.0`` to e.g. - # ``wasi-sdk-{WASI_SDK_VERSION}.0-x86_64-linux``. - potential_sdks = [ - path - for path in opt_path.glob(f"wasi-sdk-{wasi_sdk_version}.0*") - if path.is_dir() - ] - if len(potential_sdks) == 1: - wasi_sdk_path = potential_sdks[0] - elif (default_path := opt_path / "wasi-sdk").is_dir(): - wasi_sdk_path = default_path - - # Starting with WASI SDK 25, a VERSION file is included in the root - # of the SDK directory that we can read to warn folks when they are using - # an unsupported version. - if wasi_sdk_path and (version_file := wasi_sdk_path / "VERSION").is_file(): - version_details = version_file.read_text(encoding="utf-8") - found_version = version_details.splitlines()[0] - # Make sure there's a trailing dot to avoid false positives if somehow the - # supported version is a prefix of the found version (e.g. `25` and `2567`). - if not found_version.startswith(f"{wasi_sdk_version}."): - major_version = found_version.partition(".")[0] - log( - "โš ๏ธ", - f" Found WASI SDK {major_version}, " - f"but WASI SDK {wasi_sdk_version} is the supported version", - ) - elif not wasi_sdk_path: - raise ValueError( - f"WASI SDK {wasi_sdk_version} not found; " - "download from " - "https://github.com/WebAssembly/wasi-sdk and install in " - f"{os.fsdecode(opt_path)!r} or specify the SDK via " - "$WASI_SDK_PATH or --wasi-sdk" - ) - - # Cache the result. - context.wasi_sdk_path = wasi_sdk_path - return wasi_sdk_path + _shared.log("๐ŸŽ‰", f"{binary} {version}") def wasi_sdk_env(context): """Calculate environment variables for building with wasi-sdk.""" - wasi_sdk_path = wasi_sdk(context) + wasi_sdk_path = context.wasi_sdk_path sysroot = wasi_sdk_path / "share" / "wasi-sysroot" env = { "CC": "clang", @@ -324,31 +210,18 @@ def wasi_sdk_env(context): return env -def host_triple(context): - """Determine the target triple for the WASI host build.""" - if context.host_triple: - return context.host_triple - - with (HERE / "config.toml").open("rb") as file: - config = tomllib.load(file) - - # Cache the result. - context.host_triple = config["targets"]["host-triple"] - return context.host_triple - - -@subdir(lambda context: CROSS_BUILD_DIR / host_triple(context), clean_ok=True) +@subdir("wasi_build_path", clean_ok=True) def configure_wasi_python(context, working_dir): """Configure the WASI/host build.""" - config_site = os.fsdecode(HERE / "config.site-wasm32-wasi") + config_site = os.fsdecode(context.here / "config.site-wasm32-wasi") - wasi_build_dir = working_dir.relative_to(CHECKOUT) + wasi_build_dir = working_dir.relative_to(context.checkout) args = { "WASMTIME": "wasmtime", "ARGV0": f"/{wasi_build_dir}/python.wasm", - "CHECKOUT": os.fsdecode(CHECKOUT), - "WASMTIME_CONFIG_PATH": os.fsdecode(HERE / "wasmtime.toml"), + "CHECKOUT": os.fsdecode(context.checkout), + "WASMTIME_CONFIG_PATH": os.fsdecode(context.here / "wasmtime.toml"), } # Check dynamically for wasmtime in case it was specified manually via # `--host-runner`. @@ -362,17 +235,17 @@ def configure_wasi_python(context, working_dir): ) host_runner = context.host_runner.format_map(args) env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner} - build_python = os.fsdecode(build_python_path()) + build_python = os.fsdecode(context.build_python_interpreter) # The path to `configure` MUST be relative, else `python.wasm` is unable # to find the stdlib due to Python not recognizing that it's being - # executed from within a checkout. + # executed from within context.checkout. configure = [ - os.path.relpath(CHECKOUT / "configure", working_dir), - f"--host={host_triple(context)}", - f"--build={BUILD_DIR.name}", + os.path.relpath(context.checkout / "configure", working_dir), + f"--host={context.host_triple}", + f"--build={context.build_python_path.name}", f"--with-build-python={build_python}", ] - if build_python_is_pydebug(): + if context.is_debug: configure.append("--with-pydebug") if context.args: configure.extend(context.args) @@ -387,11 +260,11 @@ def configure_wasi_python(context, working_dir): with exec_script.open("w", encoding="utf-8") as file: file.write(f'#!/bin/sh\nexec {host_runner} {python_wasm} "$@"\n') exec_script.chmod(0o755) - log("๐Ÿƒ", f"Created {exec_script} (--host-runner)... ") + _shared.log("๐Ÿƒ", f"Created {exec_script} (--host-runner)... ") sys.stdout.flush() -@subdir(lambda context: CROSS_BUILD_DIR / host_triple(context)) +@subdir("wasi_build_path") def make_wasi_python(context, working_dir): """Run `make` for the WASI/host build.""" call( @@ -401,8 +274,8 @@ def make_wasi_python(context, working_dir): ) exec_script = working_dir / "python.sh" - call([exec_script, "--version"], quiet=False) - log( + call([exec_script, "-c", "import sys; print(sys.version)"], quiet=False) + _shared.log( "๐ŸŽ‰", f"Use `{exec_script.relative_to(pathlib.Path().absolute())}` " "to run CPython w/ the WASI host specified by --host-runner", @@ -411,22 +284,26 @@ def make_wasi_python(context, working_dir): def clean_contents(context): """Delete all files created by this script.""" - if CROSS_BUILD_DIR.exists(): - log("๐Ÿงน", f"Deleting {CROSS_BUILD_DIR} ...") - shutil.rmtree(CROSS_BUILD_DIR) - - if LOCAL_SETUP.exists(): - if LOCAL_SETUP.read_bytes() == LOCAL_SETUP_MARKER: - log("๐Ÿงน", f"Deleting generated {LOCAL_SETUP} ...") + context.clean = True + if context.cross_build_path.exists(): + _shared.log("๐Ÿงน", f"Deleting {context.cross_build_path} ...") + shutil.rmtree(context.cross_build_path) + + if context.setup_local_path.exists(): + if context.setup_local_path.read_bytes() == LOCAL_SETUP_MARKER: + _shared.log( + "๐Ÿงน", f"Deleting generated {context.setup_local_path} ..." + ) + context.setup_local_path.unlink() -@subdir(BUILD_DIR) +@subdir("build_python_path") def pythoninfo_build_python(context, working_dir): """Display build info of the build Python.""" call(["make", "pythoninfo"], context=context) -@subdir(lambda context: CROSS_BUILD_DIR / host_triple(context)) +@subdir("wasi_build_path") def pythoninfo_wasi_python(context, working_dir): """Display build info of the host/WASI Python.""" call(["make", "pythoninfo"], context=context) diff --git a/Platforms/WASI/_package.py b/Platforms/WASI/_package.py new file mode 100644 index 00000000000000..37119799c233a2 --- /dev/null +++ b/Platforms/WASI/_package.py @@ -0,0 +1,420 @@ +__lazy_modules__ = [ + "datetime", + "os", + "pathlib", + "shutil", + "subprocess", + "_shared", +] + +import datetime +import os +import pathlib +import shutil +import subprocess + +import _shared + + +def wasmtime_script(context): + return f"""\ +#!/bin/sh + +set -eu + +script_dir=$(CDPATH= cd "$(dirname "$0")" && pwd -P) +root=$(CDPATH= cd "$script_dir/.." && pwd -P) +wasm_file="$root/lib/python{python_version(context)}/lib-wasm/python{python_version(context, debug_ok=True)}.wasm" + +exec wasmtime run \\ + --argv0 "$wasm_file" \\ + --config "$root/etc/python{python_version(context)}/wasmtime.toml" \\ + --dir "/" \\ + "$wasm_file" "$@" +""" + + +def python_version(context, debug_ok=False): + """Calculate the M.N part of Python's version. + + If *debug_ok* is True, then "d" is appended as appropriate. + """ + details = context.wasi_build_details + major = details["language"]["version_info"]["major"] + minor = details["language"]["version_info"]["minor"] + version = f"{major}.{minor}" + if debug_ok and context.is_debug: + version += "d" + return version + + +def lib_python(context): + return pathlib.PurePath("lib") / f"python{python_version(context)}" + + +def license_file(context): + """Have /LICENSE end up as lib/pythonXY/LICENSE.txt.""" + return (lib_python(context) / "LICENSE.txt", context.checkout / "LICENSE") + + +def build_dir_files(context): + """Have build/lib.* files end up in lib/pythonXY. + + Symlinks are skipped as those files are covered by handling + /Modules. + """ + return [ + (lib_python(context) / path.name, path) + for path in context.wasi_pybuilddir.iterdir() + if path.is_file(follow_symlinks=False) + ] + + +def stdlib_files(context): + """Have /Lib files end up in lib/pythonXY.""" + lib_dir = context.checkout / "Lib" + lib_files = [] + for root, dirs, files in lib_dir.walk(): + try: + dirs.remove("__pycache__") + except ValueError: + pass + + for file in files: + file_path = pathlib.Path(root) / file + details = ( + lib_python(context) / file_path.relative_to(lib_dir), + file_path, + ) + lib_files.append(details) + return lib_files + + +def pkgconfig_files(context): + """Have /Misc/python*.pc end up in lib/pkgconfig. + + Each file ends up being listed under `python3` and `python-3.N`. + """ + misc_dir = context.wasi_build_path / "Misc" + pkgconfig = pathlib.PurePath("lib") / "pkgconfig" + return [ + ( + pkgconfig / f"python-{python_version(context, debug_ok=True)}.pc", + misc_dir / "python.pc", + ), + ( + pkgconfig + / f"python-{python_version(context, debug_ok=True)}-embed.pc", + misc_dir / "python-embed.pc", + ), + ] + + +def pkgconfig_symlinks(pkgconfig_files, context): + assert len(pkgconfig_files) == 2 + embed, plain = ( + (0, 1) if pkgconfig_files[0].name.endswith("-embed.pc") else (1, 0) + ) + embed_path, plain_path = pkgconfig_files[embed], pkgconfig_files[plain] + major = context.wasi_build_details["language"]["version_info"]["major"] + paths = [ + (embed_path.parent / f"python{major}-embed.pc", embed_path), + (plain_path.parent / f"python{major}.pc", plain_path), + ] + if context.is_debug: + paths += [ + ( + embed_path.parent + / f"python-{python_version(context)}-embed.pc", + embed_path, + ), + ( + plain_path.parent / f"python-{python_version(context)}.pc", + plain_path, + ), + ] + + return paths + + +def pyconfig_file(context): + """Have /pyconfig.h end up in include/pythonXYd?/pyconfig.h.""" + return ( + pathlib.PurePath("include") + / f"python{python_version(context, debug_ok=True)}" + / "pyconfig.h", + context.wasi_build_path / "pyconfig.h", + ) + + +def header_files(context): + """Have /Include/*.h end up in include/pythonXYd?/.""" + include_dir = context.checkout / "Include" + files = [] + for root, dirs, filenames in include_dir.walk(): + for filename in filenames: + file_path = pathlib.Path(root) / filename + details = ( + pathlib.PurePath("include") + / f"python{python_version(context, debug_ok=True)}" + / file_path.relative_to(include_dir), + file_path, + ) + files.append(details) + return files + + +def man_file(context): + """Have Misc/python.man end up in share/man/man1/.""" + man_dir = pathlib.PurePath("share", "man", "man1") + man_file = context.checkout / "Misc" / "python.man" + return ( + man_dir / f"python{python_version(context)}.1", + man_file, + ) + + +def man_symlink(man_path, context): + """Symlink pythonN.M.1 to pythonN.1.""" + major = context.wasi_build_details["language"]["version_info"]["major"] + return (man_path.parent / f"python{major}.1", man_path) + + +def wasmtime_config_file(context): + """Have wasmtime.toml end up in etc/pythonXY/.""" + config = context.checkout / "Platforms" / "WASI" / "wasmtime.toml" + return ( + pathlib.PurePath("etc") + / f"python{python_version(context)}" + / "wasmtime.toml", + config, + ) + + +def wasm_file(context): + """Have python.wasm end up at lib/pythonXY/lib-wasm/pythonX.Yd?.wasm.""" + return ( + lib_python(context) + / "lib-wasm" + / f"python{python_version(context, debug_ok=True)}.wasm", + context.wasi_build_path / "python.wasm", + ) + + +def python_wasmtime_script(base, context): + """Create bin/pythonN.Md?.wasmtime.""" + script = wasmtime_script(context) + path = ( + base / f"bin/python{python_version(context, debug_ok=True)}.wasmtime" + ) + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w") as f: + f.write(script) + path.chmod(0o755) + + return path + + +def python_wasmtime_symlink(path, context): + """Symlink bin/pythonN.Md?.wasmtime. + + - bin/pythonN.M.wasmtime (if debug build) + - bin/pythonN.wasmtime (if debug build) + """ + symlinks = [ + path.parent / f"python{context.wasi_build_version['major']}.wasmtime" + ] + if context.is_debug: + # The file already has the debug name, so the missing symlink is the non-debug name. + symlinks.append( + path.parent / f"python{python_version(context)}.wasmtime" + ) + + return [(symlink, path) for symlink in symlinks] + + +def config_file(context): + """Have Misc/config.sh end up at bin/pythonN.Md?.config.""" + return ( + pathlib.PurePath("bin") + / f"python{python_version(context, debug_ok=True)}-config", + context.wasi_build_path / "Misc" / "python-config.sh", + ) + + +def config_symlink(config_path, context): + """Symlink bin/pythonN.Md?-config. + + - bin/pythonN.M-config (if debug build) + - bin/pythonNd?-config + - bin/pythonN-config (if debug build) + """ + symlinks = [ + config_path.parent + / f"python{context.wasi_build_version['major']}-config" + ] + if context.is_debug: + # The file already has the debug name, so the missing symlink is the non-debug name. + symlinks.append( + config_path.parent / f"python{python_version(context)}-config" + ) + + return [(symlink, config_path) for symlink in symlinks] + + +def filename_stem(context): + """Calculate the stem of the archive file name.""" + version_info = context.wasi_build_details["language"]["version_info"] + version = f"python-{version_info['major']}.{version_info['minor']}.{version_info['micro']}" + if version_info["releaselevel"] != "final": + version += version_info["releaselevel"][0] + str( + version_info["serial"] + ) + + return f"{version}-{context.host_triple}" + + +def copy_files(files, base): + for dest, src in files: + target = base / dest + target.parent.mkdir(parents=True, exist_ok=True) + src.copy(target) + + +def symlink_files(files, base): + for target, source in files: + target_path = base / target + source_path = base / source + target_path.symlink_to( + os.path.relpath(source_path, start=target_path.parent) + ) + + +def gather(context): + py_version = python_version(context) + py_d_version = python_version(context, debug_ok=True) + + dist = context.checkout / "dist" + if dist.exists(): + _shared.log("๐Ÿงน", f"Deleting {dist} ...") + shutil.rmtree(dist) + + indent = " " + base = dist / filename_stem(context) + _shared.log("๐Ÿ“", f"Copying files to {base} ...") + + _shared.log("๐Ÿ“", "bin/", spacing=indent * 2) + _shared.log("๐Ÿ“„", f"python{py_d_version}-config", spacing=indent * 3) + config_location = config_file(context) + copy_files([config_location], base) + symlink_files(config_symlink(config_location[0], context), base) + + _shared.log("๐Ÿ“„", f"python{py_d_version}.wasmtime", spacing=indent * 3) + script_path = python_wasmtime_script(base, context) + symlink_files(python_wasmtime_symlink(script_path, context), base) + + _shared.log("๐Ÿ“", "etc/", spacing=indent * 2) + _shared.log("๐Ÿ“", f"python{py_version}/", spacing=indent * 3) + _shared.log("๐Ÿ“„", "wasmtime.toml", spacing=indent * 4) + copy_files([wasmtime_config_file(context)], base) + + _shared.log("๐Ÿ“", "include/", spacing=indent * 2) + _shared.log("๐Ÿ“", f"python{py_version}/", spacing=indent * 3) + _shared.log("๐Ÿ“„", "pyconfig.h", spacing=indent * 4) + copy_files([pyconfig_file(context)], base) + _shared.log("๐Ÿ“„", "**/*.h", spacing=indent * 4) + copy_files(header_files(context), base) + + _shared.log("๐Ÿ“", "lib/", spacing=indent * 2) + _shared.log("๐Ÿ“", f"python{py_version}/", spacing=indent * 3) + _shared.log("๐Ÿ“", "lib-dynload/", spacing=indent * 4) + (base / lib_python(context) / "lib-dynload").mkdir( + parents=True, exist_ok=True + ) + _shared.log("๐Ÿ“„", "", spacing=indent * 5) + + _shared.log("๐Ÿ“", "lib-wasm/", spacing=indent * 4) + _shared.log("๐Ÿ“„", f"python{py_d_version}.wasm", spacing=indent * 5) + copy_files([wasm_file(context)], base) + _shared.log("๐Ÿ“„", "LICENSE.txt", spacing=indent * 4) + copy_files([license_file(context)], base) + _shared.log("๐Ÿ“„", "files in `cat pybuilddir.txt`", spacing=indent * 4) + copy_files(build_dir_files(context), base) + _shared.log("๐Ÿ“„", "**/*.py", spacing=indent * 4) + copy_files(stdlib_files(context), base) + + _shared.log("๐Ÿ“", "pkgconfig/", spacing=indent * 3) + _shared.log("๐Ÿ“„", "python*.pc", spacing=indent * 4) + pkgconfig_paths = pkgconfig_files(context) + copy_files(pkgconfig_paths, base) + symlink_files( + pkgconfig_symlinks([path for path, _ in pkgconfig_paths], context), + base, + ) + _shared.log("๐Ÿ“", "share/", spacing=indent * 2) + _shared.log("๐Ÿ“", "man/", spacing=indent * 3) + _shared.log("๐Ÿ“", "man1/", spacing=indent * 4) + _shared.log("๐Ÿ“„", "python*.1", spacing=indent * 5) + man_path = man_file(context) + copy_files([man_path], base) + symlink_files([man_symlink(man_path[0], context)], base) + + return base + + +def archive(context): + file_name = f"{filename_stem(context)}.tar.xz" + file_path = context.checkout / "dist" / file_name + if file_path.exists(): + _shared.log("๐Ÿงน", f"Deleting {file_path} ...") + file_path.unlink() + to_compress = context.checkout / "dist" / filename_stem(context) + _shared.log("๐Ÿ—œ๏ธ", f"Archiving to {file_path} ...") + mtime_format = "%Y-%m-%dT%H:%M:%SZ" + if source_date_epoch := os.environ.get("SOURCE_DATE_EPOCH"): + mtime = datetime.datetime.fromtimestamp( + int(source_date_epoch), datetime.UTC + ).strftime(mtime_format) + else: + mtime = subprocess.run( + [ + "git", + "log", + "-1", + "--format=tformat:%cd", + f"--date=format:{mtime_format}", + os.fsdecode(context.checkout), + ], + env={"TZ": "UTC0"}, + capture_output=True, + text=True, + check=True, + ).stdout.strip() + + subprocess.run( + [ + "tar", + "-c", + "-f", + os.fsdecode(file_path), + "--sort=name", + "--mtime", + mtime, + "--clamp-mtime", + "--owner=0", + "--group=0", + "--numeric-owner", + "--pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime", + "--mode=go+u,go-w", + # Explicitly using `-T` because if you don't compress with threads you can't + # uncompress with them and the size difference is negligible when using + # single-threaded compression. + "--use-compress-program", + "xz -T 0", + to_compress.name, + ], + cwd=to_compress.parent, + capture_output=True, + text=True, + check=True, + ) diff --git a/Platforms/WASI/_shared.py b/Platforms/WASI/_shared.py new file mode 100644 index 00000000000000..ee1a65859a1a4d --- /dev/null +++ b/Platforms/WASI/_shared.py @@ -0,0 +1,200 @@ +__lazy_modules__ = [ + "functools", + "json", + "os", + "pathlib", + "subprocess", + "sysconfig", + "tempfile", + "tomllib", +] + +import functools +import json +import os +import pathlib +import subprocess +import sysconfig +import tempfile +import tomllib + + +class Context: + clean = False + + def __init__(self): + self.here = pathlib.Path(__file__).parent + + @functools.cached_property + def checkout(self): + checkout = self.here + while checkout != checkout.parent: + if (checkout / "configure").is_file(): + return checkout + checkout = checkout.parent + raise FileNotFoundError( + "Unable to find the root of the CPython checkout by looking for 'configure'" + ) + + def _pybuilddir(self, build_path): + relative_dir = (build_path / "pybuilddir.txt").read_text().strip() + return build_path / relative_dir + + @functools.cached_property + def setup_local_path(self): + return self.checkout / "Modules" / "Setup.local" + + @functools.cached_property + def host_triple(self): + if self._host_triple: + return self._host_triple + + with (self.here / "config.toml").open("rb") as file: + config = tomllib.load(file) + + return config["targets"]["host-triple"] + + @functools.cached_property + def cross_build_path(self): + return self.checkout / "cross-build" + + @functools.cached_property + def build_python_path(self): + # Build platform can also be found via `config.guess`. + return self.cross_build_path / sysconfig.get_config_var( + "BUILD_GNU_TYPE" + ) + + @functools.cached_property + def build_python_interpreter(self): + binary = self.build_python_path / "python" + if not binary.is_file(): + binary = binary.with_suffix(".exe") + if not binary.is_file(): + raise FileNotFoundError( + f"Unable to find `python(.exe)` in {self.build_python_path}" + ) + + return binary + + @functools.cached_property + def is_debug(self): + pybuilddir = self._pybuilddir(self.build_python_path) + if pybuilddir.exists(): + build_details_path = pybuilddir / "build-details.json" + build_details = json.loads(build_details_path.read_text()) + return "d" in build_details["abi"]["flags"] + else: + # Python 3.13 and older. + test = "import sys, test.support; sys.exit(test.support.Py_DEBUG)" + result = subprocess.run( + [self.build_python_interpreter, "-c", test], + capture_output=True, + ) + return bool(result.returncode) + + @functools.cached_property + def wasi_build_path(self): + return self.cross_build_path / self.host_triple + + @functools.cached_property + def wasi_pybuilddir(self): + return self._pybuilddir(self.wasi_build_path) + + @functools.cached_property + def wasi_build_details(self): + with (self.wasi_pybuilddir / "build-details.json").open() as f: + return json.load(f) + + @functools.cached_property + def wasi_build_version(self): + return self.wasi_build_details["language"]["version_info"] + + @functools.cached_property + def wasi_sdk_path(self): + if wasi_sdk_path := self._wasi_sdk_path: + if not wasi_sdk_path.exists(): + raise ValueError( + "WASI SDK not found; " + "download from " + "https://github.com/WebAssembly/wasi-sdk and/or " + "specify via $WASI_SDK_PATH or --wasi-sdk" + ) + return wasi_sdk_path + + with (self.here / "config.toml").open("rb") as file: + config = tomllib.load(file) + wasi_sdk_version = config["targets"]["wasi-sdk"] + + if wasi_sdk_path_env_var := os.environ.get("WASI_SDK_PATH"): + wasi_sdk_path = pathlib.Path(wasi_sdk_path_env_var) + if not wasi_sdk_path.exists(): + raise ValueError( + f"WASI SDK not found at $WASI_SDK_PATH ({wasi_sdk_path})" + ) + else: + opt_path = pathlib.Path("/opt") + # WASI SDK versions have a ``.0`` suffix, but it's a constant; the WASI SDK team + # has said they don't plan to ever do a point release and all of their Git tags + # lack the ``.0`` suffix. + # Starting with WASI SDK 23, the tarballs went from containing a directory named + # ``wasi-sdk-{WASI_SDK_VERSION}.0`` to e.g. + # ``wasi-sdk-{WASI_SDK_VERSION}.0-x86_64-linux``. + potential_sdks = [ + path + for path in opt_path.glob(f"wasi-sdk-{wasi_sdk_version}.0*") + if path.is_dir() + ] + if not potential_sdks: + raise ValueError( + f"WASI SDK {wasi_sdk_version} not found in {opt_path}" + ) + elif len(potential_sdks) == 1: + wasi_sdk_path = potential_sdks[0] + elif (default_path := opt_path / "wasi-sdk").is_dir(): + wasi_sdk_path = default_path + elif potential_sdks: + raise ValueError( + f"Multiple WASI SDKs found in {opt_path} w/o knowing which to use" + ) + else: + raise ValueError(f"WASI SDK not found in {opt_path}") + + # Starting with WASI SDK 25, a VERSION file is included in the root + # of the SDK directory that we can read to warn folks when they are using + # an unsupported version. + if ( + wasi_sdk_path + and (version_file := wasi_sdk_path / "VERSION").is_file() + ): + version_details = version_file.read_text(encoding="utf-8") + found_version = version_details.splitlines()[0] + # Make sure there's a trailing dot to avoid false positives if somehow the + # supported version is a prefix of the found version (e.g. `25` and `2567`). + if not found_version.startswith(f"{wasi_sdk_version}."): + major_version = found_version.partition(".")[0] + log( + "โš ๏ธ", + f" Found WASI SDK {major_version}, " + f"but WASI SDK {wasi_sdk_version} is the supported version", + ) + + return wasi_sdk_path + + @functools.cached_property + def log_path(self): + if self._log_path is not None: + return self._log_path + + return pathlib.Path(tempfile.gettempdir()) + + +def log(emoji, message, *, spacing=None): + """Print a notification with an emoji. + + If 'spacing' is None, calculate the spacing based on the number of code points + in the emoji as terminals "eat" a space when the emoji has multiple code points. + """ + if spacing is None: + spacing = " " if len(emoji) == 1 else " " + print("".join([emoji, spacing, message]))