diff --git a/cpython-unix/build.py b/cpython-unix/build.py index 8e773a188..e7e944280 100755 --- a/cpython-unix/build.py +++ b/cpython-unix/build.py @@ -19,6 +19,7 @@ from pythonbuild.buildenv import build_environment from pythonbuild.cpython import ( STDLIB_TEST_PACKAGES, + configured_extension_modules, derive_setup_local, extension_modules_config, meets_python_maximum_version, @@ -660,7 +661,9 @@ def python_build_info( objs = [] - for obj in sorted(d["posix_obj_paths"]): + object_paths = d["posix_obj_paths"] | info.get("archive_obj_paths", set()) + + for obj in sorted(object_paths): obj = pathlib.Path("build") / obj log("adding object file %s for extension %s" % (obj, extension)) objs.append(str(obj)) @@ -683,6 +686,10 @@ def python_build_info( if libname.endswith(".a"): continue + # The macOS SDK's readline compatibility stub resolves to libedit. + if platform.startswith("macos_") and libname == "readline": + libname = "edit" + log("adding library %s for extension %s" % (libname, extension)) if libname in libraries: @@ -767,7 +774,6 @@ def build_cpython( extension_modules=ems, ) - enabled_extensions = setup["extensions"] setup_local_content = setup["setup_local"] extra_make_content = setup["make_data"] @@ -914,6 +920,27 @@ def build_cpython( extra_metadata = json.loads(build_env.get_file("metadata.json")) + if meets_python_minimum_version(python_version, "3.12"): + setup_directory = "out/python/build/Modules" + config_directory = extra_metadata["python_stdlib_platform_config"] + + enabled_extensions = configured_extension_modules( + python_version=python_version, + setup_files=[ + build_env.get_file(f"{setup_directory}/Setup.local"), + build_env.get_file(f"{setup_directory}/Setup.stdlib"), + build_env.get_file(f"{setup_directory}/Setup.bootstrap"), + build_env.get_file(f"{setup_directory}/Setup"), + ], + config_c=build_env.get_file(f"{setup_directory}/config.c"), + makefile=build_env.get_file(f"out/python/{config_directory}/Makefile"), + config_vars=extra_metadata["python_config_vars"], + extension_modules=setup["extensions"], + ) + else: + # TODO: Remove YAML-derived extension metadata when Python 3.11 is dropped. + enabled_extensions = setup["extensions"] + # TODO: Remove `optimizations` in the future, deprecated in favor of # `build_options` in metadata version 8. optimizations = build_options.replace("freethreaded+", "") diff --git a/pythonbuild/cpython.py b/pythonbuild/cpython.py index ea62a7a6e..100c71736 100644 --- a/pythonbuild/cpython.py +++ b/pythonbuild/cpython.py @@ -5,8 +5,9 @@ import pathlib import re import tarfile +from collections import defaultdict from collections.abc import Iterable -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import IO import jsonschema @@ -392,6 +393,192 @@ def _parse_setup( return modules, enabled_modules, enabled_lines +def _parse_makefile_variables(makefile: bytes) -> dict[str, bytes]: + """Parse Makefile assignments, including continued and appended values.""" + assignment_pattern = re.compile(rb"^([A-Za-z_][A-Za-z0-9_]*)\s*([+:?]?)=\s*(.*)$") + variables: dict[str, bytes] = {} + + for line in makefile.replace(b"\\\n", b" ").splitlines(): + if not (match := assignment_pattern.match(line)): + continue + + name = match.group(1).decode("ascii") + operator = match.group(2) + value = match.group(3).strip() + + if operator == b"+" and name in variables: + variables[name] += b" " + value + elif operator != b"?" or name not in variables: + variables[name] = value + + return variables + + +def _expand_makefile_variables(value: bytes, variables: dict[str, bytes]) -> bytes: + """Expand simple Makefile variable references in archive and object lists.""" + reference_pattern = re.compile(rb"\$\(([A-Za-z_][A-Za-z0-9_]*)\)") + expanded: dict[str, bytes] = {} + expanding: set[str] = set() + + def replace(match: re.Match[bytes]) -> bytes: + name = match.group(1).decode("ascii") + if name not in variables: + return match.group(0) + if name in expanded: + return expanded[name] + if name in expanding: + raise ValueError("recursive Makefile variable references") + expanding.add(name) + try: + result = reference_pattern.sub(replace, variables[name]) + finally: + expanding.remove(name) + expanded[name] = result + return result + + return reference_pattern.sub(replace, value) + + +def _configured_setup_lines( + setup_files: Iterable[bytes], python_version: str +) -> dict[str, bytes]: + """Return a mapping between module names and lines, respecting makesetup's first-rule precedence.""" + setup_lines: dict[str, bytes] = {} + for setup_file in setup_files: + _, _, current_setup_lines = _parse_setup(setup_file.splitlines()) + for name, line in current_setup_lines.items(): + parsed = parse_setup_line(line, python_version) + if parsed and parsed["posix_obj_paths"]: + setup_lines.setdefault(name, line) + return setup_lines + + +def _internal_archive_objects(makefile: bytes) -> dict[str, set[pathlib.Path]]: + """Map CPython's internal static archives to their constituent object files.""" + variables = _parse_makefile_variables(makefile) + archives = {} + + for name, value in variables.items(): + if name.endswith("_LIB_STATIC"): + object_variable = f"{name.removesuffix('_LIB_STATIC')}_OBJS" + elif name.endswith("_A"): + object_variable = f"{name.removesuffix('_A')}_OBJS" + else: + continue + + if object_variable not in variables: + continue + + archive = _expand_makefile_variables(value, variables).decode("ascii") + objects = _expand_makefile_variables(variables[object_variable], variables) + archives[archive] = { + pathlib.Path(path.decode("ascii")) + for path in objects.split() + if path.endswith(b".o") + } + + return archives + + +@dataclass +class ExtensionInfo: + """Build-artifact metadata consumed when generating PYTHON.json.""" + + init_fn: str + in_core: bool + setup_line: bytes + # "static", "shared", None if in core + build_mode: str | None = None + # Object files linked indirectly through CPython's internal static archives. + # For example via libHacl_HMAC.a + archive_obj_paths: set[pathlib.Path] = field(default_factory=set) + required_targets: list[str] | None = None + + def to_dict(self) -> dict[str, object]: + metadata: dict[str, object] = { + "init_fn": self.init_fn, + "in_core": self.in_core, + "setup_line": self.setup_line, + } + if self.required_targets is not None: + metadata["required-targets"] = self.required_targets + if self.build_mode is not None: + metadata["build-mode"] = self.build_mode + metadata["archive_obj_paths"] = self.archive_obj_paths + return metadata + + +def configured_extension_modules( + python_version: str, + setup_files: Iterable[bytes], + config_c: bytes, + makefile: bytes, + config_vars: dict[str, str], + extension_modules: dict[str, dict], +) -> dict[str, dict]: + """Derive extension metadata from built CPython artifacts.""" + built_modules = set(config_vars["MODBUILT_NAMES"].split()) + shared_modules = set(config_vars["MODSHARED_NAMES"].split()) + builtin_modules = parse_config_c(config_c.decode("utf-8")) + module_names = built_modules | set(builtin_modules) + + # Validate that all modules have YAML metadata and were built + if missing := module_names - set(extension_modules): + raise ValueError(f"modules lack metadata: {', '.join(missing)}") + if missing := set(extension_modules) - module_names: + raise ValueError(f"modules were not built: {', '.join(missing)}") + + setup_lines = _configured_setup_lines(setup_files, python_version) + archive_objects = _internal_archive_objects(makefile) + extensions: dict[str, ExtensionInfo] = {} + + for name in sorted(module_names): + policy = extension_modules[name] + in_core = name not in built_modules + extension_info = ExtensionInfo( + init_fn=builtin_modules.get(name, f"PyInit_{name}"), + in_core=in_core, + setup_line=name.encode("ascii"), + required_targets=policy.get("required-targets"), + ) + + if in_core: + extensions[name] = extension_info + continue + + if name not in setup_lines: + raise ValueError(f"configured extension {name} has no Modules/Setup rule") + + build_mode = "shared" if name in shared_modules else "static" + expected_build_mode = policy.get("build-mode") + if build_mode != expected_build_mode: + raise ValueError( + f"extension {name} built as {build_mode}; expected {expected_build_mode}" + ) + + setup_line = setup_lines[name] + linker_flags = config_vars.get(f"MODULE_{name.upper()}_LDFLAGS", "") + if linker_flags: + setup_line += f" {linker_flags}".encode("ascii") + + archive_obj_paths: set[pathlib.Path] = set() + for word in setup_line.split(): + if word.endswith(b".a") and not word.startswith(b"-"): + archive = word.decode("ascii") + if archive not in archive_objects: + raise ValueError( + f"extension {name} references unknown static archive {archive}" + ) + archive_obj_paths.update(archive_objects[archive]) + + extension_info.setup_line = setup_line + extension_info.build_mode = build_mode + extension_info.archive_obj_paths = archive_obj_paths + extensions[name] = extension_info + + return {name: metadata.to_dict() for name, metadata in extensions.items()} + + @dataclass class CPythonModuleInfo: # Maps extension names to lines in Modules/Setup.stdlib.in @@ -596,9 +783,7 @@ def _build_yaml_setup_line( python_version: str, target_triple: str, build_mode: str, - use_setup_stdlib: bool, - extra_cflags: dict[bytes, list[bytes]], -) -> bytes: +) -> tuple[bytes, dict[bytes, list[bytes]]]: line = name for source in info.get("sources", []): @@ -675,10 +860,10 @@ def _build_yaml_setup_line( # makesetup interprets lines containing = as configuration options. Move # -Dname=value defines into Makefile overrides for legacy Python builds. - if not use_setup_stdlib: - for match in define_pattern.finditer(parsed["line"]): - for obj_path in sorted(parsed["posix_obj_paths"]): - extra_cflags.setdefault(bytes(obj_path), []).append(match.group(0)) + module_cflags: defaultdict[bytes, list[bytes]] = defaultdict(list) + for match in define_pattern.finditer(parsed["line"]): + for obj_path in sorted(parsed["posix_obj_paths"]): + module_cflags[bytes(obj_path)].append(match.group(0)) setup_line = define_pattern.sub(b"", setup_line) @@ -688,7 +873,7 @@ def _build_yaml_setup_line( "makesetup: %s" % setup_line.decode("utf-8") ) - return setup_line + return setup_line, module_cflags def _determine_module_linkage(info: dict, build_options: set[str]) -> str: @@ -754,7 +939,7 @@ def derive_setup_local( # to be no easy way to define e.g. -Dfoo=bar in Setup.local. We hack # around this by producing a Makefile supplement that overrides the build # rules for certain targets to include these missing values. - extra_cflags: dict[bytes, list[bytes]] = {} + extra_cflags: defaultdict[bytes, list[bytes]] = defaultdict(list) enabled_extensions = {} @@ -789,32 +974,17 @@ def derive_setup_local( log(f"extension {name} enabled through distribution's Modules/Setup file") - # But we do record a placeholder Setup line in the returned metadata - # as this is what's used to derive PYTHON.json metadata. + # Preserve the Setup line for Python 3.10/3.11 metadata generation. + # Python 3.12+ reconstructs metadata from configured build files. enabled_extensions[name]["setup_line"] = module_info.setup_lines[name] continue - if use_setup_stdlib and name in module_info.stdlib_lines: - log(f"extension {name} being configured via Modules/Setup.stdlib") - else: - log(f"extension {name} being configured via YAML metadata") - - line = _build_yaml_setup_line( - name, - info, - python_version, - target_triple, - section, - use_setup_stdlib, - extra_cflags, - ) - - # The YAML-derived line is still used to describe packaged object files - # and dependency libraries in PYTHON.json, even when compilation is - # driven by Setup.stdlib. - enabled_extensions[name]["setup_line"] = line + if use_setup_stdlib: # 3.12+ + if name in module_info.stdlib_lines: + log(f"extension {name} being configured via Modules/Setup.stdlib") + else: + log(f"extension {name} being configured via Modules/Setup.bootstrap") - if use_setup_stdlib: if section == module_info.stdlib_linkage.get(name, "static"): continue @@ -824,14 +994,25 @@ def derive_setup_local( ) # Inherit the MODULE__CFLAGS/LDFLAGS produced by configure. - line = module_info.stdlib_lines[name] - - section_lines[section].append(line) + section_lines[section].append(module_info.stdlib_lines[name]) + else: # 3.10 and 3.11 + log(f"extension {name} being configured via YAML metadata") + line, module_cflags = _build_yaml_setup_line( + name, + info, + python_version, + target_triple, + section, + ) + for obj_path, flags in module_cflags.items(): + extra_cflags[obj_path].extend(flags) + enabled_extensions[name]["setup_line"] = line + section_lines[section].append(line) return { "extensions": enabled_extensions, "setup_local": _render_setup_local(section_lines), - "make_data": _render_make_data(extra_cflags), + "make_data": _render_make_data(extra_cflags), # Empty in 3.12+ }