diff --git a/docs/publishing-automation.md b/docs/publishing-automation.md index b33ea5e..85a22a3 100644 --- a/docs/publishing-automation.md +++ b/docs/publishing-automation.md @@ -131,6 +131,15 @@ differently, because their constraints differ: module scope. A *new* backend nobody classifies ships to the device, so the cost of forgetting is one file too many on a board, never a missing import on a host. +- **A package can publish to MIP on its own.** A `mip-split.toml` section + with `own-package = true` leaves the `pydevices` MIP package and becomes a + sibling (`micropython/pydevices/`), with its own `requires` as + `require()` lines, which the index resolves by including them. `bledev` is + the first: a board without a radio shouldn't carry BLE, and + `mip.install("bledev")` brings aioble with it. On PyPI nothing changes: the + package stays inside the `pydevices` wheel, and a section's + `pypi-extras = { ble = ["bleak>=1.0"] }` becomes + `pip install "pydevices[ble]"`. An unknown key in a section fails the sync. - Every publishable entry under `utils/`, plus everything publishable in `board_configs/desktop/`, is bundled into `pydevices-desktop`. - `pydevices-desktop` depends on `pydevices`, so one install gets the complete diff --git a/scripts/build_pydevices_python_distributions.py b/scripts/build_pydevices_python_distributions.py index 78a5d1d..70355ff 100755 --- a/scripts/build_pydevices_python_distributions.py +++ b/scripts/build_pydevices_python_distributions.py @@ -7,6 +7,7 @@ import shutil import subprocess import sys +import tomllib from pathlib import Path from pydevices_package_metadata import PYDEVICES_DESCRIPTIONS @@ -48,10 +49,45 @@ def copy_component(source: Path, destination: Path) -> None: shutil.copy2(source, destination) -def project_text(name: str, version: str, description: str, dependencies: list[str], source: Path) -> str: +def pypi_extras(root: Path) -> dict[str, list[str]]: + """Optional dependencies declared in the source repository's mip-split.toml. + + A lib/ package can name a PyPI extra for a library only some hosts need, + e.g. bledev's ``pypi-extras = { ble = ["bleak>=1.0"] }``, which becomes + ``pip install "pydevices[ble]"``. Two packages naming the same extra + must agree. + """ + split_path = root / "mip-split.toml" + if not split_path.exists(): + return {} + with split_path.open("rb") as handle: + declared = tomllib.load(handle) + extras: dict[str, list[str]] = {} + for package, section in declared.items(): + for extra, requirements in section.get("pypi-extras", {}).items(): + requirements = list(requirements) + if extra in extras and extras[extra] != requirements: + raise SystemExit(f"mip-split.toml: extra {extra!r} is declared twice, differently ([{package}])") + extras[extra] = requirements + return extras + + +def project_text( + name: str, + version: str, + description: str, + dependencies: list[str], + source: Path, + extras: dict[str, list[str]] | None = None, +) -> str: modules = sorted(path.stem for path in source.glob("*.py") if path.name != "__init__.py") module_line = f"py-modules = {modules!r}\n" if modules else "" dependency_lines = "\n".join(f' "{dependency}",' for dependency in dependencies) + extras_block = "" + if extras: + extras_block = "\n[project.optional-dependencies]\n" + "".join( + f"{extra} = {requirements!r}\n".replace("'", '"') for extra, requirements in sorted(extras.items()) + ) return f'''[build-system] requires = ["setuptools>=68", "wheel"] build-backend = "setuptools.build_meta" @@ -67,7 +103,7 @@ def project_text(name: str, version: str, description: str, dependencies: list[s dependencies = [ {dependency_lines} ] - +{extras_block} [project.urls] Homepage = "https://github.com/PyDevices/pydevices" Repository = "https://github.com/PyDevices/pydevices" @@ -80,12 +116,19 @@ def project_text(name: str, version: str, description: str, dependencies: list[s ''' -def write_project(stage: Path, name: str, version: str, description: str, dependencies: list[str]) -> None: +def write_project( + stage: Path, + name: str, + version: str, + description: str, + dependencies: list[str], + extras: dict[str, list[str]] | None = None, +) -> None: stage.mkdir(parents=True, exist_ok=True) (stage / "src").mkdir(exist_ok=True) (stage / "README.md").write_text(f"# {name}\n\n{description}.\n", encoding="utf-8") (stage / "pyproject.toml").write_text( - project_text(name, version, description, dependencies, stage / "src"), encoding="utf-8" + project_text(name, version, description, dependencies, stage / "src", extras), encoding="utf-8" ) @@ -118,6 +161,7 @@ def build(root: Path, output: Path, version: str) -> None: version, PYDEVICES_DESCRIPTIONS["pydevices"], [], + pypi_extras(root), ) stages.append(meta) diff --git a/scripts/pydevices_package_metadata.py b/scripts/pydevices_package_metadata.py index 911af57..05bb59f 100644 --- a/scripts/pydevices_package_metadata.py +++ b/scripts/pydevices_package_metadata.py @@ -3,4 +3,6 @@ PYDEVICES_DESCRIPTIONS = { "pydevices": "Portable display, audio, event, and timing foundations for PyDevices", "pydevices-desktop": "Complete PyDevices desktop stack and board configuration", + # MIP only: on PyPI bledev is inside the pydevices wheel. + "bledev": "Portable async Bluetooth Low Energy for PyDevices, over aioble on boards", } diff --git a/scripts/synchronize_mip_package.py b/scripts/synchronize_mip_package.py index 8f024b2..c8016ca 100755 --- a/scripts/synchronize_mip_package.py +++ b/scripts/synchronize_mip_package.py @@ -201,13 +201,54 @@ def render_pydevices_manifest(name: str, version: str, requirements: tuple[str, MIP_SPLIT_FILE = "mip-split.toml" -def read_host_only(source_root: Path) -> dict[str, frozenset[str]]: - """{package: host-only module stems}, checked against what is on disk.""" +#: Every key a mip-split.toml section may use. An unknown key fails the sync, +#: so a misspelt `own-package` can't quietly ship a package inside pydevices. +MIP_SPLIT_KEYS = frozenset({"host-only", "own-package", "requires", "pypi-extras"}) + + +def read_split(source_root: Path) -> dict[str, dict]: + """The source repository's mip-split.toml, or {} when it has none.""" split_path = source_root / MIP_SPLIT_FILE if not split_path.exists(): return {} with split_path.open("rb") as handle: declared = tomllib.load(handle) + for package, section in declared.items(): + unknown = sorted(set(section) - MIP_SPLIT_KEYS) + if unknown: + raise SystemExit( + f"{MIP_SPLIT_FILE} [{package}] has unknown keys: {', '.join(unknown)}" + ) + return declared + + +def read_own_packages(source_root: Path) -> dict[str, tuple[str, ...]]: + """{package: requires} for each lib/ package that publishes to MIP on its own. + + Such a package leaves the ``pydevices`` MIP package (it stays in the + ``pydevices`` wheel on PyPI). Its ``requires`` become ``require()`` lines, + which the index resolves by including them. + """ + own: dict[str, tuple[str, ...]] = {} + for package, section in read_split(source_root).items(): + if not section.get("own-package"): + if "requires" in section: + raise SystemExit( + f"{MIP_SPLIT_FILE} [{package}] has requires but is not an own-package; " + f"a package inside pydevices has no manifest of its own to put them in" + ) + continue + if not (source_root / "lib" / package).is_dir(): + raise SystemExit(f"{MIP_SPLIT_FILE} names package {package!r}, which is not in lib/") + if package not in PYDEVICES_DESCRIPTIONS: + raise SystemExit(f"no shared description for own-package {package!r}") + own[package] = tuple(section.get("requires", ())) + return own + + +def read_host_only(source_root: Path) -> dict[str, frozenset[str]]: + """{package: host-only module stems}, checked against what is on disk.""" + declared = read_split(source_root) host_only: dict[str, frozenset[str]] = {} for package, section in declared.items(): @@ -289,10 +330,35 @@ def synchronize_pydevices(source_root: Path, mip_root: Path, version: str) -> No # instead, which already require()s this package -- so a host installs one # thing and gets what it always got (pydevices#30). host_only = read_host_only(source_root) + own_packages = read_own_packages(source_root) host_components: list[tuple[Path, frozenset[str]]] = [] for source in sorted(filter(publishable, (source_root / "lib").iterdir()), key=lambda path: path.name): names.append(source.stem if source.is_file() else source.name) split = host_only.get(source.name, frozenset()) if source.is_dir() else frozenset() + if source.name in own_packages and source.is_dir(): + # Its own MIP package, beside pydevices rather than inside it + # (bledev: a board without a radio shouldn't carry BLE). + check_no_host_imports(source, split) + own_root = destination_root / source.name + own_root.mkdir() + device_files = sorted(path.name for path in source.glob("*.py") if path.stem not in split) + destination = own_root / source.name + destination.mkdir() + for name in device_files: + shutil.copy2(source / name, destination / name) + listed = ", ".join(f'"{name}"' for name in device_files) + (own_root / "manifest.py").write_text( + render_pydevices_manifest( + source.name, + version, + own_packages[source.name], + (f'package("{source.name}", files=({listed},))',), + ), + encoding="utf-8", + ) + if split: + host_components.append((source, split)) + continue if not split: copy_component(source, package / source.name) payloads.append(f'module("{source.name}")' if source.is_file() else f'package("{source.name}")') diff --git a/tests/test_synchronize_mip_package.py b/tests/test_synchronize_mip_package.py index 958c809..4a4b11c 100644 --- a/tests/test_synchronize_mip_package.py +++ b/tests/test_synchronize_mip_package.py @@ -341,5 +341,108 @@ def test_pydevices_profile_writes_shared_descriptions(self) -> None: self.assertNotIn("PyDevices pydevices-desktop", desktop) +class OwnPackageTests(unittest.TestCase): + """A lib/ package marked own-package publishes to MIP beside pydevices (bledev).""" + + def _source(self, root: Path, split: str) -> Path: + source = root / "source" + write_pydevices_source(source) + bledev = source / "lib" / "bledev" + bledev.mkdir(parents=True) + for name in ("__init__", "mpble", "nus", "bleak"): + (bledev / f"{name}.py").write_text(f"# {name}\n", encoding="utf-8") + (source / "mip-split.toml").write_text(textwrap.dedent(split), encoding="utf-8") + return source + + def _sync(self, root: Path, source: Path) -> subprocess.CompletedProcess[str]: + mip = root / "mip" + write_lockfile(mip, {"pydevices": "PyDevices/pydevices"}) + return run_sync(source, mip, source_name="PyDevices/pydevices", profile="pydevices", version="1.2.3") + + def test_own_package_leaves_pydevices_and_requires_aioble(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source = self._source( + root, + """ + [bledev] + own-package = true + requires = ["aioble"] + host-only = ["bleak"] + pypi-extras = { ble = ["bleak>=1.0"] } + """, + ) + result = self._sync(root, source) + self.assertEqual(result.returncode, 0, result.stderr) + out = root / "mip" / "micropython" / "pydevices" + pydevices = (out / "pydevices" / "manifest.py").read_text(encoding="utf-8") + self.assertNotIn("bledev", pydevices) + self.assertFalse((out / "pydevices" / "bledev").exists()) + manifest = (out / "bledev" / "manifest.py").read_text(encoding="utf-8") + self.assertIn('require("aioble")', manifest) + self.assertIn('package("bledev", files=("__init__.py", "mpble.py", "nus.py",))', manifest) + self.assertIn(metadata.PYDEVICES_DESCRIPTIONS["bledev"], manifest) + self.assertNotIn("pypi_publish", manifest) + self.assertEqual( + sorted(p.name for p in (out / "bledev" / "bledev").iterdir()), + ["__init__.py", "mpble.py", "nus.py"], + ) + desktop = (out / "pydevices-desktop" / "manifest.py").read_text(encoding="utf-8") + self.assertIn('package("bledev", files=("bleak.py",))', desktop) + + def test_unknown_key_fails(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source = self._source(root, '[bledev]\nown_package = true\n') + result = self._sync(root, source) + self.assertNotEqual(result.returncode, 0) + self.assertIn("unknown keys: own_package", result.stderr) + + def test_requires_without_own_package_fails(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source = self._source(root, '[bledev]\nrequires = ["aioble"]\n') + result = self._sync(root, source) + self.assertNotEqual(result.returncode, 0) + self.assertIn("not an own-package", result.stderr) + + def test_without_the_flag_bledev_ships_inside_pydevices(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source = self._source(root, '[bledev]\nhost-only = ["bleak"]\n') + result = self._sync(root, source) + self.assertEqual(result.returncode, 0, result.stderr) + out = root / "mip" / "micropython" / "pydevices" + self.assertFalse((out / "bledev").exists()) + self.assertIn('package("bledev"', (out / "pydevices" / "manifest.py").read_text(encoding="utf-8")) + + +class PypiExtrasTests(unittest.TestCase): + def test_extras_reach_the_pydevices_pyproject(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "mip-split.toml").write_text('[bledev]\npypi-extras = { ble = ["bleak>=1.0"] }\n', encoding="utf-8") + extras = build.pypi_extras(root) + self.assertEqual(extras, {"ble": ["bleak>=1.0"]}) + text = build.project_text("pydevices", "1.2.3", "d", [], Path("."), extras) + import tomllib + + self.assertEqual(tomllib.loads(text)["project"]["optional-dependencies"], {"ble": ["bleak>=1.0"]}) + + def test_no_extras_leaves_the_pyproject_unchanged(self) -> None: + text = build.project_text("pydevices", "1.2.3", "d", [], Path(".")) + self.assertNotIn("optional-dependencies", text) + + def test_conflicting_extras_fail(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "mip-split.toml").write_text( + '[a]\npypi-extras = { ble = ["bleak>=1.0"] }\n[b]\npypi-extras = { ble = ["bleak>=2"] }\n', + encoding="utf-8", + ) + with self.assertRaises(SystemExit): + build.pypi_extras(root) + + if __name__ == "__main__": unittest.main()