pydevices publishing: own-package MIP packages and PyPI extras (for bledev)

docs/publishing-automation.md

@@ -131,6 +131,15 @@ differently, because their constraints differ:
   module scope. A *new* backend nobody classifies ships to the device, so the
   cost of forgetting is one file too many on a board, never a missing import on
   a host.
+- **A package can publish to MIP on its own.** A `mip-split.toml` section
+  with `own-package = true` leaves the `pydevices` MIP package and becomes a
+  sibling (`micropython/pydevices/<name>`), with its own `requires` as
+  `require()` lines, which the index resolves by including them. `bledev` is
+  the first: a board without a radio shouldn't carry BLE, and
+  `mip.install("bledev")` brings aioble with it. On PyPI nothing changes: the
+  package stays inside the `pydevices` wheel, and a section's
+  `pypi-extras = { ble = ["bleak>=1.0"] }` becomes
+  `pip install "pydevices[ble]"`. An unknown key in a section fails the sync.
 - Every publishable entry under `utils/`, plus everything publishable in
   `board_configs/desktop/`, is bundled into `pydevices-desktop`.
 - `pydevices-desktop` depends on `pydevices`, so one install gets the complete

scripts/build_pydevices_python_distributions.py

@@ -7,6 +7,7 @@
 import shutil
 import subprocess
 import sys
+import tomllib
 from pathlib import Path
 
 from pydevices_package_metadata import PYDEVICES_DESCRIPTIONS
@@ -48,10 +49,45 @@ def copy_component(source: Path, destination: Path) -> None:
         shutil.copy2(source, destination)
 
 
-def project_text(name: str, version: str, description: str, dependencies: list[str], source: Path) -> str:
+def pypi_extras(root: Path) -> dict[str, list[str]]:
+    """Optional dependencies declared in the source repository's mip-split.toml.
+
+    A lib/ package can name a PyPI extra for a library only some hosts need,
+    e.g. bledev's ``pypi-extras = { ble = ["bleak>=1.0"] }``, which becomes
+    ``pip install "pydevices[ble]"``. Two packages naming the same extra
+    must agree.
+    """
+    split_path = root / "mip-split.toml"
+    if not split_path.exists():
+        return {}
+    with split_path.open("rb") as handle:
+        declared = tomllib.load(handle)
+    extras: dict[str, list[str]] = {}
+    for package, section in declared.items():
+        for extra, requirements in section.get("pypi-extras", {}).items():
+            requirements = list(requirements)
+            if extra in extras and extras[extra] != requirements:
+                raise SystemExit(f"mip-split.toml: extra {extra!r} is declared twice, differently ([{package}])")
+            extras[extra] = requirements
+    return extras
+
+
+def project_text(
+    name: str,
+    version: str,
+    description: str,
+    dependencies: list[str],
+    source: Path,
+    extras: dict[str, list[str]] | None = None,
+) -> str:
     modules = sorted(path.stem for path in source.glob("*.py") if path.name != "__init__.py")
     module_line = f"py-modules = {modules!r}\n" if modules else ""
     dependency_lines = "\n".join(f'  "{dependency}",' for dependency in dependencies)
+    extras_block = ""
+    if extras:
+        extras_block = "\n[project.optional-dependencies]\n" + "".join(
+            f"{extra} = {requirements!r}\n".replace("'", '"') for extra, requirements in sorted(extras.items())
+        )
     return f'''[build-system]
 requires = ["setuptools>=68", "wheel"]
 build-backend = "setuptools.build_meta"
@@ -67,7 +103,7 @@ def project_text(name: str, version: str, description: str, dependencies: list[s
 dependencies = [
 {dependency_lines}
 ]
-
+{extras_block}
 [project.urls]
 Homepage = "https://github.com/PyDevices/pydevices"
 Repository = "https://github.com/PyDevices/pydevices"
@@ -80,12 +116,19 @@ def project_text(name: str, version: str, description: str, dependencies: list[s
 '''
 
 
-def write_project(stage: Path, name: str, version: str, description: str, dependencies: list[str]) -> None:
+def write_project(
+    stage: Path,
+    name: str,
+    version: str,
+    description: str,
+    dependencies: list[str],
+    extras: dict[str, list[str]] | None = None,
+) -> None:
     stage.mkdir(parents=True, exist_ok=True)
     (stage / "src").mkdir(exist_ok=True)
     (stage / "README.md").write_text(f"# {name}\n\n{description}.\n", encoding="utf-8")
     (stage / "pyproject.toml").write_text(
-        project_text(name, version, description, dependencies, stage / "src"), encoding="utf-8"
+        project_text(name, version, description, dependencies, stage / "src", extras), encoding="utf-8"
     )
 
 
@@ -118,6 +161,7 @@ def build(root: Path, output: Path, version: str) -> None:
         version,
         PYDEVICES_DESCRIPTIONS["pydevices"],
         [],
+        pypi_extras(root),
     )
     stages.append(meta)
 

scripts/pydevices_package_metadata.py

@@ -3,4 +3,6 @@
 PYDEVICES_DESCRIPTIONS = {
     "pydevices": "Portable display, audio, event, and timing foundations for PyDevices",
     "pydevices-desktop": "Complete PyDevices desktop stack and board configuration",
+    # MIP only: on PyPI bledev is inside the pydevices wheel.
+    "bledev": "Portable async Bluetooth Low Energy for PyDevices, over aioble on boards",
 }

scripts/synchronize_mip_package.py

@@ -201,13 +201,54 @@ def render_pydevices_manifest(name: str, version: str, requirements: tuple[str,
 MIP_SPLIT_FILE = "mip-split.toml"
 
 
-def read_host_only(source_root: Path) -> dict[str, frozenset[str]]:
-    """{package: host-only module stems}, checked against what is on disk."""
+#: Every key a mip-split.toml section may use. An unknown key fails the sync,
+#: so a misspelt `own-package` can't quietly ship a package inside pydevices.
+MIP_SPLIT_KEYS = frozenset({"host-only", "own-package", "requires", "pypi-extras"})
+
+
+def read_split(source_root: Path) -> dict[str, dict]:
+    """The source repository's mip-split.toml, or {} when it has none."""
     split_path = source_root / MIP_SPLIT_FILE
     if not split_path.exists():
         return {}
     with split_path.open("rb") as handle:
         declared = tomllib.load(handle)
+    for package, section in declared.items():
+        unknown = sorted(set(section) - MIP_SPLIT_KEYS)
+        if unknown:
+            raise SystemExit(
+                f"{MIP_SPLIT_FILE} [{package}] has unknown keys: {', '.join(unknown)}"
+            )
+    return declared
+
+
+def read_own_packages(source_root: Path) -> dict[str, tuple[str, ...]]:
+    """{package: requires} for each lib/ package that publishes to MIP on its own.
+
+    Such a package leaves the ``pydevices`` MIP package (it stays in the
+    ``pydevices`` wheel on PyPI). Its ``requires`` become ``require()`` lines,
+    which the index resolves by including them.
+    """
+    own: dict[str, tuple[str, ...]] = {}
+    for package, section in read_split(source_root).items():
+        if not section.get("own-package"):
+            if "requires" in section:
+                raise SystemExit(
+                    f"{MIP_SPLIT_FILE} [{package}] has requires but is not an own-package; "
+                    f"a package inside pydevices has no manifest of its own to put them in"
+                )
+            continue
+        if not (source_root / "lib" / package).is_dir():
+            raise SystemExit(f"{MIP_SPLIT_FILE} names package {package!r}, which is not in lib/")
+        if package not in PYDEVICES_DESCRIPTIONS:
+            raise SystemExit(f"no shared description for own-package {package!r}")
+        own[package] = tuple(section.get("requires", ()))
+    return own
+
+
+def read_host_only(source_root: Path) -> dict[str, frozenset[str]]:
+    """{package: host-only module stems}, checked against what is on disk."""
+    declared = read_split(source_root)
 
     host_only: dict[str, frozenset[str]] = {}
     for package, section in declared.items():
@@ -289,10 +330,35 @@ def synchronize_pydevices(source_root: Path, mip_root: Path, version: str) -> No
     # instead, which already require()s this package -- so a host installs one
     # thing and gets what it always got (pydevices#30).
     host_only = read_host_only(source_root)
+    own_packages = read_own_packages(source_root)
     host_components: list[tuple[Path, frozenset[str]]] = []
     for source in sorted(filter(publishable, (source_root / "lib").iterdir()), key=lambda path: path.name):
         names.append(source.stem if source.is_file() else source.name)
         split = host_only.get(source.name, frozenset()) if source.is_dir() else frozenset()
+        if source.name in own_packages and source.is_dir():
+            # Its own MIP package, beside pydevices rather than inside it
+            # (bledev: a board without a radio shouldn't carry BLE).
+            check_no_host_imports(source, split)
+            own_root = destination_root / source.name
+            own_root.mkdir()
+            device_files = sorted(path.name for path in source.glob("*.py") if path.stem not in split)
+            destination = own_root / source.name
+            destination.mkdir()
+            for name in device_files:
+                shutil.copy2(source / name, destination / name)
+            listed = ", ".join(f'"{name}"' for name in device_files)
+            (own_root / "manifest.py").write_text(
+                render_pydevices_manifest(
+                    source.name,
+                    version,
+                    own_packages[source.name],
+                    (f'package("{source.name}", files=({listed},))',),
+                ),
+                encoding="utf-8",
+            )
+            if split:
+                host_components.append((source, split))
+            continue
         if not split:
             copy_component(source, package / source.name)
             payloads.append(f'module("{source.name}")' if source.is_file() else f'package("{source.name}")')

tests/test_synchronize_mip_package.py

@@ -341,5 +341,108 @@ def test_pydevices_profile_writes_shared_descriptions(self) -> None:
             self.assertNotIn("PyDevices pydevices-desktop", desktop)
 
 
+class OwnPackageTests(unittest.TestCase):
+    """A lib/ package marked own-package publishes to MIP beside pydevices (bledev)."""
+
+    def _source(self, root: Path, split: str) -> Path:
+        source = root / "source"
+        write_pydevices_source(source)
+        bledev = source / "lib" / "bledev"
+        bledev.mkdir(parents=True)
+        for name in ("__init__", "mpble", "nus", "bleak"):
+            (bledev / f"{name}.py").write_text(f"# {name}\n", encoding="utf-8")
+        (source / "mip-split.toml").write_text(textwrap.dedent(split), encoding="utf-8")
+        return source
+
+    def _sync(self, root: Path, source: Path) -> subprocess.CompletedProcess[str]:
+        mip = root / "mip"
+        write_lockfile(mip, {"pydevices": "PyDevices/pydevices"})
+        return run_sync(source, mip, source_name="PyDevices/pydevices", profile="pydevices", version="1.2.3")
+
+    def test_own_package_leaves_pydevices_and_requires_aioble(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            source = self._source(
+                root,
+                """
+                [bledev]
+                own-package = true
+                requires = ["aioble"]
+                host-only = ["bleak"]
+                pypi-extras = { ble = ["bleak>=1.0"] }
+                """,
+            )
+            result = self._sync(root, source)
+            self.assertEqual(result.returncode, 0, result.stderr)
+            out = root / "mip" / "micropython" / "pydevices"
+            pydevices = (out / "pydevices" / "manifest.py").read_text(encoding="utf-8")
+            self.assertNotIn("bledev", pydevices)
+            self.assertFalse((out / "pydevices" / "bledev").exists())
+            manifest = (out / "bledev" / "manifest.py").read_text(encoding="utf-8")
+            self.assertIn('require("aioble")', manifest)
+            self.assertIn('package("bledev", files=("__init__.py", "mpble.py", "nus.py",))', manifest)
+            self.assertIn(metadata.PYDEVICES_DESCRIPTIONS["bledev"], manifest)
+            self.assertNotIn("pypi_publish", manifest)
+            self.assertEqual(
+                sorted(p.name for p in (out / "bledev" / "bledev").iterdir()),
+                ["__init__.py", "mpble.py", "nus.py"],
+            )
+            desktop = (out / "pydevices-desktop" / "manifest.py").read_text(encoding="utf-8")
+            self.assertIn('package("bledev", files=("bleak.py",))', desktop)
+
+    def test_unknown_key_fails(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            source = self._source(root, '[bledev]\nown_package = true\n')
+            result = self._sync(root, source)
+            self.assertNotEqual(result.returncode, 0)
+            self.assertIn("unknown keys: own_package", result.stderr)
+
+    def test_requires_without_own_package_fails(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            source = self._source(root, '[bledev]\nrequires = ["aioble"]\n')
+            result = self._sync(root, source)
+            self.assertNotEqual(result.returncode, 0)
+            self.assertIn("not an own-package", result.stderr)
+
+    def test_without_the_flag_bledev_ships_inside_pydevices(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            source = self._source(root, '[bledev]\nhost-only = ["bleak"]\n')
+            result = self._sync(root, source)
+            self.assertEqual(result.returncode, 0, result.stderr)
+            out = root / "mip" / "micropython" / "pydevices"
+            self.assertFalse((out / "bledev").exists())
+            self.assertIn('package("bledev"', (out / "pydevices" / "manifest.py").read_text(encoding="utf-8"))
+
+
+class PypiExtrasTests(unittest.TestCase):
+    def test_extras_reach_the_pydevices_pyproject(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            (root / "mip-split.toml").write_text('[bledev]\npypi-extras = { ble = ["bleak>=1.0"] }\n', encoding="utf-8")
+            extras = build.pypi_extras(root)
+            self.assertEqual(extras, {"ble": ["bleak>=1.0"]})
+            text = build.project_text("pydevices", "1.2.3", "d", [], Path("."), extras)
+            import tomllib
+
+            self.assertEqual(tomllib.loads(text)["project"]["optional-dependencies"], {"ble": ["bleak>=1.0"]})
+
+    def test_no_extras_leaves_the_pyproject_unchanged(self) -> None:
+        text = build.project_text("pydevices", "1.2.3", "d", [], Path("."))
+        self.assertNotIn("optional-dependencies", text)
+
+    def test_conflicting_extras_fail(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            (root / "mip-split.toml").write_text(
+                '[a]\npypi-extras = { ble = ["bleak>=1.0"] }\n[b]\npypi-extras = { ble = ["bleak>=2"] }\n',
+                encoding="utf-8",
+            )
+            with self.assertRaises(SystemExit):
+                build.pypi_extras(root)
+
+
 if __name__ == "__main__":
     unittest.main()