From b69c24517459b4640a0ac211e7b6aa9a8254fa5c Mon Sep 17 00:00:00 2001
From: Brad Barnett <127794626+bdbarnett@users.noreply.github.com>
Date: Fri, 25 Sep 2026 09:27:23 -0500
Subject: [PATCH 1/2] generate_sites: --only REPO rewrites one page, for mip's
refresh job
mip's refresh workflow updates packages.json after every publication but
never regenerated .site/index.html, so the page lagged a release and this
repo's generator-idempotency check failed until someone ran the generator by
hand (#51, #54). --only lets that workflow regenerate mip's page with just
mip and this repository checked out: it skips the shared assets, the docs
theme and the ecosystem maps, validates sibling checkouts for the named repo
only, and fails if the page wasn't written.
---
scripts/generate_sites.py | 58 +++++++++++++++---
tests/test_generate_sites.py | 114 +++++++++++++++++++++++++++++++++++
2 files changed, 162 insertions(+), 10 deletions(-)
create mode 100644 tests/test_generate_sites.py
diff --git a/scripts/generate_sites.py b/scripts/generate_sites.py
index ad75660..f8cf2b4 100755
--- a/scripts/generate_sites.py
+++ b/scripts/generate_sites.py
@@ -1,4 +1,13 @@
#!/usr/bin/env python3
+"""Generate every PyDevices landing page from data/repos_db.json.
+
+Run with no arguments from a full workspace to rewrite every page, the shared
+chrome, the docs theme and the ecosystem maps. `--only REPO` rewrites that one
+repository's page and nothing else. mip's refresh-package-list workflow uses
+`--only mip` to regenerate its page in the same commit as packages.json
+(.github#51, #54), with only mip and this repository checked out.
+"""
+import argparse
import html
import json
import os
@@ -84,11 +93,13 @@ def get_tag_label(repo_name):
}
return tags.get(repo_name, repo_name)
-def validate_db(db):
+def validate_db(db, only=None):
"""Fail fast on a malformed database.
The generator is run by hand, so this is where the check belongs -- it runs
- exactly when someone edits the database and regenerates.
+ exactly when someone edits the database and regenerates. With `only`, the
+ checks that need a sibling checkout on disk look at that repo alone: the
+ rest of the workspace is not expected to be there.
"""
if '_meta' not in db:
raise SystemExit('repos_db.json is missing its _meta block')
@@ -126,16 +137,17 @@ def validate_db(db):
roots = [n for n, d in repos(db).items() if page_destination(d) == 'portal-root']
for name, data in repos(db).items():
destination = page_destination(data)
+ on_disk = only is None or name == only
if destination not in valid_pages:
problems.append(
f'{name}: page {destination!r} is not one of {sorted(valid_pages)}'
)
# A repo keeping its own Pages must still have somewhere to publish from.
- if destination == 'self' and not os.path.isdir(
+ if on_disk and destination == 'self' and not os.path.isdir(
os.path.join(BASE_DIR, name, '.site')
):
problems.append(f'{name}: page=self but no .site/ directory')
- if destination == 'self-subpath':
+ if on_disk and destination == 'self-subpath':
site_repo = data.get('site_repo')
site_subpath = data.get('site_subpath')
if not site_repo or not site_subpath:
@@ -426,8 +438,9 @@ def build_mip_packages_html(packages_path):
regenerates (scripts/update_package_list.py) from the index itself after
every deploy, never from PyPI or TestPyPI. Returns None when the sibling
checkout has no packages.json so the caller can leave the marker block
- alone rather than blank it. The page itself moves only when this
- generator is rerun and mip/.site is committed.
+ alone rather than blank it. mip's refresh workflow reruns this generator
+ with `--only mip` and commits mip/.site with packages.json, then redeploys
+ Pages, so the page moves with the list.
"""
if not os.path.exists(packages_path):
return None
@@ -620,20 +633,37 @@ def get_site_html_path(repo_name, data):
return os.path.join(BASE_DIR, data['site_repo'], '.site', data['site_subpath'], 'index.html')
return None
-def main():
+def parse_args(argv=None):
+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+ parser.add_argument(
+ '--only', metavar='REPO',
+ help="rewrite only this repository's page; skip the shared assets, "
+ "the docs theme and the ecosystem maps",
+ )
+ return parser.parse_args(argv)
+
+
+def main(argv=None):
+ args = parse_args(argv)
print("=== Pure Harmonized PyDevices Site Generator (.github) ===")
with open(DB_PATH, 'r', encoding='utf-8') as f:
db = json.load(f)
- validate_db(db)
+ validate_db(db, only=args.only)
+
+ if args.only is not None and args.only not in repos(db):
+ raise SystemExit(f"--only {args.only}: not in data/repos_db.json")
updated_sites = 0
- sync_assets(db)
- sync_docs_theme()
+ if args.only is None:
+ sync_assets(db)
+ sync_docs_theme()
for repo_name, data in repos(db).items():
+ if args.only is not None and repo_name != args.only:
+ continue
site_html_path = get_site_html_path(repo_name, data)
if site_html_path is None:
print(f"[SKIP] {repo_name}: no generated page (page=none)")
@@ -698,6 +728,14 @@ def main():
updated_sites += 1
print(f"[OK] Generated & Updated {repo_name} ({os.path.relpath(site_html_path, BASE_DIR)})")
+ if args.only is not None:
+ # A page asked for by name and not written is a failure: the caller
+ # commits whatever is on disk and would otherwise commit a stale page.
+ if updated_sites != 1:
+ raise SystemExit(f"--only {args.only}: page not written")
+ print(f"=== Complete! Processed {args.only} only ===")
+ return
+
written = write_ecosystem_markdown(db)
print(f"=== Complete! Processed {len(repos(db))} repos "
diff --git a/tests/test_generate_sites.py b/tests/test_generate_sites.py
new file mode 100644
index 0000000..6a70460
--- /dev/null
+++ b/tests/test_generate_sites.py
@@ -0,0 +1,114 @@
+"""Tests for scripts/generate_sites.py --only, the path mip's refresh workflow runs."""
+
+from __future__ import annotations
+
+import json
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+REPO = Path(__file__).resolve().parents[1]
+
+PAGE = """
+
+
+
+
+
+
+
+
+
+
+
+
+"""
+
+
+def _package(version: str) -> dict:
+ return {
+ "name": "pydevices",
+ "version": version,
+ "description": "Display, input and audio drivers",
+ "repository": "PyDevices/pydevices",
+ "ref": f"v{version}",
+ "release_url": f"https://github.com/PyDevices/pydevices/releases/tag/v{version}",
+ "install": 'mip.install("pydevices", index="https://PyDevices.github.io/mip")',
+ }
+
+
+class OnlyMipTests(unittest.TestCase):
+ """A workspace holding only dotgithub and mip, as the refresh job lays it out."""
+
+ def setUp(self) -> None:
+ self.root = Path(tempfile.mkdtemp())
+ self.addCleanup(shutil.rmtree, self.root)
+ dotgithub = self.root / "dotgithub"
+ (dotgithub / "scripts").mkdir(parents=True)
+ (dotgithub / "data").mkdir()
+ shutil.copy2(REPO / "scripts/generate_sites.py", dotgithub / "scripts")
+ shutil.copy2(REPO / "data/repos_db.json", dotgithub / "data")
+ shutil.copytree(REPO / "assets", dotgithub / "assets")
+ self.script = dotgithub / "scripts/generate_sites.py"
+ self.mip = self.root / "mip"
+ (self.mip / ".site").mkdir(parents=True)
+ self.page = self.mip / ".site/index.html"
+ self.page.write_text(PAGE, encoding="utf-8")
+
+ def _write_packages(self, version: str) -> None:
+ (self.mip / "packages.json").write_text(
+ json.dumps({"packages": [_package(version)]}), encoding="utf-8"
+ )
+
+ def _run(self, *args: str) -> subprocess.CompletedProcess:
+ return subprocess.run(
+ [sys.executable, str(self.script), *args],
+ capture_output=True, text=True,
+ )
+
+ def test_page_follows_packages_json(self) -> None:
+ self._write_packages("0.5.3")
+ self.assertEqual(self._run("--only", "mip").returncode, 0)
+ self.assertIn("| 0.5.3 | ", self.page.read_text(encoding="utf-8"))
+
+ self._write_packages("0.5.4")
+ result = self._run("--only", "mip")
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+ page = self.page.read_text(encoding="utf-8")
+ self.assertIn("0.5.4 | ", page)
+ self.assertIn("releases/tag/v0.5.4", page)
+ self.assertNotIn("0.5.3", page)
+ self.assertIn("", page)
+
+ def test_rerun_is_byte_identical(self) -> None:
+ self._write_packages("0.5.4")
+ self._run("--only", "mip")
+ first = self.page.read_bytes()
+ self._run("--only", "mip")
+ self.assertEqual(first, self.page.read_bytes())
+
+ def test_writes_nothing_outside_the_named_repo(self) -> None:
+ self._write_packages("0.5.4")
+ self.assertEqual(self._run("--only", "mip").returncode, 0)
+ self.assertEqual(
+ sorted(p.name for p in self.root.iterdir()), ["dotgithub", "mip"]
+ )
+
+ def test_missing_page_fails(self) -> None:
+ self._write_packages("0.5.4")
+ self.page.unlink()
+ result = self._run("--only", "mip")
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("page not written", result.stderr)
+
+ def test_unknown_repo_fails(self) -> None:
+ result = self._run("--only", "no-such-repo")
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("not in data/repos_db.json", result.stderr)
+
+
+if __name__ == "__main__":
+ unittest.main()
From c7265e9bd1c299b9a6f399875e17830fe06c9849 Mon Sep 17 00:00:00 2001
From: Brad Barnett <127794626+bdbarnett@users.noreply.github.com>
Date: Fri, 25 Sep 2026 09:47:48 -0500
Subject: [PATCH 2/2] test_generate_sites: explicit check=False for ruff
PLW1510
---
tests/test_generate_sites.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tests/test_generate_sites.py b/tests/test_generate_sites.py
index 6a70460..cc75a34 100644
--- a/tests/test_generate_sites.py
+++ b/tests/test_generate_sites.py
@@ -66,7 +66,7 @@ def _write_packages(self, version: str) -> None:
def _run(self, *args: str) -> subprocess.CompletedProcess:
return subprocess.run(
[sys.executable, str(self.script), *args],
- capture_output=True, text=True,
+ capture_output=True, text=True, check=False,
)
def test_page_follows_packages_json(self) -> None: