generate_sites.py

#!/usr/bin/env python3
"""Generate every PyDevices landing page from data/repos_db.json.

Run with no arguments from a full workspace to rewrite every page, the shared
chrome, the docs theme and the ecosystem maps. `--only REPO` rewrites that one
repository's page and nothing else. mip's refresh-package-list workflow uses
`--only mip` to regenerate its page in the same commit as packages.json
(.github#51, #54), with only mip and this repository checked out.
"""
import argparse
import html
import json
import os
import re
import shutil

BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))
DOTGITHUB_DIR = os.path.join(BASE_DIR, 'dotgithub')
DB_PATH = os.path.join(DOTGITHUB_DIR, 'data/repos_db.json')
ASSETS_DIR = os.path.join(DOTGITHUB_DIR, 'assets')

def get_gradient_dark(color_var):
    """The darker companion a tier's logo-badge gradient fades into.

    The colours themselves live in assets/css/site.css as --tier-N-deep; this
    only maps a repo's theme_color onto the matching token so a palette change
    stays a single edit to that stylesheet's token block.
    """
    mapping = {
        'var(--accent)': 'var(--accent-strong)',
        'var(--tier-1-amber)': 'var(--tier-1-deep)',
        'var(--tier-2-emerald)': 'var(--tier-2-deep)',
        'var(--tier-3-blue)': 'var(--tier-3-deep)',
        'var(--tier-4-purple)': 'var(--tier-4-deep)',
        'var(--tier-5-steel)': 'var(--tier-5-deep)',
    }
    return mapping.get(color_var, 'var(--tier-5-deep)')

def get_card_icon(repo_name):
    icons = {
        'pydevices': '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="2" y="3" width="20" height="14" rx="2"/><path d="M8 21h8M12 17v4"/></svg>',
        'displayif': '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="3" y="4" width="18" height="12" rx="2"/><path d="M8 20h8M7 16l1-4M17 16l-1-4"/></svg>',
        'audiodsp': '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M4 10v4M8 7v10M12 4v16M16 7v10M20 10v4"/></svg>',
        'audioif': '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M4 10v4M8 7v10M12 4v16M16 7v10M20 10v4"/></svg>',
        'audiocomponents': '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M4 21v-7M4 10V3M12 21v-9M12 8V3M20 21v-5M20 12V3M1 14h6M9 8h6M17 16h6"/></svg>',
        'usbif': '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M12 2v14"/><path d="M9 5l3-3 3 3"/><path d="M12 12l-5-3V7M12 14l5-3V9"/><circle cx="12" cy="19" r="3"/></svg>',
        'cameraif': '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M14.5 4h-5L7 7H4a2 2 0 0 0-2 2v9a2 2 0 0 0 2 2h16a2 2 0 0 0 2-2V9a2 2 0 0 0-2-2h-3z"/><circle cx="12" cy="13" r="3"/></svg>',
        'micropython-pydevices': '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M12 2L2 7l10 5 10-5-10-5z"/><path d="M2 17l10 5 10-5M2 12l10 5 10-5"/></svg>',
        'workbench': '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="2" y="4" width="20" height="16" rx="2"/><path d="M10 10l-2 2 2 2M14 10l2 2-2 2"/></svg>',
        'mpvst': '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M9 18V5l12-2v13"/><circle cx="6" cy="18" r="3"/><circle cx="18" cy="16" r="3"/></svg>',
        'pydevices-examples': '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="4" y="4" width="16" height="16" rx="2"/><path d="M9 9h6M9 12h6M9 15h4"/><circle cx="17" cy="15" r="1.5"/></svg>',
        'pygraphics': '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><circle cx="12" cy="12" r="9"/><path d="M3 12h18M12 3a13 13 0 0 1 0 18 13 13 0 0 1 0-18z"/></svg>',
        'pdwidgets': '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="3" y="4" width="18" height="16" rx="2"/><path d="M7 8h10M7 12h6M7 16h8"/></svg>',
        'palettes': '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><circle cx="12" cy="12" r="9"/><path d="M12 3v18M3 12h18"/></svg>',
        'lvgl-bindings': '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="3" y="3" width="7" height="7" rx="1"/><rect x="14" y="3" width="7" height="7" rx="1"/><rect x="3" y="14" width="7" height="7" rx="1"/><rect x="14" y="14" width="7" height="7" rx="1"/></svg>',
        'lvgl-micropython': '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M4 17V7l8-4 8 4v10l-8 4z"/><path d="M4 7l8 4 8-4M12 11v10"/></svg>',
        'lvgl-python': '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M12 2a4 4 0 0 1 4 4v1h1a3 3 0 0 1 3 3v1a4 4 0 0 1-4 4h-1v1a4 4 0 0 1-4 4 4 4 0 0 1-4-4v-1H6a4 4 0 0 1-4-4v-1a3 3 0 0 1 3-3h1V6a4 4 0 0 1 4-4z"/></svg>',
        'lvgl-circuitpython': '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M12 3v3M12 18v3M5.6 5.6l2.1 2.1M16.3 16.3l2.1 2.1M3 12h3M18 12h3M5.6 18.4l2.1-2.1M16.3 7.7l2.1-2.1"/><circle cx="12" cy="12" r="4"/></svg>',
        'pyscript-template': '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="3" y="4" width="18" height="14" rx="2"/><path d="M8 21h8M12 18v3"/><path d="M8 9h8M8 13h5"/></svg>',
        'android-template': '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="6" y="2" width="12" height="20" rx="2"/><path d="M10 19h4"/></svg>',
        'mip': '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M21 16V8a2 2 0 0 0-1-1.7l-7-4a2 2 0 0 0-2 0l-7 4A2 2 0 0 0 3 8v8a2 2 0 0 0 1 1.7l7 4a2 2 0 0 0 2 0l7-4A2 2 0 0 0 21 16z"/><path d="M3.3 7L12 12l8.7-5M12 22V12"/></svg>',
        'mpftp': '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="6" y="3" width="12" height="18" rx="2"/><path d="M9 7h6M9 11h6M9 15h4"/><circle cx="15" cy="15" r="1"/><path d="M15 16v3M13 19h4"/></svg>',
        # Same glyph as pydevices-examples -- these are sub-pages of that repo.
        'pydevices-examples-gallery': '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="4" y="4" width="16" height="16" rx="2"/><path d="M9 9h6M9 12h6M9 15h4"/><circle cx="17" cy="15" r="1.5"/></svg>',
        'pydevices-examples-pyscript': '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="4" y="4" width="16" height="16" rx="2"/><path d="M9 9h6M9 12h6M9 15h4"/><circle cx="17" cy="15" r="1.5"/></svg>',
    }
    return icons.get(repo_name, '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><path d="M12 2L2 7l10 5 10-5-10-5z"/></svg>')

def get_tag_label(repo_name):
    tags = {
        'pydevices': 'Core Flagship',
        'displayif': 'C Bus Usermod',
        'audiodsp': 'C Audio Usermod',
        'audioif': 'C Audio Driver',
        'audiocomponents': 'Pure-Python Audio',
        'usbif': 'C USB Usermod',
        'cameraif': 'C Camera Usermod',
        'micropython-pydevices': 'Runtime Overlay',
        'workbench': 'Browser IDE',
        'mpvst': 'VST3 Plug-in',
        'pydevices-examples': 'Companion Showcase',
        'pygraphics': '0-Dependency',
        'pdwidgets': 'Pure-Python UI',
        'palettes': 'Color Engine',
        'lvgl-bindings': 'LVGL Generator',
        'lvgl-micropython': 'MicroPython C',
        'lvgl-python': 'CPython / Pyodide',
        'lvgl-circuitpython': 'CircuitPython C',
        'pyscript-template': 'PWA Template',
        'android-template': 'Android APK',
        'mip': 'MIP Index',
        'mpftp': 'IDE Extension'
    }
    return tags.get(repo_name, repo_name)

def validate_db(db, only=None):
    """Fail fast on a malformed database.

    The generator is run by hand, so this is where the check belongs -- it runs
    exactly when someone edits the database and regenerates. With `only`, the
    checks that need a sibling checkout on disk look at that repo alone: the
    rest of the workspace is not expected to be there.
    """
    if '_meta' not in db:
        raise SystemExit('repos_db.json is missing its _meta block')

    tiers = db['_meta']['tiers']
    problems = []
    for name, data in repos(db).items():
        # `is None`, not falsiness: the portal repo is legitimately tier 0.
        for field in ('tier', 'tier_name', 'description', 'buttons'):
            if data.get(field) is None:
                problems.append(f'{name}: missing {field}')
        tier = str(data.get('tier'))
        if tier not in tiers:
            problems.append(f'{name}: tier {tier} has no _meta.tiers entry')
        elif data.get('tier_name') != tiers[tier]['name']:
            problems.append(
                f'{name}: tier_name {data.get("tier_name")!r} disagrees with '
                f'_meta.tiers[{tier}].name {tiers[tier]["name"]!r}'
            )

        hero_canvas = data.get('hero_canvas')
        if hero_canvas is not None:
            if not isinstance(hero_canvas, dict):
                problems.append(f'{name}: hero_canvas must be an object')
            else:
                for req in ('app', 'width', 'height'):
                    if req not in hero_canvas:
                        problems.append(f'{name}: hero_canvas missing {req}')
                app_name = hero_canvas.get('app')
                app_path = os.path.join(ASSETS_DIR, 'apps', f'{app_name}.py')
                if app_name and not os.path.exists(app_path):
                    problems.append(f'{name}: hero_canvas app {app_name}.py not found in {os.path.relpath(app_path, BASE_DIR)}')

    valid_pages = {'portal-root', 'portal-subdir', 'self', 'self-subpath', 'none'}
    roots = [n for n, d in repos(db).items() if page_destination(d) == 'portal-root']
    for name, data in repos(db).items():
        destination = page_destination(data)
        on_disk = only is None or name == only
        if destination not in valid_pages:
            problems.append(
                f'{name}: page {destination!r} is not one of {sorted(valid_pages)}'
            )
        # page=none generates no landing page, so a hero there runs nowhere.
        if destination == 'none' and data.get('hero_canvas'):
            problems.append(
                f'{name}: hero_canvas on page=none; nothing generates a page to show it'
            )
        # A repo keeping its own Pages must still have somewhere to publish from.
        if on_disk and destination == 'self' and not os.path.isdir(
            os.path.join(BASE_DIR, name, '.site')
        ):
            problems.append(f'{name}: page=self but no .site/ directory')
        if on_disk and destination == 'self-subpath':
            site_repo = data.get('site_repo')
            site_subpath = data.get('site_subpath')
            if not site_repo or not site_subpath:
                problems.append(f'{name}: page=self-subpath needs site_repo and site_subpath')
            elif not os.path.isdir(os.path.join(BASE_DIR, site_repo, '.site', site_subpath)):
                problems.append(
                    f'{name}: page=self-subpath but no {site_repo}/.site/{site_subpath}/ directory'
                )
        # Every landing page carries exactly four CTA buttons: the house shape
        # (repo, docs, release or index, one neighbour). Gallery sub-pages
        # carry none, and page=none renders no hero at all.
        if destination in ('portal-root', 'portal-subdir', 'self'):
            count = len(data.get('buttons') or [])
            if count != 4:
                problems.append(f'{name}: {count} CTA buttons; a landing page has exactly 4')
    if len(roots) != 1:
        problems.append(
            f'expected exactly one page=portal-root repo, found {roots or "none"}'
        )

    if problems:
        raise SystemExit('repos_db.json is invalid:\n  ' + '\n  '.join(problems))
    print(f"[OK] Database valid: {len(repos(db))} repos across {len(tiers)} tiers")


def repos(db):
    """Repository entries only; keys beginning with '_' are metadata."""
    return {k: v for k, v in db.items() if not k.startswith('_')}


def tier_meta(db):
    """{int tier: (name, hint)} from the database, not hardcoded here."""
    return {int(t): (m['name'], m['hint']) for t, m in db['_meta']['tiers'].items()}


def build_ecosystem_markdown(db):
    """The org repository map, emitted from the same database that drives the sites."""
    meta = tier_meta(db)
    by_tier = {}
    for repo_name, data in repos(db).items():
        if repo_name == 'PyDevices.github.io' or not data.get('portal_grid', True):
            continue
        by_tier.setdefault(data.get('tier', 5), []).append((repo_name, data))

    lines = [
        '<!-- PYDEVICES-ECOSYSTEM: START -->',
        '<!-- Generated by dotgithub/scripts/generate_sites.py from data/repos_db.json.',
        '     Do not edit between these markers; edit the database instead. -->',
        '',
    ]
    for tier in sorted(by_tier):
        name, hint = meta[tier]
        lines += [f'### Tier {tier}: {name}', '', f'{hint}', '']
        for repo_name, data in by_tier[tier]:
            url = f'https://github.com/PyDevices/{repo_name}'
            lines.append(f'- [`{repo_name}`]({url}) — {data.get("description", "")}')
        lines.append('')

    # Repos that were folded into another one and archived. Their pages stay up
    # as pointers (portal_grid: false keeps them off the grid), and the map
    # says where each went so a reader of the map isn't sent to an archive.
    moved = db['_meta'].get('moved')
    if moved:
        lines += ['**Moved into other repositories:**', '']
        for repo_name, where in moved.items():
            lines.append(f'- **{repo_name}** — {where}')
        lines.append('')

    lines += ['**Deliberately absent from this map:**', '']
    for repo_name, reason in db['_meta']['excluded'].items():
        lines.append(f'- **{repo_name}** — {reason}')
    lines += ['', '<!-- PYDEVICES-ECOSYSTEM: END -->']
    return '\n'.join(lines)


ECOSYSTEM_TARGETS = (
    ('dotgithub', 'profile/README.md'),
    ('pydevices', 'docs/ecosystem.md'),
)


def write_ecosystem_markdown(db):
    """Rewrite the marker block in every consumer of the org map."""
    block = build_ecosystem_markdown(db)
    written = 0
    for repo_name, rel in ECOSYSTEM_TARGETS:
        path = os.path.join(BASE_DIR, repo_name, rel)
        if not os.path.exists(path):
            print(f"[SKIP] Ecosystem target missing: {path}")
            continue
        with open(path, 'r', encoding='utf-8') as f:
            content = f.read()
        updated = update_html_section(
            content,
            '<!-- PYDEVICES-ECOSYSTEM: START -->',
            '<!-- PYDEVICES-ECOSYSTEM: END -->',
            block,
        )
        if updated == content and '<!-- PYDEVICES-ECOSYSTEM: START -->' not in content:
            print(f"[SKIP] No ECOSYSTEM markers in {repo_name}/{rel}")
            continue
        with open(path, 'w', encoding='utf-8') as f:
            f.write(updated)
        written += 1
        print(f"[OK] Ecosystem map -> {repo_name}/{rel}")
    return written


def build_head_tags_html(repo_name, data):
    title = data.get('title', f'PyDevices - {repo_name}')
    return (
        f'  <!-- PYDEVICES-HEAD-TAGS: START -->\n'
        f'  <title>{title}</title>\n'
        f'  <meta name="description" content="{data.get("description", "")}">\n'
        f'  <link rel="icon" type="image/svg+xml" href="/assets/img/logo.svg">\n'
        f'  <!-- PYDEVICES-HEAD-TAGS: END -->'
    )

def build_above_the_fold_html(repo_name, data):
    # The hero shows the repo's own mark -- the same glyph as its portal card.
    # The portal keeps the org logo: it is the org's page, and get_card_icon
    # has no entry for it, so it would otherwise fall through to the generic
    # default. The glyphs are stroke="currentColor", hence color on the badge.
    mark = (
        '<img src="/assets/img/logo.svg" alt="PyDevices" width="112" height="112">'
        if page_destination(data) == 'portal-root'
        else get_card_icon(repo_name)
    )
    theme_color = data.get('theme_color', 'var(--tier-5-steel)')
    dark_gradient = get_gradient_dark(theme_color)
    # The org's own page shows the PyDevices mark free-standing on the ground.
    # Every product page keeps a tier-coloured tile: that badge is what carries
    # the repo's tier, so it stays even though the org mark no longer needs one.
    if page_destination(data) == 'portal-root':
        badge_html = f'<div class="logo-badge logo-badge--free">{mark}</div>'
    else:
        badge_html = (
            f'<div class="logo-badge product-mark" style="background: '
            f'linear-gradient(135deg, {theme_color}, {dark_gradient}); '
            f'color: var(--on-accent);">{mark}</div>'
        )
    eyebrow = data.get('eyebrow', repo_name)
    headline = data.get('headline', f'{repo_name} — PyDevices library.')
    description = data.get('description', '')
    # The org homepage and repo sub-pages (e.g. a gallery) aren't "a repo among
    # repos" -- skip the <code>reponame</code> prefix that every repo's own
    # top-level landing page uses to name itself.
    description_html = (
        description
        if page_destination(data) in ('portal-root', 'self-subpath')
        else f'<code>{repo_name}</code> {description}'
    )
    # Optional second hero paragraph (e.g. gallery pages' "click a card" note).
    hero_note = data.get('hero_note', '')

    button_html_list = []
    for btn in data.get('buttons', []):
        label, href = btn['label'], btn['href']
        if btn.get('primary', False):
            b_html = (
                f'      <a class="btn primary" style="background: var(--github-ink); color: var(--on-accent); border-color: var(--github-ink);" href="{href}">\n'
                f'        <svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><path d="M9 19c-5 1.5-5-2.5-7-3m14 6v-3.9a3.4 3.4 0 0 0-1-2.6c3-.3 6-1.5 6-6.5a5 5 0 0 0-1.4-3.5 4.6 4.6 0 0 0-.1-3.5s-1.1-.3-3.5 1.3a12 12 0 0 0-6 0C6.6 1.7 5.5 2 5.5 2a4.6 4.6 0 0 0-.1 3.5A5 5 0 0 0 4 9c0 5 3 6.2 6 6.5a3.4 3.4 0 0 0-1 2.6V22"/></svg>\n'
                f'        {label}\n'
                f'      </a>'
            )
        else:
            b_html = f'      <a class="btn" style="border-color: {theme_color}; color: {theme_color};" href="{href}">{label}</a>'
        button_html_list.append(b_html)

    buttons_joined = '\n'.join(button_html_list)

    hero_canvas = data.get('hero_canvas')
    if hero_canvas:
        app_name = hero_canvas.get('app', 'watch')
        width = hero_canvas.get('width', 240)
        height = hero_canvas.get('height', 240)
        shape = hero_canvas.get('shape', 'round')
        app_url = hero_canvas.get('app_url', f'https://PyDevices.github.io/assets/apps/{app_name}.py')
        canvas_id = 'hero_canvas'
        container_class = 'hero-canvas-circle' if shape == 'round' else 'hero-canvas-square'

        canvas_html = (
            f'    <div class="hero-canvas-wrap" data-hero-canvas="{canvas_id}" data-hero-app="{app_name}" data-hero-app-url="{app_url}">\n'
            f'      <div class="{container_class}">\n'
            f'        <div class="hero-canvas-loader">\n'
            f'          <div class="hero-spinner"></div>\n'
            f'          <span class="hero-canvas-status">Live Pure-Python Device</span>\n'
            f'        </div>\n'
            f'        <canvas id="{canvas_id}" width="{width}" height="{height}" tabindex="0"></canvas>\n'
            f'      </div>\n'
            f'    </div>'
        )

        hero_note_html = f'      <p>{hero_note}</p>\n' if hero_note else ''
        cta_html = (
            f'      <div class="cta">\n{buttons_joined}\n      </div>\n' if buttons_joined else ''
        )
        return (
            f'  <!-- PYDEVICES-ABOVE-THE-FOLD: START -->\n'
            f'  <div id="pydevices-site-header"></div>\n\n'
            f'  <!-- Hero Banner -->\n'
            f'  <section class="hero wrap has-hero-canvas">\n'
            f'    <div class="hero-main">\n'
            f'      <div class="hero-lead">\n'
            f'        {badge_html}\n'
            f'        <span class="eyebrow" style="color: {theme_color};">{eyebrow}</span>\n'
            f'      </div>\n'
            f'      <h1>{headline}</h1>\n'
            f'      <p>{description_html}</p>\n'
            f'{hero_note_html}'
            f'{cta_html}'
            f'    </div>\n'
            f'{canvas_html}\n'
            f'  </section>\n'
            f'  <!-- PYDEVICES-ABOVE-THE-FOLD: END -->'
        )

    hero_note_html = f'    <p>{hero_note}</p>\n' if hero_note else ''
    cta_html = (
        f'    <div class="cta">\n{buttons_joined}\n    </div>\n' if buttons_joined else ''
    )
    return (
        f'  <!-- PYDEVICES-ABOVE-THE-FOLD: START -->\n'
        f'  <div id="pydevices-site-header"></div>\n\n'
        f'  <!-- Hero Banner -->\n'
        f'  <section class="hero wrap">\n'
        f'    <div class="hero-lead">\n'
        f'      <div class="hero-brand">\n'
        f'        {badge_html}\n'
        f'        <span class="eyebrow" style="color: {theme_color};">{eyebrow}</span>\n'
        f'      </div>\n'
        f'      <h1>{headline}</h1>\n'
        f'    </div>\n'
        f'    <p>{description_html}</p>\n'
        f'{hero_note_html}'
        f'{cta_html}'
        f'  </section>\n'
        f'  <!-- PYDEVICES-ABOVE-THE-FOLD: END -->'
    )

def build_portal_grids_html(db):
    meta = tier_meta(db)

    tier_repos = {1: [], 2: [], 3: [], 4: [], 5: []}
    for repo_name, data in repos(db).items():
        if repo_name != 'PyDevices.github.io' and data.get('portal_grid', True):
            tier_repos[data.get('tier', 5)].append((repo_name, data))

    sections_html = []
    for tier in sorted(tier_repos.keys()):
        name, hint = meta[tier]
        title = f'{tier}: {name}'
        cards_html = []
        for repo_name, data in tier_repos[tier]:
            cards_html.append(
                f'      <a class="card card-tier-{tier}" href="/{repo_name}/">\n'
                f'        <div class="card-top">\n'
                f'          <span class="icon">{get_card_icon(repo_name)}</span>\n'
                f'          <span class="tag tag-tier-{tier}">{get_tag_label(repo_name)}</span>\n'
                f'        </div>\n'
                f'        <h3>{repo_name}</h3>\n'
                f'        <p>{data.get("description", "")}</p>\n'
                f'        <span class="go">Visit site <svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M5 12h14M13 6l6 6-6 6"/></svg></span>\n'
                f'      </a>'
            )

        cards_joined = '\n\n'.join(cards_html)
        sec_style = ' style="margin-top: 40px;"' if tier > 1 else ''

        sections_html.append(
            f'    <!-- Tier {tier} Section -->\n'
            f'    <div class="section-head"{sec_style}>\n'
            f'      <h2>{title}</h2>\n'
            f'      <span class="hint">{hint}</span>\n'
            f'    </div>\n'
            f'    <div class="grid">\n'
            f'{cards_joined}\n'
            f'    </div>'
        )

    return (
        f'  <!-- PYDEVICES-PORTAL-GRIDS: START -->\n'
        f'{"\n\n".join(sections_html)}\n'
        f'  <!-- PYDEVICES-PORTAL-GRIDS: END -->'
    )


MIP_PACKAGES_START = '<!-- PYDEVICES-MIP-PACKAGES: START -->'
MIP_PACKAGES_END = '<!-- PYDEVICES-MIP-PACKAGES: END -->'


def build_mip_packages_html(packages_path):
    """The "Packages in this index" section of the mip landing page.

    Reads mip/packages.json, which mip's refresh-package-list workflow
    regenerates (scripts/update_package_list.py) from the index itself after
    every deploy, never from PyPI or TestPyPI. Returns None when the sibling
    checkout has no packages.json so the caller can leave the marker block
    alone rather than blank it. mip's refresh workflow reruns this generator
    with `--only mip` and commits mip/.site with packages.json, then redeploys
    Pages, so the page moves with the list.
    """
    if not os.path.exists(packages_path):
        return None
    with open(packages_path, 'r', encoding='utf-8') as f:
        doc = json.load(f)
    packages = doc.get('packages', [])
    esc = html.escape

    rows = []
    for p in packages:
        source = f'{p["repository"]} @ {p["ref"]}'
        rows.append(
            f'          <tr>\n'
            f'            <td><code>{esc(p["name"])}</code></td>\n'
            f'            <td>{esc(p["version"])}</td>\n'
            f'            <td>{esc(p["description"])}</td>\n'
            f'            <td><a href="{esc(p["release_url"])}">{esc(source)}</a></td>\n'
            f'          </tr>'
        )
    installs = '\n'.join(esc(p['install']) for p in packages)
    hint = (
        f'{len(packages)} packages, each built from the source tag pinned in '
        f'pydevices-lock.json.'
    )
    return (
        f'    {MIP_PACKAGES_START}\n'
        f'    <div class="section-head" style="margin-top: 40px;">\n'
        f'      <h2>Packages in this index</h2>\n'
        f'      <span class="hint">{esc(hint)}</span>\n'
        f'    </div>\n'
        f'    <div class="spec-block">\n'
        f'      <p>These are the PyDevices packages this index publishes; the\n'
        f'      micropython-lib packages the index also carries are upstream\'s.</p>\n'
        f'      <table class="matrix-table">\n'
        f'        <thead>\n'
        f'          <tr><th>Package</th><th>Version</th><th>Description</th><th>Source</th></tr>\n'
        f'        </thead>\n'
        f'        <tbody>\n'
        f'{chr(10).join(rows)}\n'
        f'        </tbody>\n'
        f'      </table>\n'
        f'    </div>\n'
        f'    <div class="spec-block" style="margin-top: 24px;">\n'
        f'      <h3>Install</h3>\n'
        f'      <pre><code>import mip\n'
        f'\n'
        f'{installs}</code></pre>\n'
        f'    </div>\n'
        f'    {MIP_PACKAGES_END}'
    )


PORTAL_REPO = 'PyDevices.github.io'

# Written only when a portal subdirectory has no page yet; every run then
# rewrites the marker blocks in place, so this is a one-time scaffold.
PAGE_SKELETON = '''<!doctype html>
<html lang="en">
<head>
  <meta charset="utf-8">
  <meta name="viewport" content="width=device-width, initial-scale=1">
  <!-- PYDEVICES-HEAD-TAGS: START -->
  <!-- PYDEVICES-HEAD-TAGS: END -->
  <link rel="stylesheet" href="/assets/chrome/site.css">
</head>
<body>
  <!-- PYDEVICES-ABOVE-THE-FOLD: START -->
  <!-- PYDEVICES-ABOVE-THE-FOLD: END -->
  <div id="pydevices-site-footer"></div>
  <script src="/assets/chrome/site-chrome.js"></script>
</body>
</html>
'''


def asset_prefix(data):
    """Prefix for shared-asset URLs in this repo's page."""
    return '/'


def page_destination(data):
    """Where this repo's landing page is written.

    'portal-root'   the portal's own index.html
    'portal-subdir' PyDevices.github.io/<portal_path or repo>/index.html
    'self'          the repo's own .site/index.html, for repos that still
                    publish their own Pages because they serve real payload
    'self-subpath'  <site_repo>/.site/<site_subpath>/index.html -- a
                    sub-page of another repo's own Pages (e.g. a gallery),
                    generated as a database entry but excluded from the
                    portal grid/ecosystem map via portal_grid: false. The
                    rest of that page (its own GEN:* marker blocks) stays
                    owned by that repo's own generator.
    'none'          no generated page; the repo keeps only its portal card
    """
    return data.get('page', 'portal-subdir')


def _copy_chrome_into(site_root):
    chrome_dir = os.path.join(site_root, 'assets/chrome')
    img_dir = os.path.join(site_root, 'assets/img')
    os.makedirs(chrome_dir, exist_ok=True)
    os.makedirs(img_dir, exist_ok=True)

    for fname in ('site.css', 'site-chrome.js', 'theme-toggle.js', 'hero-runtime.js', 'docs-runtime.js'):
        src = os.path.join(ASSETS_DIR, 'js' if fname.endswith('.js') else 'css', fname)
        if os.path.exists(src):
            shutil.copy2(src, os.path.join(chrome_dir, fname))

    for img_name in ('logo.svg', 'logo-512.png', 'logo-avatar.png'):
        src_img = os.path.join(ASSETS_DIR, 'img', img_name)
        if os.path.exists(src_img):
            shutil.copy2(src_img, os.path.join(img_dir, img_name))


DOCS_THEME_REPOS = ('pygraphics', 'pdwidgets', 'palettes')


def sync_docs_theme():
    """Copy the shared Material for MkDocs stylesheet into the repos that publish docs sites.

    assets/css/docs-extra.css is the master; each repo's
    docs/stylesheets/extra.css is a pure sync target, exactly like the
    chrome under PyDevices.github.io/assets/. Read the Docs builds those
    repos independently, so the synced copies must be committed in their
    own repositories for a palette change to reach the published docs.
    """
    src = os.path.join(ASSETS_DIR, 'css', 'docs-extra.css')
    if not os.path.exists(src):
        print("[SKIP] docs theme: assets/css/docs-extra.css is missing")
        return
    for repo in DOCS_THEME_REPOS:
        dest_dir = os.path.join(BASE_DIR, repo, 'docs', 'stylesheets')
        if not os.path.isdir(dest_dir):
            print(f"[SKIP] {repo}: no docs/stylesheets (checkout missing)")
            continue
        shutil.copy2(src, os.path.join(dest_dir, 'extra.css'))
        print(f"[SYNC] {repo}/docs/stylesheets/extra.css")


def sync_assets(db):
    """Copy the shared chrome and apps to the portal repository root (PyDevices.github.io)."""
    portal_root = os.path.join(BASE_DIR, PORTAL_REPO)
    _copy_chrome_into(portal_root)

    # Sync static standalone apps into portal assets/apps/
    apps_src = os.path.join(ASSETS_DIR, 'apps')
    if os.path.exists(apps_src):
        portal_apps = os.path.join(portal_root, 'assets/apps')
        os.makedirs(portal_apps, exist_ok=True)
        for app_file in os.listdir(apps_src):
            shutil.copy2(os.path.join(apps_src, app_file), os.path.join(portal_apps, app_file))

    # The first-party PyScript template is intentionally Pyodide-only and is
    # published beneath its generated portal landing page. Keep its complete
    # offline PWA payload together, including the pinned vendored interpreter.
    template_pwa = os.path.join(BASE_DIR, 'pyscript-template', 'pwa')
    if os.path.isdir(template_pwa):
        shutil.copytree(
            template_pwa,
            os.path.join(portal_root, 'pyscript-template', 'pwa'),
            dirs_exist_ok=True,
        )

def update_html_section(content, marker_start, marker_end, new_html):
    # Consume any existing indentation before the start marker. The replacement
    # carries its own, so without this every run prepends another level and the
    # marker drifts right forever.
    pattern = re.compile(
        r'[ \t]*' + re.escape(marker_start) + r'.*?' + re.escape(marker_end),
        re.DOTALL,
    )
    if pattern.search(content):
        # A lambda, not a string: the replacement is arbitrary HTML/markdown and
        # must never be scanned for backreference escapes.
        return pattern.sub(lambda _m: new_html, content)
    return content

def get_site_html_path(repo_name, data):
    """Absolute path of the index.html this repo's entry writes, or None."""
    destination = page_destination(data)
    portal = os.path.join(BASE_DIR, PORTAL_REPO)
    if destination == 'portal-root':
        return os.path.join(portal, 'index.html')
    if destination == 'portal-subdir':
        return os.path.join(portal, data.get('portal_path', repo_name), 'index.html')
    if destination == 'self':
        return os.path.join(BASE_DIR, repo_name, '.site/index.html')
    if destination == 'self-subpath':
        return os.path.join(BASE_DIR, data['site_repo'], '.site', data['site_subpath'], 'index.html')
    return None

def parse_args(argv=None):
    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    parser.add_argument(
        '--only', metavar='REPO',
        help="rewrite only this repository's page; skip the shared assets, "
             "the docs theme and the ecosystem maps",
    )
    return parser.parse_args(argv)


def main(argv=None):
    args = parse_args(argv)
    print("=== Pure Harmonized PyDevices Site Generator (.github) ===")
    
    with open(DB_PATH, 'r', encoding='utf-8') as f:
        db = json.load(f)

    validate_db(db, only=args.only)

    if args.only is not None and args.only not in repos(db):
        raise SystemExit(f"--only {args.only}: not in data/repos_db.json")

    updated_sites = 0

    if args.only is None:
        sync_assets(db)
        sync_docs_theme()

    for repo_name, data in repos(db).items():
        if args.only is not None and repo_name != args.only:
            continue
        site_html_path = get_site_html_path(repo_name, data)
        if site_html_path is None:
            print(f"[SKIP] {repo_name}: no generated page (page=none)")
            continue

        # A portal subdirectory is ours to create; a repo's own .site is not.
        if page_destination(data) in ('portal-root', 'portal-subdir'):
            os.makedirs(os.path.dirname(site_html_path), exist_ok=True)
            if not os.path.exists(site_html_path):
                with open(site_html_path, 'w', encoding='utf-8') as f:
                    f.write(PAGE_SKELETON)
                print(f"[NEW] Created {os.path.relpath(site_html_path, BASE_DIR)}")

        if not os.path.exists(site_html_path):
            print(f"[SKIP] HTML file missing: {site_html_path}")
            continue

        with open(site_html_path, 'r', encoding='utf-8') as f:
            content = f.read()

        # Update section markers cleanly
        # A page with no marker blocks is not a page this generator manages --
        # report it rather than claiming success. pydevices sat like this for
        # months after its landing page was replaced by a redirect stub.
        missing = [
            marker for marker in ('PYDEVICES-HEAD-TAGS', 'PYDEVICES-ABOVE-THE-FOLD')
            if f'<!-- {marker}: START -->' not in content
        ]
        if missing:
            print(f"[WARN] {repo_name}: no {', '.join(missing)} marker block -- nothing written")
            continue

        content = update_html_section(content, '<!-- PYDEVICES-HEAD-TAGS: START -->', '<!-- PYDEVICES-HEAD-TAGS: END -->', build_head_tags_html(repo_name, data))
        content = update_html_section(content, '<!-- PYDEVICES-ABOVE-THE-FOLD: START -->', '<!-- PYDEVICES-ABOVE-THE-FOLD: END -->', build_above_the_fold_html(repo_name, data))

        if repo_name == 'PyDevices.github.io':
            content = update_html_section(content, '<!-- PYDEVICES-PORTAL-GRIDS: START -->', '<!-- PYDEVICES-PORTAL-GRIDS: END -->', build_portal_grids_html(db))

        # mip's below-the-fold package list, from the sibling checkout's
        # packages.json. Absent file or absent markers: log and leave the
        # block as it is -- never blank a list that was there.
        if repo_name == 'mip':
            packages_path = os.path.join(BASE_DIR, 'mip', 'packages.json')
            packages_html = build_mip_packages_html(packages_path)
            if packages_html is None:
                print(f"[SKIP] mip: no {os.path.relpath(packages_path, BASE_DIR)} -- package list not written")
            elif MIP_PACKAGES_START not in content:
                print("[WARN] mip: no PYDEVICES-MIP-PACKAGES marker block -- package list not written")
            else:
                content = update_html_section(content, MIP_PACKAGES_START, MIP_PACKAGES_END, packages_html)

        # Remove a duplicate hero section left outside the marker block.
        marker = '<!-- PYDEVICES-ABOVE-THE-FOLD: END -->'
        if marker in content:
            head, tail = content.split(marker, 1)
            tail = re.sub(r'<section\s+class=["\']hero\s+wrap["\']>.*?</section>', '', tail, flags=re.DOTALL | re.IGNORECASE)
            content = head + marker + tail

        with open(site_html_path, 'w', encoding='utf-8') as f:
            f.write(content)

        updated_sites += 1
        print(f"[OK] Generated & Updated {repo_name} ({os.path.relpath(site_html_path, BASE_DIR)})")

    if args.only is not None:
        # A page asked for by name and not written is a failure: the caller
        # commits whatever is on disk and would otherwise commit a stale page.
        if updated_sites != 1:
            raise SystemExit(f"--only {args.only}: page not written")
        print(f"=== Complete! Processed {args.only} only ===")
        return

    written = write_ecosystem_markdown(db)

    print(f"=== Complete! Processed {len(repos(db))} repos "
          f"({updated_sites} sites, {written} ecosystem maps) ===")

if __name__ == '__main__':
    main()