pkg-attach and pkg-audit: an uploaded pkg that no group carries is inert

orchestrator/orchestrator/cli.py

@@ -255,6 +255,43 @@ def add_to_group(
     )
 
 
+@_app.command()
+def pkg_audit(
+    include_store: bool = typer.Option(
+        False, "--include-store", help="Also consider apple-store apps (noisy; they reach devices "
+        "by other means)."
+    ),
+) -> None:
+    """Which uploaded pkgs is no assignment group carrying? (read-only)
+
+    Uploading a pkg and attaching it are separate steps in SimpleMDM, and an unattached app is
+    inert with nothing surfacing that fact. Run this after any upload. Also flags the same bundle
+    id uploaded twice, where which copy a group carries decides what devices get.
+    """
+    workflow.step_pkg_audit(include_store=include_store)
+
+
+@_app.command()
+def pkg_attach(
+    app: str = typer.Argument(..., help="App id, or a unique substring of its name/bundle id."),
+    group_id: int = typer.Option(
+        0, "--group-id", help="Group to attach to (default: settings.bootstrap_group_id)."
+    ),
+    push: bool = typer.Option(
+        False,
+        "--push",
+        help="Force delivery now. Re-pushes EVERY app in the group to EVERY member, including "
+        "postinstalls — never do this to a group carrying the bootstrap pkg while hosts are busy.",
+    ),
+) -> None:
+    """Attach an uploaded pkg to a group and verify the group really carries it.
+
+    Verifies by re-reading the group, not by trusting the POST. Production groups are refused:
+    attaching a new app there pushes it to every member mid-task.
+    """
+    workflow.step_pkg_attach(app, group_id=group_id or None, push=push)
+
+
 @_app.command()
 def group_parity(
     group_id: int = typer.Option(

orchestrator/orchestrator/clients/simplemdm.py

@@ -190,6 +190,42 @@ def push_apps(group_id: int) -> None:
     _request("POST", f"/assignment_groups/{group_id}/push_apps")
 
 
+def apps() -> list[dict]:
+    """Every app (pkg) in the account. One paginated sweep."""
+    return _paginated("/apps")
+
+
+def assignment_group_app_ids(group_id: int) -> list[int]:
+    """App IDs attached to the group.
+
+    NB: relationship IDs come back as ints here, as with devices.
+    """
+    rel = get_assignment_group(group_id).get("relationships", {})
+    return [int(a["id"]) for a in rel.get("apps", {}).get("data", [])]
+
+
+def add_app_to_assignment_group(group_id: int, app_id: int) -> None:
+    """Attach an app to an assignment group. This is what makes an upload do anything.
+
+    Uploading a pkg and attaching it are separate operations in SimpleMDM, and an unattached app
+    is completely inert with nothing surfacing that fact — p_role_tart_worker sat uploaded and
+    unattached on 2026-08-19 while the hosts it was built for went on with no role file.
+
+    Production groups are refused, using the same guard as the device writes. Attaching a NEW app
+    to a live production group pushes it to every member: 2017918 had 130+ devices taking work.
+    That is the "never add the bootstrap pkg to a production group" footgun in its other form, and
+    a change with that blast radius should be made in the UI with a human looking at it.
+    """
+    if group_id in PROTECTED_GROUP_IDS:
+        raise ReprovisionError(
+            f"refusing to attach app {app_id} to assignment group {group_id} "
+            f"({PROTECTED_GROUP_IDS[group_id]}). Attaching an app there pushes it to every member, "
+            "mid-task. Attach to the bootstrap/staging group instead, or do it in the UI "
+            "deliberately."
+        )
+    _request("POST", f"/assignment_groups/{group_id}/apps/{app_id}")
+
+
 def wipe(device_id: int, *, obliteration_behavior: str = "DoNotObliterate") -> None:
     """
     Erase the device. Default `DoNotObliterate` = EACS-only: if Erase All Content & Settings

orchestrator/orchestrator/workflow.py

@@ -584,6 +584,167 @@ def step_group_parity(
     )
 
 
+def _resolve_app(spec: str) -> dict:
+    """An app by id, or by a unique substring of its name or bundle identifier."""
+    catalog = simplemdm.apps()
+    if spec.isdigit():
+        for a in catalog:
+            if int(a["id"]) == int(spec):
+                return a
+        raise ReprovisionError(f"no app with id {spec} in this SimpleMDM account")
+
+    needle = spec.lower()
+    hits = [
+        a for a in catalog
+        if needle in (a.get("attributes", {}).get("name") or "").lower()
+        or needle in (a.get("attributes", {}).get("bundle_identifier") or "").lower()
+    ]
+    if not hits:
+        raise ReprovisionError(f"no app matching {spec!r} — check the name or pass the numeric id")
+    if len(hits) > 1:
+        listed = "\n  ".join(
+            f"{a['id']}  {a['attributes'].get('name')!r}  {a['attributes'].get('bundle_identifier')}"
+            for a in hits[:8]
+        )
+        raise ReprovisionError(
+            f"{spec!r} matches {len(hits)} apps — narrow it or pass the id:\n  {listed}"
+        )
+    return hits[0]
+
+
+def step_pkg_audit(*, include_store: bool = False) -> None:
+    """Which uploaded pkgs are attached to no assignment group? Read-only, API-only.
+
+    Uploading a pkg and attaching it to a group are separate operations in SimpleMDM, and an
+    unattached app is completely inert with nothing surfacing that fact. On 2026-08-19
+    `p_role_tart_worker` was uploaded and left unattached while the eight hosts it was built for
+    carried no role file — invisible until someone thought to check the group's app list by hand.
+
+    Only `app_type: custom` apps are considered by default — the pkgs we build and upload. The
+    account also holds ~20 `apple store` apps (1Password, Duo, Google Drive, ...) for iOS devices,
+    which legitimately reach devices by other means; including them buried the three real findings
+    in noise on the first run. `include_store` restores the unfiltered view.
+
+    Also reports duplicate bundle identifiers, attached or not: two uploads of the same pkg is its
+    own hazard, because which one a group carries decides what devices get. The r8 role pkg is
+    currently uploaded twice (`-Signed` and `-wrapped`, both
+    `com.github.munki.pkg.p_role_gecko_t_osx_1400_r8`).
+
+    CAVEAT: this only looks at assignment groups. An app reaching devices solely through a legacy
+    *device* group would be reported here as an orphan. That is the right default for this fleet —
+    verified 2026-08-19 that the m4 and tart devices all have `device_group_id: None`, so
+    assignment groups are the only live delivery path — but check before acting on a surprise.
+    """
+    ui.step("PKG AUDIT", "uploaded pkgs that no assignment group carries (read-only)")
+
+    carried: dict[int, list[str]] = {}
+    for group in simplemdm.assignment_groups():
+        gname = group.get("attributes", {}).get("name", "?")
+        for app in group.get("relationships", {}).get("apps", {}).get("data", []):
+            carried.setdefault(int(app["id"]), []).append(f"{group['id']} ({gname})")
+
+    everything = simplemdm.apps()
+    catalog = everything if include_store else [
+        a for a in everything if a.get("attributes", {}).get("app_type") == "custom"
+    ]
+    scope = "app(s)" if include_store else "custom pkg(s)"
+    ui.info(
+        f"{len(catalog)} {scope} in the account "
+        f"({len(everything) - len(catalog)} apple-store app(s) excluded), "
+        f"{len(carried)} app(s) attached to at least one group"
+    )
+
+    # Duplicate bundle ids: two uploads of the same pkg, where which one a group carries decides
+    # what devices actually get.
+    by_bundle: dict[str, list[dict]] = {}
+    for a in catalog:
+        bundle = a.get("attributes", {}).get("bundle_identifier") or ""
+        if bundle:
+            by_bundle.setdefault(bundle, []).append(a)
+    # Only duplicates where at least one copy is attached to nothing. Sharing a bundle id across
+    # deliberate per-flavour variants is normal here and all-attached sets are not findings:
+    # com.mozilla.pkg.SignerBootstrap has six copies, one per signer group (vpn/tb/fx/dep/adhoc/
+    # ff-ent), and puppet-agent's ARM and Intel builds share com.puppetlabs.puppet-agent. Flagging
+    # those buried the one real case. A duplicate that includes a stray upload is the smell.
+    dupes = {
+        b: v for b, v in by_bundle.items()
+        if len(v) > 1 and any(int(a["id"]) not in carried for a in v)
+    }
+    if dupes:
+        ui.warn(f"{len(dupes)} bundle id(s) uploaded more than once with a copy attached to nothing:")
+        for bundle, group in sorted(dupes.items()):
+            ui.warn(f"    {bundle}")
+            for a in sorted(group, key=lambda a: int(a["id"])):
+                where = carried.get(int(a["id"]))
+                ui.warn(f"        {a['id']}  {a['attributes'].get('name')!r}  "
+                        f"{'carried by ' + ', '.join(where) if where else 'ATTACHED TO NOTHING'}")
+
+    orphans = [a for a in catalog if int(a["id"]) not in carried]
+    if not orphans:
+        ui.ok(f"every {scope.rstrip('(s)')} is attached to at least one group")
+        return
+
+    ui.warn(f"{len(orphans)} {scope} attached to NOTHING — uploaded but inert:")
+    for a in sorted(orphans, key=lambda a: int(a["id"])):
+        at = a.get("attributes", {})
+        ui.warn(f"    {a['id']}  {at.get('name')!r}  bundle={at.get('bundle_identifier')}")
+    ui.info("attach one with:  reprovision pkg-attach <id> --group-id <group>")
+
+
+def step_pkg_attach(app_spec: str, *, group_id: int | None = None, push: bool = False) -> None:
+    """Attach an uploaded pkg to an assignment group, then VERIFY the group really carries it.
+
+    Verifies by re-reading the group rather than trusting the POST, for the same reason
+    step_add_to_group does: with this API a write returning 2xx is not proof the state changed.
+
+    `push` is OFF by default and that is deliberate. `push_apps` re-pushes EVERY app in the group
+    to EVERY member, so pushing the m4 bootstrap group would re-run the bootstrap pkg's postinstall
+    on hosts that are mid-task. Without a push the pkg still lands, just on each device's next
+    check-in — which on these boxes is often boot-only, so it can take a reboot. Choose knowingly:
+    the count of affected devices is printed either way.
+    """
+    s = get_settings()
+    gid = group_id or s.bootstrap_group_id
+    app = _resolve_app(app_spec)
+    aid = int(app["id"])
+    at = app.get("attributes", {})
+
+    group = simplemdm.get_assignment_group(gid)
+    gname = group.get("attributes", {}).get("name", "?")
+    members = len(simplemdm.assignment_group_device_ids(gid))
+
+    ui.step("PKG ATTACH", "make an uploaded pkg actually reach hosts")
+    ui.info(f"app {aid} = {at.get('name')!r}  bundle={at.get('bundle_identifier')}")
+    ui.info(f"group {gid} = {gname} ({members} device(s))")
+
+    if aid in simplemdm.assignment_group_app_ids(gid):
+        ui.ok(f"already attached to {gname} — no change")
+    else:
+        ui.wire(f"POST /assignment_groups/{gid}/apps/{aid}")
+        simplemdm.add_app_to_assignment_group(gid, aid)
+        if push:
+            ui.warn(
+                f"pushing: re-pushes every app in {gname} to all {members} device(s), including "
+                "any postinstall they run"
+            )
+            ui.wire(f"POST /assignment_groups/{gid}/push_apps")
+            simplemdm.push_apps(gid)
+        else:
+            ui.info(
+                f"not pushing — the pkg lands on each of the {members} device(s) at its next "
+                "check-in (often boot-only). Pass --push to force it now."
+            )
+
+    # Re-read: the POST returning 2xx is not evidence the group changed.
+    after = simplemdm.assignment_group_app_ids(gid)
+    if aid not in after:
+        raise ReprovisionError(
+            f"attach reported success but group {gid} ({gname}) still does not list app {aid}. "
+            "Check the group in SimpleMDM before assuming the pkg will be delivered."
+        )
+    ui.ok(f"verified: {gname} carries app {aid} (group now has {len(after)} app(s))")
+
+
 def step_validate(ctx: HostContext, *, expected_refresh_hz: float | None = None) -> None:
     """Read-only fitness check on a bootstrapped host: is it actually able to run tasks?