diff --git a/CMakeLists.txt b/CMakeLists.txt index 288321f6a..3225a890e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -265,6 +265,11 @@ add_test(NAME pack_version COMMAND ${CMAKE_SOURCE_DIR}/tests/pack_version_test.s # resolves outside the payload, and a runtime set shipped without the aqlprofile agent libhsa opens. Each # arm asserts the row's message, and a clean tree is the control. No package build, no GPU, runs in CI. add_test(NAME pack_check COMMAND ${CMAKE_SOURCE_DIR}/tests/pack_check_test.sh) +# engine #1504: the GPU hold queue is the FLEET's when the shared state is installed, with no environment to say +# so - the failure it guards against is silent (a caller under another account taking its own token). Two arms: +# fleet state present -> the claim uses the fleet's queue and token; absent -> the old ~/GPU-* paths, so a box +# that has not been set up is unchanged. No GPU: the claim holds the token for three seconds and runs `true`. +add_test(NAME gpu_hold_queue COMMAND ${CMAKE_SOURCE_DIR}/tests/gpu_hold_queue_test.sh) # engine #713: the pinned HRX toolchain (config/hrx-toolchain.config, scripts/fetch-rocm-sdk.sh) -- the # pin file's refusals, the completeness check against a fabricated tree, and that cmake and # weekly.sh both use the pin instead of the box's /opt/rocm-therock. No network, no SDK, no GPU. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dbd2de9db..71e81462e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -70,6 +70,10 @@ an RFC or issue. - **PR-Agent** reviews every pull request with a local model (Qwen3-Coder-30B-A3B) on the Strix Halo box (`.github/workflows/pr-agent.yml`); comment `/review`, `/describe`, `/improve` or `/ask` for more. +- **The box's runners.** `pr-agent` and `hrx-build` run on Strix Halo's own self-hosted runners, because those + jobs need the GPU, the NPU and the local LLM. How they are installed — the dedicated user, the restart policy + and the OOM guard that engine [#1504](https://github.com/1bit-MONSTER/engine/issues/1504) asked for — is + [docs/ci-runners.md](docs/ci-runners.md). - **Issue agent** (`tools/issue_agent.py`) takes a first pass on issues opened by people outside the project: one label, a request for the bug-report fields when a bug skipped the form, a registry lookup for linked Hugging Face models, and a pointer to Q&A for questions. The model only picks diff --git a/docs/ci-runners.md b/docs/ci-runners.md new file mode 100644 index 000000000..0550eedf4 --- /dev/null +++ b/docs/ci-runners.md @@ -0,0 +1,277 @@ + + +# Self-hosted CI runners on the Strix Halo box + +Two runners serve this repository, both on Strix Halo, both installed by +`scripts/install-ci-runner.sh`: + +| runner | labels | takes | +|---|---|---| +| `strixhalo-engine` | `self-hosted,Linux,X64,pr-agent,strixhalo` | `pr-agent`, `issue-agent` | +| `strixhalo-bench` | `self-hosted,Linux,X64,strixhalo,bench` | `hrx-build` and the bench/gate jobs | + +Everything the GitHub-hosted jobs need comes from `ubuntu-*` runners; these two exist because those jobs need +*this box* — its GPU, its NPU and the local LLM at `127.0.0.1:8090`. + +## Why this page exists (engine #1504) + +On 2026-11-08 both runners were OOM-killed at 08:37 and **stayed dead until 12:23**: + +``` +$ systemctl show actions.runner.1bit-MONSTER-engine.strixhalo-bench.service -p Result -p NRestarts +Result=oom-kill +NRestarts=0 +``` + +`NRestarts=0` is the whole bug. The units shipped with no restart policy, and `runsvc.sh` exits **0** after the +listener stops ("Runner listener exit with 0 return code, stop the service, no retry needed"), so even +`Restart=on-failure` would never have fired. While they were down, 9 `hrx-build` and 15 `pr-agent` jobs queued, +and every check that needs the box read as *pending* — never green, never red, which looks exactly like "queued +behind a busy runner". It had happened before (2026-11-06, 41 `pr-agent` runs queued behind the same unit). + +A check whose runner is gone is the same silent-absence class as [engine #1077](https://github.com/1bit-MONSTER/engine/issues/1077); +a restart policy is what turns a four-hour outage into a thirty-second one. + +## The unit, and why each line is there + +`scripts/install-ci-runner.sh` writes this (values shown for the defaults): + +```ini +[Unit] +Description=GitHub Actions runner strixhalo-engine (dedicated user ghrunner) +After=network-online.target +Wants=network-online.target +StartLimitIntervalSec=0 + +[Service] +Type=simple +User=ghrunner +WorkingDirectory=/var/lib/ghrunner/actions-runner-strixhalo-engine +ExecStart=/var/lib/ghrunner/actions-runner-strixhalo-engine/runsvc.sh +Restart=always +RestartSec=30 +OOMPolicy=continue +MemoryHigh=43G +MemoryMax=48G +TasksMax=8192 +KillMode=process +KillSignal=SIGTERM +TimeoutStopSec=5min + +[Install] +WantedBy=multi-user.target +``` + +- **`Restart=always`, not `on-failure`** — because `runsvc.sh` reports success when it is killed. `always` is the + only value that brings a runner back from that exit code. `RestartSec=30` keeps a crash-looping runner from + hammering the box and the GitHub API. +- **`StartLimitIntervalSec=0`** — a runner must never be rate-limited out of existence. The 30-second spacing is + the rate limit; systemd giving up after five starts in ten seconds (the default) is how "stays dead" happens + again. +- **`OOMPolicy=continue`** — this is the line the old units lacked. With the default (`stop`), a process in the + cgroup being OOM-killed stops the *unit* with `Result=oom-kill`. With `continue`, the kernel's victim is the + one that dies — the job — and the runner keeps running, so the job fails with a reason instead of the runner + vanishing. +- **`MemoryHigh`/`MemoryMax`** — the cgroup hits its own ceiling before the box's OOM killer has to choose + between "the largest allocation on the machine" and everything else. The default 48G matches the bench rule + (`systemd-run --user --scope -p MemoryMax=48G`), so a heavy job is killed by *its* limit, not by the box's. +- **`KillMode=process`** with the 5-minute stop timeout — the behaviour the shipped units already had: stop the + listener, let the job's own timeout end its children. + +## The dedicated user + +The jobs used to run as the owner (`User=bcloud`). They now run as `ghrunner`: no shell, home +`/var/lib/ghrunner`, and the runner's `_work` tree under it. Anything a job writes into `$HOME` lands there, not +in the owner's tree, and the account holds no ssh keys and no fleet git remotes of its own. + +**It is not read-isolation of the owner's files, though** — `/home/bcloud` is `drwxrwxr-x` on this box, so a job +as `ghrunner` can read whatever in the owner's home is world-readable. The isolation is per-process and +per-account (a job cannot use the owner's keys to push, and its `$HOME` writes are its own); if read-isolation is +wanted too, that is `chmod o-rx /home/bcloud` — the owner's call, because it changes the box for every other lane +as well. + +### One queue, two accounts + +The workflows reach the box's hold state through `$HOME`: `~/GPU-TOKEN`, `~/GPU-QUEUE` (and `npu-hold`, which +hard-codes `~/NPU-TOKEN` and `~/NPU-QUEUE` around `gpu-hold`). A runner with a different home therefore does not +merely lose `thermal-run.sh` — it takes **its own** GPU token, and the hardware gate then runs GPU work +concurrently with whatever lane holds the real one, which is the one thing the hold rule exists to prevent. + +The obvious repair — put the runner's account in the owner's group — is wrong, and the review of this PR measured +why: `/home/bcloud` is `0775`, so group write on the **directory** lets a job rename or replace any entry in it. +A job could rewrite `~/.local/bin/gpu-hold` (which every lane then executes), replace `~/.bashrc`, or move +`~/.ssh` aside and put its own directory in its place. That is an owner-level code path created by a recipe whose +whole point is that jobs no longer run as the owner. + +So the hold state lives in its own directory, owned by its own group: + +``` +/var/lib/1bit-fleet/ 0770 fleethold, setgid + gpu-token 0660 fleethold <- what gpu-hold writes, and rm -f's on release + gpu-queue/ 0770 fleethold, setgid <- tickets inherit the group + npu-token, npu-queue/ +``` + +- `ghrunner` joins **`fleethold` and nothing else**. The shared tools (`~/lb`, `~/.local/bin`) are already + world-readable and executable, so no group is needed to *run* them — only the hold files need the group. +- The unit gets `Environment=GPU_TOKEN_FILE=/var/lib/1bit-fleet/gpu-token GPU_QUEUE_DIR=/var/lib/1bit-fleet/gpu-queue`, + which `gpu-hold` honours, so a job queues with the fleet. +- **The owner's side has to name the same two paths**, or the split simply reappears in the other direction. The + installer writes `/etc/profile.d/1bit-fleet.sh` exporting them for the owner's shells; a long-running owner + service needs them in its own unit or environment. +- **Why a shared *file* inside the owner's home cannot work at all:** `gpu-hold` releases with + `rm -f "$TOKEN"`, a write to the containing **directory**. `chgrp` on the file is not enough, and a symlink is + worse — `rm` removes the link and leaves the real token held for ever. +- `npu-hold` wraps `gpu-hold` with `exec env GPU_QUEUE_DIR=~/NPU-QUEUE GPU_TOKEN_FILE=~/NPU-TOKEN …`, so it + overrides whatever the environment says: **NPU sharing needs that wrapper changed** to honour the two + variables. Until then the NPU queue is per-account, and that is a named follow-up rather than a claim. +### The tools are reached by a home-relative path + +`hrx-build.yml:111,132` and `hw-gate.yml` call `~/lb/thermal-run.sh`, and `hw-gate.sh` resolves `gpu-hold` with +`command -v` — a **home-relative path**, not a permission. So the installer links `~/lb` and `~/.local/bin` from the +runner's home into the account that owns them and puts `.local/bin` first on the unit's `PATH`. The runner reaches +both through `other` (`r-x`), so it can **run** the tools and cannot rewrite them — which is the difference between +this and the owner's group, and the reason the links are back after the review's first round. + +`thermal-run.sh` runs its jobs under `systemctl --user`, so the installer also does +`loginctl enable-linger `; without a user manager for that account it falls back to a process-group +`SIGSTOP`, which is the weaker thermal brake the script itself warns about. + +`hw-gate.sh` has two more `$HOME` defaults (`:98-99`), and both resolve into an empty home for a runner under +another account: + +| variable | default | the installer sets it to | +|---|---|---| +| `HW_GATE_MODELS` | `$HOME/models` | `--owner-user`'s `models/` when it exists — read-only | +| `HW_GATE_LOCK` | `$HOME/npu.lock` | `/npu.lock` — the **fleet's** lock, so a runner cannot take NPU work on a lock of its own (the NPU version of the GPU-token bypass) | + +The same reasoning is applied at the other end: `scripts/hw-gate.sh` now prefers `/npu.lock` when that +directory exists, and `scripts/gpu-hold` prefers `/{gpu-token,gpu-queue}`, so **the default is what +moved, not an environment variable**. That matters because the lanes are long-lived tmux/`pi` sessions and +`systemd --user` services, none of them login shells: `/etc/profile.d` reaches none of them, and a caller without +the variables would otherwise fall back to `~/GPU-TOKEN` and silently fork the queue. `ONEBIT_HOLD_DIR` overrides +the directory if it ever lives elsewhere. + +One consequence worth writing down, found by `tests/gpu_hold_queue_test.sh`: `gpu-hold` treats the token file +**existing** as "held" — the holder creates it with `noclobber` — so the installer must not pre-create an empty +`gpu-token`. It creates the directory and the queues, and lets the first claim create the token. + +- `--no-hold-state` exists for a runner that runs no hardware jobs — the `pr-agent` runner does not (it talks to + the local LLM over HTTP on `127.0.0.1:8090`), so it needs none of this. +- `--migrate-hold-state` moves an existing `~/GPU-{TOKEN,QUEUE}`, `~/NPU-{TOKEN,QUEUE}` into `/var/lib/1bit-fleet`, + and **refuses when the queue is not empty**: moving a live queue out from under the lanes is not a step to + take blind. + +What is deliberately **not** shared is `~/.cache`, which is `0700` on the owner's account: the persistent build +trees (`~/.cache/hrx-build/bench`, `~/.cache/hw-gate`) and the `pr-agent-venv-*` environments move, so the first +runs on a fresh install are cold — `hrx-build`'s "rebuild only what changed" starts from zero once. + +What it still needs, and the honest price of each: + +| grant | why | price | +|---|---|---| +| `docker` group | the bench jobs run containers (podman/docker images for the distro proofs) | **docker group is root-equivalent** — a job that runs `docker run -v /:/h …` can read the box. It is narrower than the owner's account but not a sandbox | +| `render` (or `video`) group | `/dev/kfd` and `/dev/dri/renderD*` at `0660` for the GPU jobs | device access only | +| `fleethold` (its own group) | write the hold state in `/var/lib/1bit-fleet` | the runner can write the queue and the token files **and nothing else** — not the owner's home, not the tools it runs | + +If that trade is not wanted, the next step is rootless podman (or a docker socket proxy that only allows the +images the workflows name) and dropping the `docker` group — that is a change to the workflows, not to this +recipe, and it is not in #1504's scope. + +## A fresh install, start to finish + +```bash +# 1. the runner, unpacked and verified. Pin the version; the script checks the release's SHA256SUMS and +# refuses an unverified tarball. +curl -fsSLO https://github.com/actions/runner/releases/download/v2.329.0/actions-runner-linux-x64-2.329.0.tar.gz +mkdir -p /srv/actions-runner-2.329.0 && tar -C /srv/actions-runner-2.329.0 -xzf actions-runner-linux-x64-2.329.0.tar.gz + +# 2. a registration token: repo Settings -> Actions -> Runners -> New self-hosted runner, or +# gh api -X POST repos/1bit-MONSTER/engine/actions/runners/registration-token --jq .token > /root/runner-token +# It is a one-hour secret: write it to a root-only file and delete it after the install. + +# 3. install both runners +sudo scripts/install-ci-runner.sh --name strixhalo-engine --labels pr-agent \ + --token-file /root/runner-token --runner-dir /srv/actions-runner-2.329.0 +sudo scripts/install-ci-runner.sh --name strixhalo-bench --labels bench \ + --token-file /root/runner-token --runner-dir /srv/actions-runner-2.329.0 +sudo rm -f /root/runner-token + +# 4. the owner's own lanes must name the same hold state (the installer writes this file); a long-running +# owner service needs the two variables in its own unit instead +. /etc/profile.d/1bit-fleet.sh +``` + +**Order matters, and the installer enforces it.** The default that keeps the queue un-forked lives in +`scripts/gpu-hold`; what *runs* on the box is `~/.local/bin/gpu-hold`, which every lane and the runner reach +through their own home. So the installer puts the repository's copy in place (keeping a dated `.bak-*`, as this +box does for every other tool) before anything else, and `--migrate-hold-state` **refuses** to run against a copy +that still defaults to `~/GPU-TOKEN` — otherwise the migration moves the token and the first lane without +`GPU_TOKEN_FILE` recreates the old path, which is two queues again. + +On a box that already has hold state in the owner's home, the first install takes +`--migrate-hold-state --owner-user `, with the queue drained — the installer refuses a non-empty queue. + +**NPU is not shared yet, and the installer says so at install time.** `npu-hold` is not in this repository, and it +pins the queue with `exec env GPU_QUEUE_DIR="$HOME/NPU-QUEUE" GPU_TOKEN_FILE="$HOME/NPU-TOKEN" …` +(`~/.local/bin/npu-hold:31`), which resolves to a private NPU queue for any other account whatever `gpu-hold`'s +default is. The unit sets `HW_GATE_LOCK` to the fleet's lock, so the *lock* is shared; the *queue* is not. Until +`npu-hold` honours the two variables, a runner under another account must not take NPU work. + +`--dry-run` prints the unit and runs `systemd-analyze verify` on it without changing anything; `--verify` prints +the installed unit's `Result`/`NRestarts`/`OOMPolicy` and exits. The script is idempotent: an existing `ghrunner` +user, an already-configured runner directory and an existing unit are reused or rewritten, not re-created. + +## Verifying an install + +```bash +# the unit is up and has not been restarting +systemctl show actions.runner.1bit-MONSTER-engine.strixhalo-engine.service \ + -p ActiveState -p Result -p NRestarts -p OOMPolicy -p MemoryMax +# ActiveState=active Result=success NRestarts=0 OOMPolicy=continue MemoryMax=51539607552 + +# GitHub agrees it is online and idle +gh api repos/1bit-MONSTER/engine/actions/runners --jq '.runners[] | "\(.name) \(.status) busy=\(.busy)"' +# strixhalo-bench online busy=false +# strixhalo-engine online busy=false +``` + +And the OOM behaviour itself, which is what #1504 is about: with `OOMPolicy=continue`, a process killed inside the +unit's cgroup does not stop the unit (`systemd.service(5)`: "the service manager will not take any action"), so +the job fails and the runner keeps listening; with the default `stop` it ends up exactly as it did on 11-08 — +`Result=oom-kill`, `NRestarts=0`. + +```bash +# the state to read after a heavy job has been killed, on a freshly installed runner: +systemctl show actions.runner.1bit-MONSTER-engine.strixhalo-bench.service -p Result -p NRestarts +# Result=success <- the job died, the runner did not +# NRestarts=0 <- nothing had to restart it +``` + +**Not yet demonstrated end to end here.** A throwaway `systemd-run --user` pair with and without +`OOMPolicy=continue` (128M cap, a child allocating 256M) both came back `Result=success` on this box — the child's +allocation failed as an exception rather than as an OOM kill, so that experiment says nothing either way. The +directive is the documented mechanism and it is what the fresh install's single end-to-end run should confirm, +together with the rest of #1504's acceptance. + +## What a restart policy cannot do + +It cannot notice that a runner is *not* coming back (a version mismatch, a revoked token, a full disk under +`/var/lib/ghrunner`). That is [engine #1504](https://github.com/1bit-MONSTER/engine/issues/1504)'s item 3 and +still open: a cheap row that asserts both self-hosted runners are `online=busy=false` through the API the release +lanes already use, so a runner that is gone turns a check red instead of leaving it pending for ever. diff --git a/scripts/gpu-hold b/scripts/gpu-hold index b029b12f3..d75b682bf 100755 --- a/scripts/gpu-hold +++ b/scripts/gpu-hold @@ -35,8 +35,14 @@ # -t MINUTES max hold, default $GPU_HOLD_MINUTES or 14 (0.05 = 3 seconds) # -l LABEL label written into the token, default "-$$" # -w SECONDS how long to queue before giving up, default $GPU_HOLD_WAIT or 5400; exit 75 on timeout -# -q DIR queue directory, default $GPU_QUEUE_DIR or ~/GPU-QUEUE -# GPU_TOKEN_FILE the token file, default ~/GPU-TOKEN +# -q DIR queue directory, default $GPU_QUEUE_DIR, else the fleet's queue when it exists +# ($ONEBIT_HOLD_DIR/gpu-queue, /var/lib/1bit-fleet by default), else ~/GPU-QUEUE +# GPU_TOKEN_FILE the token file, same order of preference, else ~/GPU-TOKEN +# ONEBIT_HOLD_DIR where the fleet's hold state lives (engine #1504); when that directory exists it IS +# the queue for every caller, whoever runs: a runner under another account, a tmux lane, a +# systemd --user service. Without it a second account silently takes its own token and runs +# GPU work beside the fleet, and with it a lane that never sourced a profile file still +# agrees, because the default - not an environment variable - is what moved. # GPU_HOLD_RELEASE_WAIT seconds a label must wait after releasing, default 180 # GPU_HOLD_TCTL_START start gate in C, default 75 - a hot box is not taken # GPU_HOLD_TCTL_ABORT abort in C, default 93 @@ -54,8 +60,10 @@ set -u MINUTES=${GPU_HOLD_MINUTES:-14} LABEL="" WAIT=${GPU_HOLD_WAIT:-5400} -QUEUE=${GPU_QUEUE_DIR:-$HOME/GPU-QUEUE} -TOKEN=${GPU_TOKEN_FILE:-$HOME/GPU-TOKEN} +HOLD=${ONEBIT_HOLD_DIR:-/var/lib/1bit-fleet} +# the fleet's state wins when it is installed, and ~ is the fallback for a box that has not been set up that way +QUEUE=${GPU_QUEUE_DIR:-$([ -d "$HOLD/gpu-queue" ] && printf '%s' "$HOLD/gpu-queue" || printf '%s' "$HOME/GPU-QUEUE")} +TOKEN=${GPU_TOKEN_FILE:-$([ -d "$HOLD" ] && printf '%s' "$HOLD/gpu-token" || printf '%s' "$HOME/GPU-TOKEN")} RELEASE_WAIT=${GPU_HOLD_RELEASE_WAIT:-180} TCTL_START=${GPU_HOLD_TCTL_START:-75} TCTL_ABORT=${GPU_HOLD_TCTL_ABORT:-93} diff --git a/scripts/hw-gate.sh b/scripts/hw-gate.sh index 3a8c043c9..62f5c7f2d 100755 --- a/scripts/hw-gate.sh +++ b/scripts/hw-gate.sh @@ -96,7 +96,10 @@ fi ROOT=$(cd "$(dirname "$0")/.." && pwd) STORE=${HW_GATE_STORE:-$HOME/.cache/hw-gate} MODELS=${HW_GATE_MODELS:-$HOME/models} -LOCK=${HW_GATE_LOCK:-$HOME/npu.lock} +# engine #1504: the NPU lock is fleet state, like the GPU token - when the shared hold directory exists it is +# THE lock, so a process running under another account cannot take NPU work on a lock of its own +_HOLD=${ONEBIT_HOLD_DIR:-/var/lib/1bit-fleet} +LOCK=${HW_GATE_LOCK:-$([ -d "$_HOLD" ] && printf '%s' "$_HOLD/npu.lock" || printf '%s' "$HOME/npu.lock")} LOCK_WAIT=${HW_GATE_LOCK_WAIT:-5400} # The NPU array's fair claim, the one the rest of the box uses: npu-hold puts the command on ~/NPU-QUEUE and # ~/NPU-TOKEN, and npu-serve-keeper stops the owner's serve while a ticket waits. ~/npu.lock is a *private* diff --git a/scripts/install-ci-runner.sh b/scripts/install-ci-runner.sh new file mode 100755 index 000000000..d50770ef9 --- /dev/null +++ b/scripts/install-ci-runner.sh @@ -0,0 +1,350 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# install-ci-runner.sh: install one self-hosted GitHub Actions runner as a systemd service, owned by a +# dedicated unprivileged user, with the restart and OOM policy that engine #1504 asks for. +# +# sudo scripts/install-ci-runner.sh --name strixhalo-engine --labels pr-agent \ +# --token-file /root/runner-token --runner-dir /srv/actions-runner-2.329.0 +# +# It runs on the box the jobs run on (Strix Halo), not from CI. It creates the user, the runner directory, +# the unit, and starts it; it is idempotent (an existing user or runner config is reused) and it never +# prints the registration token. +# +# Why each policy is here, in one line each - the detail is in docs/ci-runners.md: +# Restart=always runsvc.sh exits 0 after an OOM kill ("no retry needed"), so on-failure never fires +# OOMPolicy=continue an OOM kill inside the cgroup kills the JOB; the runner keeps running +# MemoryHigh/Max the cgroup hits its own ceiling first, so the box's OOM killer is not choosing +# between "the biggest allocation on the machine" and everything else +# User=ghrunner the jobs stop running as the owner: no ssh keys, no ~/models, no fleet remotes +# StartLimitIntervalSec=0 a runner must never be rate-limited out of existence; RestartSec bounds the loop +# +# Options: +# --name NAME runner name and unit suffix (default: strixhalo-engine) +# --labels LIST extra labels beyond self-hosted,Linux,X64 (default: pr-agent) +# --token-file FILE file holding a registration token (repo Settings -> Actions -> Runners -> New) +# --runner-dir DIR an already-unpacked runner (config.sh/runsvc.sh in it). Downloaded if omitted, +# with --runner-version, and verified against the release's SHA256SUMS +# --runner-version VER e.g. 2.329.0 (only used when downloading) +# --user USER dedicated user (default: ghrunner) +# --groups LIST extra groups for that user (default: docker; use "" for none) +# --memory-max SIZE MemoryMax for the unit (default: 48G); MemoryHigh is 90% of it +# --hold-state DIR where the fleet's GPU/NPU token and queue files live (default /var/lib/1bit-fleet). +# The installer creates it group-owned by --hold-group, mode 0770 with the setgid bit so +# tickets inherit the group, and puts the runner's account in that group - NOT in the +# owner's group. The unit is given GPU_TOKEN_FILE/GPU_QUEUE_DIR so the job queues with +# the fleet instead of beside it, and the owner's own environment must name the same +# two variables (docs/ci-runners.md, "One queue, two accounts"). A file inside the +# owner's home cannot be shared this way: gpu-hold RELEASES with `rm -f "$TOKEN"`, +# which needs write on the DIRECTORY, i.e. on the owner's home itself. +# --hold-group GROUP the group that owns the hold state (default fleethold) +# --no-hold-state leave the runner without hold state, for a runner that runs no hardware jobs (the +# pr-agent runner does not: it talks to the local LLM over HTTP). A runner without it +# must not run hw-gate/hrx-build, which is why the default is to set it up +# --migrate-hold-state move an existing ~/GPU-TOKEN, ~/GPU-QUEUE, ~/NPU-TOKEN, ~/NPU-QUEUE of +# --owner-user into --hold-state. Requires a quiet queue: it is the one step that +# changes the paths under every running lane +# --env-file FILE optional EnvironmentFile= for the unit +# --dry-run print the unit and verify it, change nothing +# --verify only report the installed unit's state +set -euo pipefail + +name=strixhalo-engine labels=pr-agent user=ghrunner groups=docker memory_max=48G +token_file='' runner_dir='' runner_version='' env_file='' hold_state=/var/lib/1bit-fleet hold_group=fleethold owner_user=bcloud no_hold=0 migrate=0 dry=0 verify=0 +usage() { sed -n '16,48p' "$0"; } + +while [ $# -gt 0 ]; do + case $1 in + --name) name=$2; shift 2 ;; + --labels) labels=$2; shift 2 ;; + --token-file) token_file=$2; shift 2 ;; + --runner-dir) runner_dir=$2; shift 2 ;; + --runner-version) runner_version=$2; shift 2 ;; + --user) user=$2; shift 2 ;; + --groups) groups=$2; shift 2 ;; + --memory-max) memory_max=$2; shift 2 ;; + --env-file) env_file=$2; shift 2 ;; + --hold-state) hold_state=$2; shift 2 ;; + --hold-group) hold_group=$2; shift 2 ;; + --owner-user) owner_user=$2; shift 2 ;; + --no-hold-state) no_hold=1; shift ;; + --migrate-hold-state) migrate=1; shift ;; + --dry-run) dry=1; shift ;; + --verify) verify=1; shift ;; + -h|--help) usage; exit 0 ;; + *) echo "install-ci-runner.sh: unknown option $1" >&2; usage >&2; exit 2 ;; + esac +done + +unit=actions.runner.1bit-MONSTER-engine.$name.service +unit_path=/etc/systemd/system/$unit +dir=/var/lib/$user/actions-runner-$name +# MemoryHigh is 90% of MemoryMax IN ITS OWN UNIT: awk's numeric coercion reads "512M" as 512 and "2T" as 2, +# so the old one-liner turned 512M into 460G and 2T into 1G. A size with no K/M/G/T suffix is passed through +# unconverted, which makes MemoryHigh equal to MemoryMax: a hard limit with no throttle band above it, which is a +# fine choice for a runner (the job dies at the ceiling) and is what the code does. +share_home=$(getent passwd "$owner_user" 2>/dev/null | cut -d: -f6) || share_home='' +memory_high=$(awk -v m="$memory_max" 'BEGIN{ + if (m !~ /^[0-9.]+[KMGT]$/) { exit } # no suffix: leave MemoryHigh alone + v = substr(m, 1, length(m) - 1) + 0; u = substr(m, length(m)) + printf "%.1f%s", v * 0.9, u +}') +[ -n "$memory_high" ] || memory_high=$memory_max + +# <-- the unit this recipe installs +unit_text() { + cat </dev/null +} + +if [ "$verify" = 1 ]; then + state + exit 0 +fi + +if [ "$dry" = 1 ]; then + unit_text + # verify the DIRECTIVES, not the install: ExecStart does not exist yet, and systemd-analyze treats a missing + # binary as an error. The copy it verifies points at /bin/true, so an unknown or malformed setting still + # fails here while the not-yet-installed path does not. + tmp=$(mktemp --suffix=.service) + unit_text | sed "s|^ExecStart=.*|ExecStart=/bin/true|" > "$tmp" + if systemd-analyze verify "$tmp"; then echo "dry run: every directive verifies; nothing was changed" + else echo "dry run: the unit does not verify" >&2; rm -f "$tmp"; exit 1; fi + rm -f "$tmp" + exit 0 +fi + +[ "$(id -u)" = 0 ] || { echo "install-ci-runner.sh: run as root (it creates a user and a unit)" >&2; exit 2; } +[ -n "$token_file" ] && [ -s "$token_file" ] || { echo "install-ci-runner.sh: --token-file is required and must not be empty" >&2; exit 2; } + +# the dedicated user (engine #1504's audit finding): the jobs must not run as the owner +if ! id "$user" >/dev/null 2>&1; then + useradd --system --create-home --home-dir "/var/lib/$user" --shell /usr/sbin/nologin "$user" + echo "created user $user (no shell, home /var/lib/$user)" +fi +for g in ${groups//,/ }; do + getent group "$g" >/dev/null 2>&1 || { echo "warning: no group '$g' on this box; $user is not in it (jobs that need it will fail loudly)" >&2; continue; } + id -nG "$user" | tr ' ' '\n' | grep -qx "$g" || usermod -aG "$g" "$user" +done + +# the runner itself +mkdir -p "/var/lib/$user" && chown "$user:" "/var/lib/$user" +if [ ! -x "$dir/config.sh" ]; then + if [ -n "$runner_dir" ]; then + [ -x "$runner_dir/config.sh" ] || { echo "install-ci-runner.sh: $runner_dir is not an unpacked runner" >&2; exit 2; } + mkdir -p "$dir" && cp -a "$runner_dir/." "$dir/" + else + [ -n "$runner_version" ] || { echo "install-ci-runner.sh: --runner-dir or --runner-version is required" >&2; exit 2; } + tarball=actions-runner-linux-x64-$runner_version.tar.gz + tmpd=$(mktemp -d) + curl -fsSL -o "$tmpd/$tarball" \ + "https://github.com/actions/runner/releases/download/v$runner_version/$tarball" + curl -fsSL -o "$tmpd/SHA256SUMS" \ + "https://github.com/actions/runner/releases/download/v$runner_version/SHA256SUMS" || true + if [ -s "$tmpd/SHA256SUMS" ]; then + ( cd "$tmpd" && grep " $tarball\$" SHA256SUMS | sha256sum -c - ) || { + echo "install-ci-runner.sh: $tarball does not match the release's SHA256SUMS" >&2; exit 1; } + else + echo "install-ci-runner.sh: no SHA256SUMS on that release; refusing to install an unverified runner" >&2 + exit 1 + fi + mkdir -p "$dir" && tar -C "$dir" -xzf "$tmpd/$tarball" + rm -rf "$tmpd" + fi + chown -R "$user:" "$dir" +fi + +# register, as the runner's own user, with the token read from the file and never echoed +if [ ! -f "$dir/.runner" ]; then + tok=$(mktemp) + cp "$token_file" "$tok"; chown "$user:" "$tok"; chmod 0600 "$tok" + # config.sh takes the token as an argument and there is no stdin form, so it is in the child's argv for the + # length of the call; the file is 0600 and root-owned, this copy is 0600 and the runner user's, and both the + # copy and the temp dir go away here. The token itself is a one-hour registration token, never a PAT. + sudo -u "$user" bash -c "cd '$dir' && ./config.sh --unattended --replace \ + --url https://github.com/1bit-MONSTER/engine \ + --token \"\$(cat '$tok')\" \ + --name '$name' --labels '$labels' --work '_work'" > /dev/null + rm -f "$tok" + echo "registered runner $name with labels self-hosted,Linux,X64,$labels" +fi +chown -R "$user:" "$dir" + +# --- the fleet's hold state, in a directory the fleet group owns ----------------------------------------- +# The workflows reach the hold through $HOME (~/GPU-TOKEN, ~/GPU-QUEUE, and npu-hold's hard-coded +# ~/NPU-TOKEN, ~/NPU-QUEUE), and gpu-hold RELEASES a token with `rm -f "$TOKEN"` - a write to the DIRECTORY, +# not to the file. So a hold file that stays inside the owner's home cannot be shared with another account by +# chgrp'ing the file: either the runner is in the owner's group, which is group-write on the owner's whole home +# (/home/bcloud is 0775 here, so a job could replace ~/.bashrc or move ~/.ssh aside - an owner-level escalation +# created by a recipe whose point is that jobs no longer run as the owner), or the file is not shareable at all. +# +# So the hold state moves to a directory the FLEET group owns, and both sides name it: +# /{gpu-token,gpu-queue,npu-token,npu-queue} group-owned, 0770, queues setgid so tickets +# created by either account inherit the group +# The runner's account joins THAT group and nothing else; the shared tools (~/lb, ~/.local/bin) are already +# world-readable and executable, so no group is needed to run them. +# the workflows reach the shared tools by a HOME-RELATIVE path (~/lb/thermal-run.sh, and `command -v gpu-hold` +# for ~/.local/bin), so the runner's home needs those two entries. They are symlinks into the account that owns +# them and the runner reaches them through "other" (r-x), NOT through a group: it can run the tools and cannot +# rewrite them - which is the escalation the first version of this recipe created. +if [ -n "$share_home" ]; then + for rel in lb .local/bin; do + src="$share_home/$rel"; dst="/var/lib/$user/$rel" + [ -d "$src" ] || continue + mkdir -p "$(dirname "$dst")" + if [ -e "$dst" ] && [ ! -L "$dst" ]; then + echo "warning: /var/lib/$user/$rel exists as a real path; leaving it (move it aside to share $src)" >&2 + continue + fi + ln -sfn "$src" "$dst" + echo "tools: ~/$rel -> $src (read-only through \"other\", no group)" + done +fi + +if [ "$no_hold" = 0 ]; then + getent group "$hold_group" >/dev/null 2>&1 || groupadd --system "$hold_group" + id -nG "$user" | tr ' ' '\n' | grep -qx "$hold_group" || usermod -aG "$hold_group" "$user" + install -d -m 0770 -g "$hold_group" "$hold_state" "$hold_state/gpu-queue" "$hold_state/npu-queue" + chmod g+s "$hold_state/gpu-queue" "$hold_state/npu-queue" + # NOT the token files: `gpu-hold` treats the token's EXISTENCE as "held" (the holder creates it with + # noclobber), so pre-creating an empty one makes every claim wait for ever - found by tests/gpu_hold_queue_test.sh. + # The directory is group-writable so any fleet member can create the token; an existing one is handed over. + for f in gpu-token npu-token; do + [ -e "$hold_state/$f" ] || continue + chgrp "$hold_group" "$hold_state/$f"; chmod 0660 "$hold_state/$f" + done + # The default that keeps the queue un-forked lives in scripts/gpu-hold, and what RUNS is the copy on the + # box: ~/.local/bin/gpu-hold, which every lane and this runner reach through its own home. If that copy is + # older it still defaults to ~/GPU-TOKEN, so a migration would move the token and the first lane without + # GPU_TOKEN_FILE would recreate it - two queues again. So the installer puts the repository's copy in place + # (keeping a dated backup, the way this box keeps every other tool), and the migration refuses to run + # against a copy that cannot resolve the fleet default. + tool_dir=$share_home/.local/bin + if [ -d "$tool_dir" ] && [ -f "$(dirname "$0")/gpu-hold" ] && ! cmp -s "$(dirname "$0")/gpu-hold" "$tool_dir/gpu-hold"; then + # owned by the account whose tool it is, not by root: this script runs as root, and a root-owned file in + # ~/.local/bin can be replaced (the directory is theirs) but not edited in place by its owner + owner_group=$(id -gn "$owner_user") + install -m 0755 -o "$owner_user" -g "$owner_group" "$(dirname "$0")/gpu-hold" "$tool_dir/gpu-hold.new" && + { [ -f "$tool_dir/gpu-hold" ] && cp -a "$tool_dir/gpu-hold" "$tool_dir/gpu-hold.bak-$(date +%Y%m%d-%H%M%S)" && + chown "$owner_user:$owner_group" "$tool_dir"/gpu-hold.bak-*; } + mv -f "$tool_dir/gpu-hold.new" "$tool_dir/gpu-hold" + echo "installed scripts/gpu-hold over $tool_dir/gpu-hold (previous copy kept as a .bak-*), so every lane's default is the fleet's" + fi + if [ "$migrate" = 1 ]; then + # never migrate against a gpu-hold that would still default to ~/GPU-TOKEN + if [ -f "$tool_dir/gpu-hold" ] && ! grep -q 'ONEBIT_HOLD_DIR\|1bit-fleet' "$tool_dir/gpu-hold"; then + echo "install-ci-runner.sh: refusing --migrate-hold-state: $tool_dir/gpu-hold does not resolve the fleet default" >&2 + echo " (it still defaults to \$HOME/GPU-TOKEN, so the first lane without GPU_TOKEN_FILE would recreate the old path)" >&2 + exit 1 + fi + owner_home=$(getent passwd "$owner_user" | cut -d: -f6) + [ -n "$owner_home" ] || { echo "install-ci-runner.sh: --migrate-hold-state needs --owner-user that exists" >&2; exit 2; } + # a live queue is not a thing to move under the lanes: refuse unless it is empty + [ -z "$(ls -A "$owner_home/GPU-QUEUE" 2>/dev/null)" ] || { + echo "install-ci-runner.sh: $owner_home/GPU-QUEUE is not empty: let the queue drain before migrating (or migrate by hand)" >&2; exit 1; } + for pair in "GPU-TOKEN:gpu-token" "NPU-TOKEN:npu-token"; do + src="$owner_home/${pair%%:*}"; dst="$hold_state/${pair##*:}" + if [ -s "$src" ]; then cp -a "$src" "$dst" && chgrp "$hold_group" "$dst" && chmod 0660 "$dst"; fi + rm -f "$src" # empty or moved: either way no file stays behind to read as a held token + done + for pair in "GPU-QUEUE:gpu-queue" "NPU-QUEUE:npu-queue"; do + src="$owner_home/${pair%%:*}"; dst="$hold_state/${pair##*:}" + [ -d "$src" ] && { cp -a "$src/." "$dst/" 2>/dev/null || true; rm -rf "$src"; } + done + chgrp -R "$hold_group" "$hold_state"; chmod -R g+rwX,o-rwx "$hold_state" + echo "migrated the hold state from $owner_home into $hold_state" + fi + # the owner's side has to name the same two paths, or the runner queues with the fleet's files and the + # owner's lanes with their own - which is the bug this whole block exists to prevent + cat > /etc/profile.d/1bit-fleet.sh </dev/null || echo "warning: could not enable-linger $user; thermal-run.sh will use its process-group fallback" + if [ -f "$share_home/.local/bin/npu-hold" ] && grep -q 'GPU_TOKEN_FILE="\$HOME/NPU-TOKEN"' "$share_home/.local/bin/npu-hold" 2>/dev/null; then + echo "warning: npu-hold pins ~/NPU-QUEUE and ~/NPU-TOKEN, so NPU work from $user uses a PRIVATE NPU queue." >&2 + echo " The unit sets HW_GATE_LOCK to the fleet's lock, so the lock is shared; the queue is not until npu-hold" >&2 + echo " honours GPU_QUEUE_DIR/GPU_TOKEN_FILE (docs/ci-runners.md). Until then do not let this runner take NPU work." >&2 + fi + echo "hold state: $hold_state (group $hold_group, 0770, queues setgid); $user joined $hold_group" + echo "the owner's shells pick up GPU_TOKEN_FILE/GPU_QUEUE_DIR from /etc/profile.d/1bit-fleet.sh;" + echo "a long-running owner service needs them in its own unit or environment (docs/ci-runners.md)" +fi + +unit_text > "$unit_path" +chmod 0644 "$unit_path" +systemd-analyze verify "$unit_path" +systemctl daemon-reload +systemctl enable --now "$unit" +sleep 5 +echo +echo "installed $unit as user $user, MemoryMax=$memory_max, OOMPolicy=continue, Restart=always" +state +cat < a claim with NO environment at all lands in the fleet queue and uses the +# fleet token file; +# 2. the fleet state is absent -> the old ~/GPU-* paths are used, so a box that has not been set up is +# unchanged (that is what every caller did before). +set -uo pipefail +here=$(cd "$(dirname "$0")" && pwd) +hold=$(mktemp -d); fake_home=$(mktemp -d) +trap 'rm -rf "$hold" "$fake_home"' EXIT +fail=0 + +claim() { # claim: run one very short claim with the given HOME and hold dir, nothing else in the environment + # the output is kept: gpu-hold logs "took ", which is the only evidence of WHICH token a claim used + # after it released it (release is `rm -f "$TOKEN"`, by design) + HOME=$fake_home ONEBIT_HOLD_DIR=$hold \ + GPU_HOLD_TCTL_START=${GPU_HOLD_TCTL_START:-999} GPU_HOLD_RELEASE_WAIT=0 \ + bash "$here/../scripts/gpu-hold" -l gpu-hold-queue-test -t 0.05 -w 10 true 2>&1 +} + +# 1. the fleet's state is installed: the queue and the token are the fleet's, with no environment to say so +mkdir -p "$hold/gpu-queue" # and NOT the token: its existence means "held", so the holder must create it +if out=$(claim); then + if printf '%s' "$out" | grep -q "took $hold/gpu-token" && + [ -e "$hold/gpu-queue/.released-at.gpu-hold-queue-test" ] && [ ! -d "$fake_home/GPU-QUEUE" ]; then + echo "PASS a claim with no environment queues on the fleet's state and takes the fleet token" + else + echo "FAIL the claim did not use the fleet's state (fleet queue: $(ls -A "$hold/gpu-queue" 2>/dev/null | tr '\n' ' '); ~/GPU-QUEUE: $([ -d "$fake_home/GPU-QUEUE" ] && echo present || echo absent)); gpu-hold said:" + printf '%s\n' "$out" | tail -3 | sed 's/^/ /' + fail=$((fail + 1)) + fi +else + echo "FAIL the claim itself failed (rc $?); see the gpu-hold output above" + fail=$((fail + 1)) +fi + +# 2. nothing installed: the old paths, so this change is inert on a box without the shared state +hold2=$(mktemp -d); rm -rf "$hold2"; mkdir -p "$fake_home" +if HOME=$fake_home ONEBIT_HOLD_DIR=$hold2 GPU_HOLD_TCTL_START=${GPU_HOLD_TCTL_START:-999} GPU_HOLD_RELEASE_WAIT=0 \ + bash "$here/../scripts/gpu-hold" -l gpu-hold-queue-test2 -t 0.05 -w 10 true >/dev/null 2>&1; then + if [ -e "$fake_home/GPU-QUEUE/.released-at.gpu-hold-queue-test2" ]; then + echo "PASS with no fleet state the claim falls back to ~/GPU-QUEUE, as before" + else + echo "FAIL with no fleet state the claim did not use ~/GPU-QUEUE"; fail=$((fail + 1)) + fi +else + echo "FAIL the fallback claim failed (rc $?)"; fail=$((fail + 1)) +fi +r=$([ "$fail" = 0 ] && echo ok || echo no); rm -rf "$hold2" +[ "$r" = ok ] && echo "gpu_hold_queue: all rows PASS" +[ "$fail" = 0 ]