From 8b778035f7f56a9e00f6df938ef6355a6d149da7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krystian=20Paw=C5=82ow?= Date: Thu, 12 Nov 2026 20:42:36 +0100 Subject: [PATCH 1/6] Release only green commits, and let a release-installed Runner update itself A Runner installed from a release (no build_folder) could not update at all, and the release workflow put out commits whose CI was red. - "Runner build" builds the Linux programs on every push to main. "Runner release" runs on CI, Image and Runner build ending and publishes a commit only once all of its push's runs are green on main (scripts/release-gate.sh), the rule the console rolls out by. - ReleaseUpdate: a Runner with no build_folder lists only update-to, finds that commit's release (by github-actions[bot], at the whole commit, on main and newer than the running one), fetches SHA256SUMS and its own architecture's program, checks the checksum and that the program runs here and names the commit, then drains, installs and guards the start as a build update does. It waits up to 16 minutes for a release not out yet. - cf-ghapp Releases: tokenless reads, compare, and a streamed, hashed download that restarts after a break and stops on a full disk. - host-setup.sh --version is a minimum: an enrolled Runner on a newer commit of main stays, and stays when GitHub cannot say. --- .github/workflows/release.yml | 158 ++-- .github/workflows/runner-build.yml | 81 ++ Cargo.lock | 2 + README.md | 19 +- crates/console-client/src/error.rs | 2 +- crates/daemon/src/service.rs | 8 +- crates/daemon/src/testing.rs | 10 + crates/daemon/src/update.rs | 14 + crates/daemon/tests/service.rs | 69 ++ crates/ghapp/Cargo.toml | 1 + crates/ghapp/src/error.rs | 5 +- crates/ghapp/src/http.rs | 2 +- crates/ghapp/src/lib.rs | 4 + crates/ghapp/src/release.rs | 372 +++++++++ crates/ghapp/tests/release.rs | 417 ++++++++++ crates/runner/Cargo.toml | 1 + crates/runner/src/main.rs | 2 + crates/runner/src/release_update.rs | 1190 +++++++++++++++++++++++++++ crates/runner/src/service.rs | 48 +- crates/runner/src/update.rs | 160 ++-- docs/MAP.md | 10 +- scripts/host-setup.sh | 74 +- scripts/release-gate.sh | 96 +++ xtask/tests/host_setup.rs | 396 +++++++++ 24 files changed, 2978 insertions(+), 163 deletions(-) create mode 100644 .github/workflows/runner-build.yml create mode 100644 crates/ghapp/src/release.rs create mode 100644 crates/ghapp/tests/release.rs create mode 100644 crates/runner/src/release_update.rs create mode 100755 scripts/release-gate.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8d8dcdc..1d5530f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,114 +1,122 @@ name: Runner release -# The Runner's Linux program for each architecture a host runs: one file that holds all of it (the guest bundle and -# cloud-hypervisor are built in), released as `runner-` (`0.1.0-446125c0a1b2` for the version -# `0.1.0+446125c0a1b2` the program says), with `codefusion-runner-linux-amd64`, `-arm64`, the `host-setup.sh` of the -# same commit and their SHA256SUMS. A host takes them with `host-setup.sh install --version --sha256 `, the script checked against its own line first, so it needs no Rust, no checkout and builds nothing. Built in Debian 12, the oldest system a host runs, so the program runs on its glibc and on Ubuntu 24.04's. +# Puts out the Runner release of a commit of main once the commit is green by the rule the console rolls Runners out +# by: its push's CI green, and Image green too when the push made one, the newest run of each counting. Its programs +# are those the push's "Runner build" (runner-build.yml) made, released as `runner-` (`0.1.0-446125c0a1b2` +# for the version `0.1.0+446125c0a1b2` the program says), with `codefusion-runner-linux-amd64`, `-arm64`, the +# `host-setup.sh` of the same commit and their SHA256SUMS. A host takes them with `host-setup.sh install --version +# --sha256 `, and a Runner installed so updates itself to the release of a commit the console names +# (`update-to`), which it refuses when there is none: so no red commit is ever put out, and each green one is. # -# `build` holds no secret, so a pull request that changes this workflow runs it too. `publish` runs only on main, -# needs nothing but `gh`, and never replaces a release: a host's settings name its SHA-256, so the bytes of a version -# never change. Each run builds from nothing, without a cache another run could have written. +# Each of the three workflows ending starts the check (scripts/release-gate.sh); the one that ends last finds them all +# green and publishes. A red run, a commit main does not have, or one whose runs have not ended gets no release, and a +# notice says why. A re-run that ends green starts it again. By hand, `workflow_dispatch` with a commit does the same +# check. A pull request that changes this workflow or the gate runs the check for its base commit, and publishes +# nothing. +# +# `gate` checks out main's own workflow scripts, never the pushed commit's, and holds no write permission. `publish` +# runs no code of the repository, needs nothing but `gh`, and never replaces a release: a host's settings name its +# SHA-256, so the bytes of a version never change. on: - workflow_dispatch: - push: + workflow_run: + workflows: [CI, Image, Runner build] + types: [completed] branches: [main] - paths: - - crates/** - - xtask/** - - Cargo.toml - - Cargo.lock - - rust-toolchain.toml - - scripts/host-setup.sh - - .github/workflows/release.yml + workflow_dispatch: + inputs: + commit: + description: The whole commit of main to release, when its runs are green + required: true + type: string pull_request: paths: - .github/workflows/release.yml + - scripts/release-gate.sh + - scripts/host-setup.sh permissions: {} concurrency: - group: release-${{ github.event_name == 'pull_request' && github.ref || github.sha }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + group: release-${{ github.event.workflow_run.head_sha || inputs.commit || github.ref }} + cancel-in-progress: false jobs: - build: - name: Build the ${{ matrix.platform }} Runner on Debian 12 - strategy: - fail-fast: false - matrix: - include: - - platform: arm64 - runner: ubuntu-24.04-arm - - platform: amd64 - runner: ubuntu-24.04 - runs-on: ${{ matrix.runner }} - container: debian:12@sha256:268fb24c39bca92d326dd83ea3abebe4a3f7751c804ffef23bb6d857dd343ff8 - timeout-minutes: 60 + gate: + name: Is the commit green on main + # Only a push of this repository's main: a fork's pull request from a branch it named main starts no release. + if: >- + github.event_name != 'workflow_run' || (github.event.workflow_run.event == 'push' && + github.event.workflow_run.head_repository.full_name == github.repository) + runs-on: ubuntu-latest + timeout-minutes: 30 permissions: + actions: read contents: read + outputs: + commit: ${{ steps.gate.outputs.commit }} + build: ${{ steps.gate.outputs.build }} steps: - - name: Tools - run: | - apt-get update -q - apt-get install -y -q --no-install-recommends build-essential pkg-config libssl-dev curl git ca-certificates - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 with: persist-credentials: false - - name: Rust toolchain - run: | - curl --proto '=https' --tlsv1.2 -fsSL --retry 5 https://sh.rustup.rs | - sh -s -- -y -q --profile minimal --default-toolchain none --no-modify-path - echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" - - name: Build - shell: bash + sparse-checkout: scripts + - id: gate + name: Check + env: + GITHUB_TOKEN: ${{ github.token }} + COMMIT: ${{ github.event.workflow_run.head_sha || inputs.commit || github.event.pull_request.base.sha }} run: | set -euo pipefail - # The checkout belongs to another user than the container's root; the build reads its commit. - git config --global --add safe.directory "$GITHUB_WORKSPACE" - rustup toolchain install - cargo xtask build - program=target/release/codefusion-runner - said=$("$program" version) + said=$(scripts/release-gate.sh "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY" "$COMMIT") echo "$said" - version=${said#codefusion-runner } - [[ $version =~ ^[0-9]+\.[0-9]+\.[0-9]+\+[0-9a-f]{12}$ ]] || { echo "::error::version $said"; exit 1; } - [[ ${version#*+} == "${GITHUB_SHA:0:12}" ]] || { echo "::error::$said is not of $GITHUB_SHA"; exit 1; } - mkdir out - cp "$program" "out/codefusion-runner-linux-${{ matrix.platform }}" - # One copy of the script, from one of the two builds, so the merged artifacts hold it once. - if [[ ${{ matrix.platform }} == amd64 ]]; then cp scripts/host-setup.sh out/; fi - echo "$version" > "out/version-${{ matrix.platform }}" - - if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: runner-${{ matrix.platform }} - path: out - retention-days: 1 - if-no-files-found: error + case $said in + publish\ *) + build=${said#publish } + [[ $build =~ ^[0-9]+$ ]] || { echo "::error::the gate named no run: $said"; exit 1; } + echo "::notice::$COMMIT is green on main: its release goes out with the programs of run $build" + { echo "commit=$COMMIT"; echo "build=$build"; } >> "$GITHUB_OUTPUT" + ;; + wait\ * | skip\ *) echo "::notice::$COMMIT gets no release now: ${said#* }" ;; + *) echo "::error::the gate said '$said'"; exit 1 ;; + esac publish: name: Publish the Runner release - needs: build - if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' + needs: gate + if: needs.gate.outputs.build != '' && github.event_name != 'pull_request' runs-on: ubuntu-latest timeout-minutes: 15 permissions: + actions: read contents: write + env: + GH_TOKEN: ${{ github.token }} + COMMIT: ${{ needs.gate.outputs.commit }} + BUILD: ${{ needs.gate.outputs.build }} steps: - - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - pattern: runner-* - merge-multiple: true - path: out + - name: The build's programs + run: | + set -euo pipefail + for platform in amd64 arm64; do + for try in 1 2 3 4 5; do + rm -rf "in/$platform" + if gh run download "$BUILD" --repo "$GITHUB_REPOSITORY" --name "runner-$platform" --dir "in/$platform"; then + break + fi + ((try < 5)) || { echo "::error::run $BUILD's runner-$platform could not be fetched"; exit 1; } + sleep $((try * 15)) + done + done + mkdir out + cp in/amd64/* in/arm64/* out/ - name: Publish - env: - GH_TOKEN: ${{ github.token }} run: | set -euo pipefail cd out [ "$(cat version-amd64)" = "$(cat version-arm64)" ] || { echo "::error::the two builds differ in version"; exit 1; } version=$(cat version-amd64) + [[ $version =~ ^[0-9]+\.[0-9]+\.[0-9]+\+[0-9a-f]{12}$ && ${version#*+} == "${COMMIT:0:12}" ]] || + { echo "::error::the build says $version, not of $COMMIT"; exit 1; } tag="runner-${version/+/-}" rm version-* [ -f host-setup.sh ] || { echo "::error::no host-setup.sh among the builds"; exit 1; } @@ -119,5 +127,5 @@ jobs: exit 0 fi gh release create "$tag" codefusion-runner-linux-* host-setup.sh SHA256SUMS --repo "$GITHUB_REPOSITORY" \ - --target "$GITHUB_SHA" --latest=false --title "$tag" \ + --target "$COMMIT" --latest=false --title "$tag" \ --notes "The Runner $version for Linux. A host checks host-setup.sh against its line in SHA256SUMS, then takes the Runner with \`host-setup.sh install --version ${version/+/-} --sha256 \`." diff --git a/.github/workflows/runner-build.yml b/.github/workflows/runner-build.yml new file mode 100644 index 0000000..7bbb67f --- /dev/null +++ b/.github/workflows/runner-build.yml @@ -0,0 +1,81 @@ +name: Runner build + +# The Runner's Linux program for each architecture a host runs, built for every push to main: one file that holds all +# of it (the guest bundle and cloud-hypervisor are built in), with the `host-setup.sh` of the same commit, kept as the +# artifacts runner-amd64 and runner-arm64. Built in Debian 12, the oldest system a host runs, so the program runs on its +# glibc and on Ubuntu 24.04's. Every push, whatever it changes: the console rolls out each green commit of main, and a +# Runner installed from a release updates only to a commit that has one. +# +# It publishes nothing. "Runner release" (release.yml) puts these programs out once the push's CI, and Image when the +# push made one, are green as well, the rule the console rolls Runners out by. It holds no secret, so a pull request +# that changes it runs it too. Each run builds from nothing, without a cache another run could have written. +on: + workflow_dispatch: + push: + branches: [main] + pull_request: + paths: + - .github/workflows/runner-build.yml + +permissions: {} + +concurrency: + group: runner-build-${{ github.event_name == 'pull_request' && github.ref || github.sha }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + build: + name: Build the ${{ matrix.platform }} Runner on Debian 12 + strategy: + fail-fast: false + matrix: + include: + - platform: arm64 + runner: ubuntu-24.04-arm + - platform: amd64 + runner: ubuntu-24.04 + runs-on: ${{ matrix.runner }} + container: debian:12@sha256:268fb24c39bca92d326dd83ea3abebe4a3f7751c804ffef23bb6d857dd343ff8 + timeout-minutes: 60 + permissions: + contents: read + steps: + - name: Tools + run: | + apt-get update -q + apt-get install -y -q --no-install-recommends build-essential pkg-config libssl-dev curl git ca-certificates + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + with: + persist-credentials: false + - name: Rust toolchain + run: | + curl --proto '=https' --tlsv1.2 -fsSL --retry 5 https://sh.rustup.rs | + sh -s -- -y -q --profile minimal --default-toolchain none --no-modify-path + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + - name: Build + shell: bash + run: | + set -euo pipefail + # The checkout belongs to another user than the container's root; the build reads its commit. + git config --global --add safe.directory "$GITHUB_WORKSPACE" + rustup toolchain install + cargo xtask build + program=target/release/codefusion-runner + said=$("$program" version) + echo "$said" + version=${said#codefusion-runner } + [[ $version =~ ^[0-9]+\.[0-9]+\.[0-9]+\+[0-9a-f]{12}$ ]] || { echo "::error::version $said"; exit 1; } + [[ ${version#*+} == "${GITHUB_SHA:0:12}" ]] || { echo "::error::$said is not of $GITHUB_SHA"; exit 1; } + mkdir out + cp "$program" "out/codefusion-runner-linux-${{ matrix.platform }}" + # One copy of the script, from one of the two builds, so the merged artifacts hold it once. + if [[ ${{ matrix.platform }} == amd64 ]]; then cp scripts/host-setup.sh out/; fi + echo "$version" > "out/version-${{ matrix.platform }}" + - if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: runner-${{ matrix.platform }} + path: out + # A week, so a run of CI re-run green days later still finds its programs. + retention-days: 7 + if-no-files-found: error diff --git a/Cargo.lock b/Cargo.lock index 7479c14..d3b2ef6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -233,6 +233,7 @@ dependencies = [ "ring", "serde", "serde_json", + "sha2", "tempfile", "thiserror", "time", @@ -495,6 +496,7 @@ dependencies = [ "libc", "serde", "serde_json", + "sha2", "tempfile", "tokio", "tracing", diff --git a/README.md b/README.md index 75b6e4e..bfdc862 100644 --- a/README.md +++ b/README.md @@ -174,6 +174,19 @@ install and start again. A new program that does not stay up for a minute three and the console hears why; a start that ends as asked (a signal, `restart`) does not count. `restart` ends the Runner at once and launchd (systemd on Linux) starts it again; machines that run go on and are adopted. +A Linux Runner installed from a release (`host-setup.sh --version`, no `build_folder`) builds nothing: it lists only +`update-to`, and the console names the commit. It takes that commit's release of this repository, `runner-` +(`0.1.0-<12 hex>`), from GitHub's API with no token: the release must be published by `github-actions[bot]`, made at +the whole commit, on `main` and newer than the commit that runs (GitHub's compare). It fetches `SHA256SUMS` and +`codefusion-runner-linux-amd64` or `-arm64` for its own architecture, checks the program against its line, asks it its +`version`, which must run on this host and name the commit, and then drains, installs and checks the start as a build +does. A commit whose release is not out yet is waited for up to 16 minutes, asked every 2; a commit with no release, +an older one, a checksum that does not match, a missing file, a program of another architecture or a disk with no +room leaves the running program as it was and says why. Releases are not signed: the checksum guards against a +broken download, and HTTPS to GitHub and the publisher's name against anyone else. "Runner release" puts out a commit +only once its push's CI, Image when it made one, and "Runner build" are green on main (`scripts/release-gate.sh`), the +rule the console rolls out by, so every commit the console names has a release. + A configuration written before the console held the App's key names `app_dir`, a folder with the key (`app.pem`) beside `app.json`. While the key file is there the Runner signs with it and makes its scale set calls itself, as before, and at each look at GitHub asks the console's scale set calls for its scale sets of a repository it serves. Once one @@ -226,8 +239,10 @@ console, so no App key goes on it; run again (a reboot's startup script), it onl sudo scripts/host-setup.sh --machine --console --code-file ``` -The script's head lists its options (`--home`, `--image`, `--ref`, `--check`). It reads GitHub's API for the image -with `GITHUB_TOKEN` when that is set, and waits out GitHub's rate limit as the answer says. By hand, once, as root (Ubuntu 24.04 +The script's head lists its options (`--home`, `--image`, `--ref`, `--version`, `--check`). It reads GitHub's API for +the image with `GITHUB_TOKEN` when that is set, and waits out GitHub's rate limit as the answer says. A `--version` is +a minimum: an enrolled host whose Runner updated itself to a newer commit of main keeps it, and so does one when +GitHub cannot say which is newer; the run says so, and a later run asks again. By hand, once, as root (Ubuntu 24.04 also needs the line about user namespaces): ```sh diff --git a/crates/console-client/src/error.rs b/crates/console-client/src/error.rs index f0b4435..0564861 100644 --- a/crates/console-client/src/error.rs +++ b/crates/console-client/src/error.rs @@ -194,7 +194,7 @@ impl From for Error { other => Error::Protocol(format!("{} answered {other}", s.path)), }, HttpError::Transport(why) => Error::Unavailable(why), - HttpError::Refused(why) => Error::Protocol(why), + HttpError::Refused(why) | HttpError::Local(why) => Error::Protocol(why), HttpError::Decode { what, why } => Error::Protocol(format!("{what}: {why}")), HttpError::Key(_) => Error::Protocol("a key was asked for".into()), } diff --git a/crates/daemon/src/service.rs b/crates/daemon/src/service.rs index 11b029c..6bdd758 100644 --- a/crates/daemon/src/service.rs +++ b/crates/daemon/src/service.rs @@ -2469,8 +2469,11 @@ impl Service { fn commands(&self) -> Vec { let shared = &self.shared; let mut kinds = vec!["pause", "resume", "restart", "stop-runner"]; - if shared.updates.is_some() { - kinds.extend(["update", "update-to"]); + if let Some(updates) = &shared.updates { + if updates.takes_head() { + kinds.push("update"); + } + kinds.push("update-to"); } if shared.settings.is_some() { kinds.push("settings"); @@ -2691,6 +2694,7 @@ impl Service { return Some((false, "this Runner has nowhere to update from".into())); }; let commit = match commit { + Ok(None) if !updates.takes_head() => return Some((false, crate::update::NO_HEAD.into())), Ok(commit) => commit, Err(why) => return Some((false, why)), }; diff --git a/crates/daemon/src/testing.rs b/crates/daemon/src/testing.rs index 1d39f98..c542acd 100644 --- a/crates/daemon/src/testing.rs +++ b/crates/daemon/src/testing.rs @@ -869,6 +869,8 @@ struct UpdateScript { /// What each build was asked for: `None` for the head of the branch. asked: Vec>, installed: Vec, + /// Updates only to a commit named, as a host that takes releases. + commits_only: bool, } impl FakeUpdate { @@ -884,6 +886,11 @@ impl FakeUpdate { self.0.lock().expect("update").installs.push_back(answer); } + /// Takes no update to the head of the branch from now on, as a host that takes releases. + pub fn commits_only(&self) { + self.0.lock().expect("update").commits_only = true; + } + /// Every build takes this long. pub fn build_takes(&self, time: Duration) { self.0.lock().expect("update").build_takes = time; @@ -944,4 +951,7 @@ impl crate::update::SelfUpdate for FakeUpdate { } answer } + fn takes_head(&self) -> bool { + !self.0.lock().expect("update").commits_only + } } diff --git a/crates/daemon/src/update.rs b/crates/daemon/src/update.rs index 884932d..fee4858 100644 --- a/crates/daemon/src/update.rs +++ b/crates/daemon/src/update.rs @@ -38,8 +38,22 @@ pub trait SelfUpdate: Send + Sync { /// Puts the program built last in place of the running one, keeping the running one to go back to, with /// `pending` written down first. Refused while a machine has a record. fn install(&self, pending: &Pending) -> Result<(), String>; + + /// Whether [`SelfUpdate::build`] takes the head of the branch (`update`), as a host that builds does. A host + /// that takes the Runner's releases updates only to a commit named (`update-to`), whose release it checks. + fn takes_head(&self) -> bool { + true + } + + /// Ends what a build or a download under way started, as the Runner ends: the console sends its command again + /// to the Runner that starts next. Nothing by default. + fn end(&self) {} } +/// What the console hears of a plain `update` on a host whose updates come as releases. +pub const NO_HEAD: &str = "this Runner was installed from a release: it updates to a commit the console names (update-to), never to the \ + head of main"; + /// How many characters a commit's name has: at least what git shortens one to, at most all of it. pub const COMMIT_CHARS: std::ops::RangeInclusive = 7..=40; diff --git a/crates/daemon/tests/service.rs b/crates/daemon/tests/service.rs index 1c66a12..5a10d39 100644 --- a/crates/daemon/tests/service.rs +++ b/crates/daemon/tests/service.rs @@ -3906,6 +3906,75 @@ async fn an_update_to_a_commit_builds_that_commit_and_installs_what_it_built() { .await; } +/// A Runner installed from a release offers `update-to` alone: a plain `update` is refused and fetches nothing, and +/// an `update-to` that comes while a job runs is fetched at once and put in place only once the machine is gone. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_runner_from_a_release_updates_only_to_a_commit_and_waits_for_its_machines() { + let rig = Rig::new(2).await; + rig.updates.commits_only(); + rig.service.discover().await; + let calls = rig.console.calls(); + until("a report", || rig.console.calls() > calls).await; + let commands = last_status(&rig.console)["commands"].clone(); + assert_eq!( + commands, + json!([ + "pause", + "resume", + "restart", + "stop-runner", + "update-to", + "settings", + "test-run" + ]) + ); + let plain = rig.console.queue_command("update", None); + until("the result", || !results_of(&rig.console, &plain).is_empty()).await; + let heard = &results_of(&rig.console, &plain)[0]; + assert_eq!( + (&heard["ok"], &heard["message"]), + (&json!(false), &json!(cf_daemon::update::NO_HEAD)) + ); + assert_eq!(rig.updates.built(), 0, "nothing was fetched"); + + let runner = rig.job_running().await; + rig.updates.build_gives(Ok(NEWER)); + let to = rig + .console + .queue_command("update-to", Some(json!({"commit": "5c0ffee0a1b2"}))); + rig.settles("no new job while the machine stands", || { + last_status(&rig.console)["reason"] + == json!(format!("updating to {NEWER}: waiting for 1 machine to end")) + }) + .await; + assert_eq!( + rig.updates.asked(), + [Some("5c0ffee0a1b2".to_string())], + "fetched while the job runs" + ); + assert_eq!( + rig.updates.installed(), + [], + "not put in place while the machine stands" + ); + // A second one while the first waits is refused, and fetches nothing. + let again = rig + .console + .queue_command("update-to", Some(json!({"commit": "5c0ffee0a1b2"}))); + until("the result", || !results_of(&rig.console, &again).is_empty()).await; + assert_eq!( + results_of(&rig.console, &again)[0]["message"], + json!("an update is under way already") + ); + assert_eq!(rig.updates.built(), 1); + rig.finish(&runner, "succeeded"); + rig.settles("installed", || rig.updates.installed().len() == 1) + .await; + assert_eq!(rig.updates.installed()[0].id, to); + rig.settles("the Runner is to end", || rig.service.restart_requested()) + .await; +} + /// An `update-to` whose arguments name no commit is answered with why, and nothing is built: the Mac goes on taking /// jobs. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/crates/ghapp/Cargo.toml b/crates/ghapp/Cargo.toml index 258a7b0..5c6c72a 100644 --- a/crates/ghapp/Cargo.toml +++ b/crates/ghapp/Cargo.toml @@ -15,6 +15,7 @@ reqwest = { version = "0.13", default-features = false, features = ["native-tls" ring = "0.17" serde.workspace = true serde_json.workspace = true +sha2.workspace = true rand = "0.9" thiserror.workspace = true time = { version = "0.3", features = ["formatting", "parsing", "serde-well-known"] } diff --git a/crates/ghapp/src/error.rs b/crates/ghapp/src/error.rs index aaaf950..176d652 100644 --- a/crates/ghapp/src/error.rs +++ b/crates/ghapp/src/error.rs @@ -57,6 +57,9 @@ pub enum Error { /// business naming), or an answer too long to read. #[error("refused: {0}")] Refused(String), + /// This host could not keep what it read: a file that could not be written, a disk with no room. + #[error("{0}")] + Local(String), } impl Error { @@ -104,7 +107,7 @@ impl Error { Error::Status(s) => { s.status == 429 || s.status >= 500 || (s.status == 403 && s.retry_after.is_some()) } - Error::Decode { .. } | Error::Key(_) | Error::Refused(_) => false, + Error::Decode { .. } | Error::Key(_) | Error::Refused(_) | Error::Local(_) => false, } } } diff --git a/crates/ghapp/src/http.rs b/crates/ghapp/src/http.rs index 10bd6b9..fb3781b 100644 --- a/crates/ghapp/src/http.rs +++ b/crates/ghapp/src/http.rs @@ -250,7 +250,7 @@ pub(crate) fn retry_after(headers: &HeaderMap) -> Option { } /// An exponential wait with jitter: between half and the whole of `base * 2^attempt`, at most `cap`. -fn backoff(base: Duration, cap: Duration, attempt: u32) -> Duration { +pub(crate) fn backoff(base: Duration, cap: Duration, attempt: u32) -> Duration { let full = base .saturating_mul(1u32.checked_shl(attempt.min(20)).unwrap_or(u32::MAX)) .min(cap); diff --git a/crates/ghapp/src/lib.rs b/crates/ghapp/src/lib.rs index 012ce51..e09f5ac 100644 --- a/crates/ghapp/src/lib.rs +++ b/crates/ghapp/src/lib.rs @@ -14,6 +14,7 @@ mod error; mod http; mod jwt; mod key; +mod release; mod rest; mod source; @@ -28,4 +29,7 @@ pub use http::{ Http, MAX_BODY_BYTES, Response, RetryPolicy, is_safe_address, is_safe_address_text, same_origin, }; pub use key::KeyTokens; +pub use release::{ + Asset, Author, Compared, Downloaded, RELEASES_PAGE, Release, Releases, Sink, is_whole_commit, +}; pub use source::{Monotonic, Pending, SystemMonotonic, TokenSource}; diff --git a/crates/ghapp/src/release.rs b/crates/ghapp/src/release.rs new file mode 100644 index 0000000..dc684a7 --- /dev/null +++ b/crates/ghapp/src/release.rs @@ -0,0 +1,372 @@ +//! A public repository's releases, read with no token: the Runner's own, which a host that took it from a release +//! updates from. What a release lists and where its files are come from GitHub's API; a file is fetched from the +//! address the release names, which GitHub sends on to its storage, and is hashed as it is written, never held whole. + +use std::io::Write; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use reqwest::redirect::Policy; +use reqwest::{StatusCode, Url}; +use serde::Deserialize; +use sha2::{Digest as _, Sha256}; + +use crate::error::{Error, StatusError}; +use crate::http::{Http, RetryPolicy, is_safe_address, retry_after, server_time}; + +/// How many redirects a download follows: GitHub sends a release's file on once, to its storage. +const MAX_REDIRECTS: usize = 5; + +/// How long a download may go without a byte before it counts as dropped. +const READ_TIMEOUT: Duration = Duration::from_secs(60); + +/// How many releases one page lists, GitHub's most. +pub const RELEASES_PAGE: u32 = 100; + +/// A release as GitHub's API lists it, with the fields an update reads. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +pub struct Release { + /// Its tag, such as `runner-0.1.0-446125c0a1b2`. + pub tag_name: String, + /// The commit its tag was made at, as the release names it: the whole name when the workflow gave one. + pub target_commitish: String, + /// A draft is no release yet. + #[serde(default)] + pub draft: bool, + /// Who made it: the release workflow is `github-actions[bot]`. + #[serde(default)] + pub author: Option, + /// Its files. + #[serde(default)] + pub assets: Vec, +} + +/// Who made a release. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +pub struct Author { + /// The account's login. + pub login: String, +} + +/// A file of a release. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +pub struct Asset { + /// Its name. + pub name: String, + /// Its size in bytes. + pub size: u64, + /// Where it is fetched from. + pub browser_download_url: String, +} + +impl Release { + /// The file named `name`, when the release has one. + pub fn asset(&self, name: &str) -> Option<&Asset> { + self.assets.iter().find(|asset| asset.name == name) + } +} + +/// How two commits stand in a branch's history, as GitHub's compare says from `base` to `head`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Compared { + /// `head` has `base` in its history, and more. + Ahead, + /// `base` has `head` in its history, and more. + Behind, + /// The same commit. + Identical, + /// Each has commits the other does not. + Diverged, +} + +/// What a download wrote. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Downloaded { + /// How many bytes. + pub bytes: u64, + /// Their SHA-256. + pub sha256: [u8; 32], +} + +/// Where a download writes: a sink that can be emptied to begin again when a try breaks off. +pub trait Sink: Write { + /// Empties the sink, so the next try writes from the first byte. + fn restart(&mut self) -> std::io::Result<()>; +} + +impl Sink for std::fs::File { + fn restart(&mut self) -> std::io::Result<()> { + use std::io::Seek as _; + self.set_len(0)?; + self.rewind().map(|_| ()) + } +} + +impl Sink for Vec { + fn restart(&mut self) -> std::io::Result<()> { + self.clear(); + Ok(()) + } +} + +/// The releases of one public repository, read without a token. +#[derive(Clone, Debug)] +pub struct Releases { + http: Http, + files: reqwest::Client, + retry: RetryPolicy, + api: Url, + repo: String, +} + +impl Releases { + /// The releases of `repo` (`owner/name`) at the API `api` (`https://api.github.com`, or this machine's fake), + /// each call tried again as `retry` says. + pub fn new(api: &str, repo: &str, retry: RetryPolicy) -> Result { + let api = Url::parse(api.trim_end_matches('/')) + .map_err(|error| Error::Refused(format!("{api} is no address: {error}")))?; + if !is_safe_address(&api) { + return Err(Error::Refused(format!("{api} is not an HTTPS address"))); + } + let plain = |part: &str| { + !part.is_empty() + && part + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'.')) + }; + if !repo + .split_once('/') + .is_some_and(|(owner, name)| plain(owner) && plain(name)) + { + return Err(Error::Refused(format!("{repo} is no owner/name"))); + } + Ok(Releases { + http: Http::new(Duration::from_secs(30), retry), + files: file_client(), + retry, + api, + repo: repo.to_string(), + }) + } + + fn url(&self, path: &str) -> String { + format!( + "{}/repos/{}/{path}", + self.api.as_str().trim_end_matches('/'), + self.repo + ) + } + + async fn get(&self, what: &str, path: &str) -> Result { + let url = self.url(path); + let response = self + .http + .send(|client| { + client + .get(&url) + .header("Accept", "application/vnd.github+json") + .header("X-GitHub-Api-Version", "2022-11-28") + }) + .await?; + serde_json::from_slice(&response.body).map_err(|error| Error::not_the_json(what, &error)) + } + + /// The release of `tag`, or `None` when there is none (GitHub answers 404). + pub async fn by_tag(&self, tag: &str) -> Result, Error> { + if tag.is_empty() + || !tag + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'.' | b'_')) + { + return Err(Error::Refused(format!("{tag} is no plain tag"))); + } + match self.get("the release", &format!("releases/tags/{tag}")).await { + Ok(release) => Ok(Some(release)), + Err(error) if error.status() == Some(404) => Ok(None), + Err(error) => Err(error), + } + } + + /// One page of the releases, the newest first (`page` from 1). + pub async fn page(&self, page: u32) -> Result, Error> { + self.get( + "the releases", + &format!("releases?per_page={RELEASES_PAGE}&page={page}"), + ) + .await + } + + /// The commit a branch is at: its whole name, read from the branch's ref, which no tag of the same name stands + /// for. + pub async fn branch_head(&self, branch: &str) -> Result { + #[derive(Deserialize)] + struct Ref { + object: Object, + } + #[derive(Deserialize)] + struct Object { + sha: String, + } + let what = format!("the branch {branch}"); + let found: Ref = self.get(&what, &format!("git/ref/heads/{branch}")).await?; + if !is_whole_commit(&found.object.sha) { + return Err(Error::Decode { + what, + why: "the answer names no commit".into(), + }); + } + Ok(found.object.sha) + } + + /// How `head` stands to `base` in the repository's history. + pub async fn compare(&self, base: &str, head: &str) -> Result { + #[derive(Deserialize)] + struct Comparison { + status: String, + } + let what = format!("the compare of {base} and {head}"); + // One commit a page: only the status is read. + let found: Comparison = self + .get(&what, &format!("compare/{base}...{head}?per_page=1")) + .await?; + match found.status.as_str() { + "ahead" => Ok(Compared::Ahead), + "behind" => Ok(Compared::Behind), + "identical" => Ok(Compared::Identical), + "diverged" => Ok(Compared::Diverged), + _ => Err(Error::Decode { + what, + why: "the answer's status is none of ahead, behind, identical and diverged".into(), + }), + } + } + + /// Fetches the file at `url` into `sink`, at most `max` bytes, hashed as it is written. A try that breaks off + /// (no connection, a body that ends early, a 429 or a 5xx) begins again from the first byte, as the retry + /// policy says. A sink that cannot be written, a file over `max`, an address that is not HTTPS and `stop` set + /// end it at once. + pub async fn download( + &self, + url: &str, + sink: &mut dyn Sink, + max: u64, + stop: &AtomicBool, + ) -> Result { + let url = Url::parse(url).map_err(|_| Error::Refused("a release's file has no address".into()))?; + if !is_safe_address(&url) { + return Err(Error::Refused(format!( + "{} is not an HTTPS address", + url.host_str().unwrap_or("a release's file") + ))); + } + let mut attempt = 0u32; + loop { + if attempt > 0 { + sink.restart().map_err(|error| local(&error))?; + } + let error = match self.download_once(&url, sink, max, stop).await { + Ok(downloaded) => return Ok(downloaded), + Err(error) => error, + }; + if attempt >= self.retry.retries || !error.is_transient() || stop.load(Ordering::SeqCst) { + return Err(error); + } + let wait = match error.retry_after() { + Some(asked) if asked > self.retry.max_retry_after => return Err(error), + Some(asked) => asked, + None => crate::http::backoff(self.retry.base, self.retry.cap, attempt), + }; + tracing::debug!(attempt, ?wait, "fetching a release's file again"); + tokio::time::sleep(wait).await; + attempt += 1; + } + } + + async fn download_once( + &self, + url: &Url, + sink: &mut dyn Sink, + max: u64, + stop: &AtomicBool, + ) -> Result { + let path = url.path().to_string(); + let mut response = self + .files + .get(url.clone()) + .send() + .await + .map_err(|e| Error::Transport(e.without_url().to_string()))?; + let status = response.status(); + if status != StatusCode::OK { + let headers = response.headers().clone(); + return Err(Error::Status(StatusError { + method: "GET".into(), + path, + status: status.as_u16(), + body: String::new(), + retry_after: retry_after(&headers), + server_time: server_time(&headers), + })); + } + let too_long = || Error::Refused(format!("GET {path}: the file is over {max} bytes")); + if response.content_length().is_some_and(|n| n > max) { + return Err(too_long()); + } + let mut hash = Sha256::new(); + let mut bytes = 0u64; + while let Some(chunk) = response + .chunk() + .await + .map_err(|e| Error::Transport(format!("the file ended early: {}", e.without_url())))? + { + if stop.load(Ordering::SeqCst) { + return Err(Error::Local("the download was stopped".into())); + } + bytes += chunk.len() as u64; + if bytes > max { + return Err(too_long()); + } + sink.write_all(&chunk).map_err(|error| local(&error))?; + hash.update(&chunk); + } + sink.flush().map_err(|error| local(&error))?; + Ok(Downloaded { + bytes, + sha256: hash.finalize().into(), + }) + } +} + +/// A write on this host that failed: no room on its disk, most often. +fn local(error: &std::io::Error) -> Error { + if error.kind() == std::io::ErrorKind::StorageFull { + Error::Local(format!("no room on this host's disk: {error}")) + } else { + Error::Local(format!("the file could not be written: {error}")) + } +} + +/// Whether `name` is a whole commit's name: 40 lowercase hex characters. +pub fn is_whole_commit(name: &str) -> bool { + name.len() == 40 && name.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) +} + +/// The client of a release's files: GitHub sends the address on to its storage, on another host, so redirects are +/// followed to any HTTPS address. No token is ever sent; a file read too slowly is dropped, however long it takes +/// whole. +fn file_client() -> reqwest::Client { + let redirects = Policy::custom(|attempt| { + if attempt.previous().len() <= MAX_REDIRECTS && is_safe_address(attempt.url()) { + attempt.follow() + } else { + attempt.stop() + } + }); + reqwest::Client::builder() + .user_agent(concat!("codefusion-runner/", env!("CARGO_PKG_VERSION"))) + .connect_timeout(Duration::from_secs(10)) + .read_timeout(READ_TIMEOUT) + .redirect(redirects) + .build() + .expect("a client with these settings builds") +} diff --git a/crates/ghapp/tests/release.rs b/crates/ghapp/tests/release.rs new file mode 100644 index 0000000..669517d --- /dev/null +++ b/crates/ghapp/tests/release.rs @@ -0,0 +1,417 @@ +//! A public repository's releases against a fake GitHub at the HTTP edge: what a release lists, how two commits stand, +//! and a release's file fetched whole, hashed, begun again after a try that broke off, and refused when it cannot be +//! kept. + +use std::io::Write; +use std::sync::atomic::AtomicBool; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use cf_fake_github::httpd::{Handler, Reply, Request, Server}; +use cf_ghapp::{Compared, Error, Releases, RetryPolicy, Sink}; +use sha2::{Digest as _, Sha256}; + +const REPO: &str = "codefusion-cc/runner"; + +/// A server that answers each request by `reply` and keeps them all. +struct Fake { + server: Server, + requests: Arc>>, +} + +fn fake(reply: impl Fn(&Request, usize) -> Reply + Send + Sync + 'static) -> Fake { + let requests = Arc::new(Mutex::new(Vec::new())); + let seen = requests.clone(); + let handler: Handler = Arc::new(move |request: Request| { + let reply = { + let mut seen = seen.lock().unwrap(); + seen.push(request.clone()); + reply(&request, seen.len()) + }; + Box::pin(async move { reply }) + }); + Fake { + server: futures_block(Server::start(handler)), + requests, + } +} + +/// Server::start needs the test's runtime; it is already in one. +fn futures_block(future: F) -> F::Output { + tokio::task::block_in_place(|| tokio::runtime::Handle::current().block_on(future)) +} + +impl Fake { + fn requests(&self) -> Vec { + self.requests.lock().unwrap().iter().map(Request::line).collect() + } +} + +fn quick() -> RetryPolicy { + RetryPolicy { + retries: 3, + base: Duration::from_millis(10), + cap: Duration::from_millis(50), + max_retry_after: Duration::from_secs(5), + } +} + +fn releases(fake: &Fake) -> Releases { + Releases::new(&fake.server.url(), REPO, quick()).expect("releases") +} + +fn file(body: &[u8]) -> Reply { + Reply::Response { + status: 200, + headers: vec![("content-type".into(), "application/octet-stream".into())], + body: body.to_vec(), + } +} + +/// The program a release would carry: a megabyte that is not all the same byte. +fn program() -> Vec { + (0..1_048_576u32).map(|n| (n % 251) as u8).collect() +} + +fn sha256(bytes: &[u8]) -> [u8; 32] { + Sha256::digest(bytes).into() +} + +/// A file in a folder of the test's own. +struct Target { + _folder: tempfile::TempDir, + path: std::path::PathBuf, + file: std::fs::File, +} + +fn target() -> Target { + let folder = tempfile::tempdir().expect("folder"); + let path = folder.path().join("download"); + let file = std::fs::File::create(&path).expect("file"); + Target { + _folder: folder, + path, + file, + } +} + +static NO_STOP: AtomicBool = AtomicBool::new(false); + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_file_is_fetched_whole_and_hashed() { + let bytes = program(); + let body = bytes.clone(); + let fake = fake(move |_, _| file(&body)); + let mut target = target(); + let url = format!( + "{}/download/runner-0.1.0-aaaaaaaaaaaa/codefusion-runner-linux-arm64", + fake.server.url() + ); + let got = releases(&fake) + .download(&url, &mut target.file, 2 << 20, &NO_STOP) + .await + .expect("fetched"); + assert_eq!(got.bytes, bytes.len() as u64); + assert_eq!(got.sha256, sha256(&bytes)); + assert_eq!(std::fs::read(&target.path).expect("the file"), bytes); + assert_eq!(fake.requests().len(), 1); +} + +/// GitHub sends a release's file on to its storage, on another host: followed, and no token goes anywhere. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_file_sent_on_to_another_host_is_followed() { + let bytes = program(); + let body = bytes.clone(); + let storage = fake(move |_, _| file(&body)); + let to = format!("{}/objects/1", storage.server.url()); + let github = fake(move |_, _| Reply::empty(302).with_header("location", &to)); + let mut target = target(); + let got = releases(&github) + .download( + &format!("{}/download/x", github.server.url()), + &mut target.file, + 2 << 20, + &NO_STOP, + ) + .await + .expect("fetched"); + assert_eq!(got.sha256, sha256(&bytes)); + assert_eq!(storage.requests(), ["GET /objects/1"]); + let sent = storage.requests.lock().unwrap()[0].clone(); + assert_eq!(sent.header("authorization"), None); +} + +/// A download that breaks off half way begins again from the first byte: the file holds the program once. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_interrupted_download_begins_again_and_holds_the_file_once() { + let bytes = program(); + let body = bytes.clone(); + let fake = fake(move |_, n| { + if n == 1 { + Reply::Truncated { + status: 200, + body: body.clone(), + sent: body.len() / 2, + } + } else { + file(&body) + } + }); + let mut target = target(); + let got = releases(&fake) + .download( + &format!("{}/f", fake.server.url()), + &mut target.file, + 2 << 20, + &NO_STOP, + ) + .await + .expect("fetched on the second try"); + assert_eq!(fake.requests().len(), 2); + assert_eq!(got.bytes, bytes.len() as u64); + assert_eq!(got.sha256, sha256(&bytes)); + assert_eq!(std::fs::read(&target.path).expect("the file"), bytes); +} + +/// One that breaks off every time ends after the tries, as a network fault. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_download_that_always_breaks_off_ends_after_the_tries() { + let body = program(); + let fake = fake(move |_, _| Reply::Truncated { + status: 200, + body: body.clone(), + sent: 1000, + }); + let mut target = target(); + let error = releases(&fake) + .download( + &format!("{}/f", fake.server.url()), + &mut target.file, + 2 << 20, + &NO_STOP, + ) + .await + .expect_err("broke off"); + assert!( + matches!(&error, Error::Transport(why) if why.contains("ended early")), + "{error:?}" + ); + assert_eq!(fake.requests().len(), 4, "the first try and three more"); +} + +/// GitHub's rate limit and its server errors are waited out; a file that is not there is not asked for again. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_429_or_a_5xx_is_asked_again_and_a_404_is_not() { + for (first, asked) in [ + (Reply::empty(429).with_header("retry-after", "1"), 2), + (Reply::empty(502), 2), + (Reply::empty(503), 2), + ] { + let bytes = program(); + let body = bytes.clone(); + let first = Mutex::new(Some(first)); + let fake = fake(move |_, _| first.lock().unwrap().take().unwrap_or_else(|| file(&body))); + let mut target = target(); + let got = releases(&fake) + .download( + &format!("{}/f", fake.server.url()), + &mut target.file, + 2 << 20, + &NO_STOP, + ) + .await + .expect("fetched after the wait"); + assert_eq!(got.sha256, sha256(&bytes)); + assert_eq!(fake.requests().len(), asked); + } + let fake = fake(|_, _| Reply::status_json(404, &serde_json::json!({"message": "Not Found"}))); + let mut target = target(); + let error = releases(&fake) + .download( + &format!("{}/f", fake.server.url()), + &mut target.file, + 2 << 20, + &NO_STOP, + ) + .await + .expect_err("not there"); + assert_eq!(error.status(), Some(404)); + assert_eq!(fake.requests().len(), 1); +} + +/// A sink with room for `room` bytes, as a disk that fills up. +struct Full { + written: Vec, + room: usize, + restarts: usize, +} + +impl Write for Full { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + if self.written.len() + bytes.len() > self.room { + return Err(std::io::Error::from(std::io::ErrorKind::StorageFull)); + } + self.written.extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +impl Sink for Full { + fn restart(&mut self) -> std::io::Result<()> { + self.restarts += 1; + self.written.clear(); + Ok(()) + } +} + +/// A disk that fills up ends the download at once, said as such: asking GitHub again finds no more room. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_full_disk_ends_the_download_at_once() { + let body = program(); + let fake = fake(move |_, _| file(&body)); + let mut full = Full { + written: Vec::new(), + room: 4096, + restarts: 0, + }; + let error = releases(&fake) + .download(&format!("{}/f", fake.server.url()), &mut full, 2 << 20, &NO_STOP) + .await + .expect_err("no room"); + assert!( + matches!(&error, Error::Local(why) if why.starts_with("no room on this host's disk")), + "{error:?}" + ); + assert_eq!((fake.requests().len(), full.restarts), (1, 0)); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_file_over_its_limit_is_refused_and_one_not_over_https_is_not_asked_for() { + let body = program(); + let fake = fake(move |_, _| file(&body)); + let mut target = target(); + let error = releases(&fake) + .download( + &format!("{}/f", fake.server.url()), + &mut target.file, + 1000, + &NO_STOP, + ) + .await + .expect_err("too long"); + assert!( + matches!(&error, Error::Refused(why) if why.contains("over 1000 bytes")), + "{error:?}" + ); + assert_eq!(fake.requests().len(), 1, "not asked again"); + + for url in ["http://example.com/f", "ftp://example.com/f", "no address"] { + let error = releases(&fake) + .download(url, &mut target.file, 1000, &NO_STOP) + .await + .expect_err(url); + assert!(matches!(error, Error::Refused(_)), "{url}: {error:?}"); + } + assert_eq!(fake.requests().len(), 1); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_download_is_stopped_when_the_runner_ends() { + let body = program(); + let fake = fake(move |_, _| file(&body)); + let mut target = target(); + let stop = AtomicBool::new(true); + let error = releases(&fake) + .download( + &format!("{}/f", fake.server.url()), + &mut target.file, + 2 << 20, + &stop, + ) + .await + .expect_err("stopped"); + assert_eq!(error, Error::Local("the download was stopped".into())); + assert_eq!(fake.requests().len(), 1, "not asked again"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_compare_says_how_two_commits_stand_and_nothing_else() { + for (status, said) in [ + ("ahead", Ok(Compared::Ahead)), + ("behind", Ok(Compared::Behind)), + ("identical", Ok(Compared::Identical)), + ("diverged", Ok(Compared::Diverged)), + ("sideways", Err(())), + ] { + let fake = fake(move |_, _| Reply::json(&serde_json::json!({ "status": status, "commits": [] }))); + let got = releases(&fake).compare("aaaaaaaaaaaa", "bbbbbbbbbbbb").await; + assert_eq!(got.map_err(|_| ()), said, "{status}"); + assert_eq!( + fake.requests(), + [format!("GET /repos/{REPO}/compare/aaaaaaaaaaaa...bbbbbbbbbbbb")] + ); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_branch_head_is_read_from_its_ref_and_must_name_a_whole_commit() { + let head = "a".repeat(40); + let answer = head.clone(); + let fake = fake(move |_, n| { + let sha = if n == 1 { answer.clone() } else { "main".into() }; + Reply::json( + &serde_json::json!({ "ref": "refs/heads/main", "object": { "sha": sha, "type": "commit" } }), + ) + }); + let releases = releases(&fake); + assert_eq!(releases.branch_head("main").await, Ok(head)); + assert!(matches!( + releases.branch_head("main").await, + Err(Error::Decode { .. }) + )); + assert_eq!( + fake.requests()[0], + format!("GET /repos/{REPO}/git/ref/heads/main") + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_page_of_releases_is_read_with_its_files() { + let fake = fake(|request, _| { + assert_eq!( + (request.query("per_page"), request.query("page")), + (Some("100"), Some("2")) + ); + Reply::json(&serde_json::json!([{ + "tag_name": "runner-0.1.0-aaaaaaaaaaaa", + "target_commitish": "a".repeat(40), + "draft": false, + "author": { "login": "github-actions[bot]" }, + "assets": [{ "name": "SHA256SUMS", "size": 300, "browser_download_url": "https://github.com/x/SHA256SUMS" }], + }])) + }); + let page = releases(&fake).page(2).await.expect("a page"); + assert_eq!(page.len(), 1); + assert_eq!(page[0].asset("SHA256SUMS").map(|asset| asset.size), Some(300)); + assert_eq!(page[0].asset("other"), None); + assert_eq!( + page[0].author.as_ref().map(|a| a.login.as_str()), + Some("github-actions[bot]") + ); +} + +#[test] +fn a_repository_or_api_that_is_not_plain_is_refused() { + for repo in ["codefusion-cc", "a/b/c", "/runner", "a/", "a b/c", "a/../b"] { + assert!( + Releases::new("https://api.github.com", repo, quick()).is_err(), + "{repo}" + ); + } + assert!(Releases::new("http://api.github.com", REPO, quick()).is_err()); + assert!(Releases::new("https://api.github.com/", REPO, quick()).is_ok()); +} diff --git a/crates/runner/Cargo.toml b/crates/runner/Cargo.toml index c3a50eb..6fb570a 100644 --- a/crates/runner/Cargo.toml +++ b/crates/runner/Cargo.toml @@ -44,6 +44,7 @@ cf-vmm-ch = { path = "../vmm-ch" } cf-budget = { path = "../budget" } cf-fake-github = { path = "../fake-github" } ed25519-dalek.workspace = true +sha2.workspace = true tempfile.workspace = true [target.'cfg(target_os = "macos")'.dev-dependencies] diff --git a/crates/runner/src/main.rs b/crates/runner/src/main.rs index 173db5b..e54ab18 100644 --- a/crates/runner/src/main.rs +++ b/crates/runner/src/main.rs @@ -18,6 +18,8 @@ mod linux; #[cfg(any(target_os = "macos", target_os = "linux"))] mod manager; #[cfg(any(target_os = "macos", target_os = "linux"))] +mod release_update; +#[cfg(any(target_os = "macos", target_os = "linux"))] mod service; #[cfg(target_os = "linux")] mod systemd; diff --git a/crates/runner/src/release_update.rs b/crates/runner/src/release_update.rs new file mode 100644 index 0000000..d7a9381 --- /dev/null +++ b/crates/runner/src/release_update.rs @@ -0,0 +1,1190 @@ +//! How a Runner installed from a release (`host-setup.sh install --version`, no build folder, no toolchain) updates +//! itself: to the commit the console's `update-to` names, from that commit's release of `codefusion-cc/runner`. The +//! service decides when, as for a build ([`cf_daemon::Service`]): the release is fetched and checked while jobs +//! run, put in place once no machine stands, and the program that starts next counts its starts and goes back to +//! the one before when it does not stay up ([`crate::update::guard`]). +//! +//! A release is taken only when it is the release workflow's, of a commit main's history has after the running one, +//! and its program for this host's architecture has the SHA-256 the release's `SHA256SUMS` names and says it is of +//! that commit. Plain `update` (main's head) is a build's: it is not offered here. + +use std::path::{Path, PathBuf}; +use std::sync::Mutex; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::{Duration, Instant}; + +use cf_daemon::update::{NO_HEAD, Pending, SelfUpdate, is_commit}; +use cf_ghapp::{Compared, Release, Releases, RetryPolicy, is_whole_commit}; + +use crate::commands::VERSION; +use crate::update::{Installer, cut, running_commit}; + +/// The repository the Runner's releases are in. +pub const REPO: &str = "codefusion-cc/runner"; +/// The branch whose history decides which commit is newer. +const BRANCH: &str = "main"; +/// Who makes the Runner's releases: the release workflow, with its own token. A release anyone else made is not +/// taken. +pub const PUBLISHER: &str = "github-actions[bot]"; +/// The list of the release's files and their SHA-256. +const SUMS: &str = "SHA256SUMS"; +/// The longest `SHA256SUMS` read: it names three files. +const MAX_SUMS: u64 = 64 * 1024; +/// The largest program taken: the Runner with its guest and cloud-hypervisor is some tens of megabytes. +const MAX_PROGRAM: u64 = 512 * 1024 * 1024; +/// How long a release that is not out is waited for: the console sends a commit once its CI is green, and the +/// release workflow publishes it minutes after, from what it built while CI ran. +pub const RELEASE_WAIT: Duration = Duration::from_secs(16 * 60); +/// How often GitHub is asked for it meanwhile: GitHub answers 60 reads an hour to a host without a token. +pub const ASK_EVERY: Duration = Duration::from_secs(2 * 60); +/// The name a release's program is fetched to beside the installed one. +const DOWNLOAD: &str = "download"; + +/// The asset of the Runner's release for this host, or `None` where the release has none. +pub fn asset_for_this_host() -> Option<&'static str> { + if !cfg!(target_os = "linux") { + return None; + } + match std::env::consts::ARCH { + "x86_64" => Some("codefusion-runner-linux-amd64"), + "aarch64" => Some("codefusion-runner-linux-arm64"), + _ => None, + } +} + +/// The tag of a release of version `version` (`0.1.0`) at the commit `short` (12 characters). +pub fn tag_of(version: &str, short: &str) -> String { + format!("runner-{version}-{short}") +} + +/// The update of a Runner that came from a release. +pub struct ReleaseUpdate { + /// Where the program fetched waits, and how it is put in place. + pub installer: Installer, + /// The running version. + pub running: String, + /// The repository's releases. + releases: Releases, + /// The file of a release this host runs. + asset: String, + /// The package version whose tag is asked for first. + package: String, + /// How long a release that is not out is waited for, asked for every so often. + wait: (Duration, Duration), + /// Set as the Runner ends: a download or a wait under way ends with it. + stop: AtomicBool, + /// Held while a release is fetched: one at a time. + fetching: Mutex<()>, +} + +impl ReleaseUpdate { + /// The update of the running program from the releases at `github_api`, on a host the Runner's release has a + /// program for; `None` elsewhere. + pub fn of_this_program(installer: Installer, github_api: &str) -> Option { + let asset = asset_for_this_host()?; + let retry = RetryPolicy { + retries: 5, + base: Duration::from_secs(2), + cap: Duration::from_secs(60), + max_retry_after: Duration::from_secs(15 * 60), + }; + match ReleaseUpdate::new(installer, github_api, asset, retry) { + Ok(update) => Some(update), + Err(why) => { + tracing::warn!(%why, "the Runner cannot update from its releases"); + None + } + } + } + + /// The update from the releases at `github_api`, taking the file `asset`, each call tried again as `retry` says. + pub fn new( + installer: Installer, + github_api: &str, + asset: &str, + retry: RetryPolicy, + ) -> Result { + Ok(ReleaseUpdate { + installer, + running: VERSION.into(), + releases: Releases::new(github_api, REPO, retry).map_err(|error| error.to_string())?, + asset: asset.to_string(), + package: env!("CARGO_PKG_VERSION").into(), + wait: (RELEASE_WAIT, ASK_EVERY), + stop: AtomicBool::new(false), + fetching: Mutex::new(()), + }) + } + + /// Waits at most `wait` for a release that is not out, asking every `every`. + #[cfg(test)] + pub fn wait_for(&mut self, wait: Duration, every: Duration) { + self.wait = (wait, every); + } + + /// The file a release's program is fetched to. + fn download(&self) -> PathBuf { + self.installer.installed.with_extension(DOWNLOAD) + } + + /// The release of the commit `short`, its tag that of the running package's version or, after a version + /// changed, any `runner--` of the newest releases. Waits for one that is not out yet. + async fn release_of(&self, short: &str) -> Result { + let (wait, every) = self.wait; + let began = Instant::now(); + loop { + if let Some(release) = self.look_for(short).await? { + return Ok(release); + } + if began.elapsed() + every > wait || self.stop.load(Ordering::SeqCst) { + return Err(format!( + "{short} has no release of {REPO} (asked for {}); nothing was downloaded", + cf_daemon::update::spoken(began.elapsed()) + )); + } + tracing::info!(commit = short, "its release is not out yet; asking again"); + self.sleep(every).await; + } + } + + async fn look_for(&self, short: &str) -> Result, String> { + let unread = |error: cf_ghapp::Error| { + format!("the releases of {REPO} could not be read: {error}; nothing was downloaded") + }; + if let Some(release) = self + .releases + .by_tag(&tag_of(&self.package, short)) + .await + .map_err(unread)? + { + return Ok(Some(release)); + } + let ends = format!("-{short}"); + let page = self.releases.page(1).await.map_err(unread)?; + Ok(page.into_iter().find(|release| { + release + .tag_name + .strip_prefix("runner-") + .and_then(|rest| rest.strip_suffix(&ends)) + .is_some_and(|version| { + let parts: Vec<&str> = version.split('.').collect(); + parts.len() == 3 + && parts + .iter() + .all(|part| !part.is_empty() && part.bytes().all(|b| b.is_ascii_digit())) + }) + })) + } + + /// Sleeps for `time`, or less once the Runner ends. + async fn sleep(&self, time: Duration) { + let until = Instant::now() + time; + while Instant::now() < until && !self.stop.load(Ordering::SeqCst) { + tokio::time::sleep(Duration::from_millis(200).min(until - Instant::now())).await; + } + } + + /// Finds the release of `name`, checks it is newer on main than `running`, and fetches and checks its program + /// into [`ReleaseUpdate::download`]. Answers `None` when `running` is that commit already. + async fn fetch(&self, name: &str, short: &str, running: &str) -> Result, String> { + let release = self.release_of(short).await?; + let tag = &release.tag_name; + let not = "nothing was downloaded"; + if release.draft { + return Err(format!("the release {tag} is a draft; {not}")); + } + let made_by = release + .author + .as_ref() + .map_or("nobody", |author| author.login.as_str()); + if made_by != PUBLISHER { + return Err(format!( + "the release {tag} was made by {}, not by the release workflow; {not}", + cut(made_by, 40) + )); + } + let full = &release.target_commitish; + if !is_whole_commit(full) || !full.starts_with(name) { + return Err(format!( + "the release {tag} names the commit `{}`, not {name}; {not}", + cut(full, 40) + )); + } + let github = |error: cf_ghapp::Error| { + format!("GitHub could not say how {short} stands on {BRANCH}: {error}; {not}") + }; + let head = self.releases.branch_head(BRANCH).await.map_err(github)?; + match self.releases.compare(full, &head).await.map_err(github)? { + Compared::Ahead | Compared::Identical => {} + Compared::Behind | Compared::Diverged => { + return Err(format!("{short} is not on {BRANCH} (at {}); {not}", &head[..12])); + } + } + match self.releases.compare(running, full).await.map_err(github)? { + Compared::Ahead => {} + Compared::Identical => return Ok(None), + Compared::Behind => { + return Err(format!( + "{short} is not newer than {running}, which runs: a Runner is not taken back; {not}" + )); + } + Compared::Diverged => { + return Err(format!( + "{short} does not have {running}, which runs and is not on {BRANCH}; update it by hand; {not}" + )); + } + } + let not = "nothing was installed"; + let Some(sums) = release.asset(SUMS) else { + return Err(format!("the release {tag} has no {SUMS}; {not}")); + }; + let Some(program) = release.asset(&self.asset) else { + return Err(format!( + "the release {tag} has no {} for this host's architecture; {not}", + self.asset + )); + }; + let fetched = |what: &str, error: cf_ghapp::Error| { + format!("{what} of {tag} could not be fetched: {error}; {not}") + }; + let mut listed = Vec::new(); + self.releases + .download(&sums.browser_download_url, &mut listed, MAX_SUMS, &self.stop) + .await + .map_err(|error| fetched(SUMS, error))?; + let want = sum_of(&listed, &self.asset) + .ok_or_else(|| format!("{SUMS} of {tag} names no single SHA-256 of {}; {not}", self.asset))?; + let path = self.download(); + let mut file = new_file(&path).map_err(|error| format!("{}: {error}; {not}", path.display()))?; + let got = self + .releases + .download(&program.browser_download_url, &mut file, MAX_PROGRAM, &self.stop) + .await + .map_err(|error| fetched(&self.asset, error))?; + drop(file); + let got = hex(&got.sha256); + if got != want { + return Err(format!( + "{} of {tag} has the SHA-256 {got}, not the {want} its {SUMS} names; {not}", + self.asset + )); + } + Ok(Some(())) + } +} + +impl SelfUpdate for ReleaseUpdate { + fn source(&self) -> String { + format!("https://github.com/{REPO} releases") + } + + fn build(&self, commit: Option<&str>) -> Result { + let running = running_commit(&self.running)?; + let Some(name) = commit else { + return Err(NO_HEAD.into()); + }; + let not = "nothing was downloaded"; + if !is_commit(name) { + return Err(format!("`{}` is no commit's name; {not}", cut(name, 40))); + } + if name.len() < 12 { + return Err(format!( + "{name} is too short to name a release: send 12 characters of it or more; {not}" + )); + } + let short = &name[..12]; + if short == running { + return Ok(self.running.clone()); + } + let Ok(_one) = self.fetching.try_lock() else { + return Err(format!("a release is being fetched already; {not}")); + }; + let path = self.download(); + let fetched = block_on(self.fetch(name, short, running)); + let offered = match fetched { + Ok(None) => Ok(self.running.clone()), + Ok(Some(())) => make_runnable(&path) + .map_err(|error| format!("{}: {error}", path.display())) + .and_then(|()| self.installer.offer(&path, short, &self.asset)), + Err(why) => Err(why), + }; + let _ = std::fs::remove_file(&path); + offered + } + + fn install(&self, pending: &Pending) -> Result<(), String> { + self.installer.install(pending) + } + + fn takes_head(&self) -> bool { + false + } + + fn end(&self) { + self.stop.store(true, Ordering::SeqCst); + } +} + +/// Runs `future` to its end from a thread that may wait: the service's runtime when the thread is one of its +/// blocking ones, else a runtime of its own. +fn block_on(future: F) -> F::Output { + match tokio::runtime::Handle::try_current() { + Ok(handle) => handle.block_on(future), + Err(_) => tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("a runtime") + .block_on(future), + } +} + +/// A new file for this user alone, in place of any left before. +fn new_file(path: &Path) -> std::io::Result { + use std::os::unix::fs::OpenOptionsExt as _; + let _ = std::fs::remove_file(path); + std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(path) +} + +fn make_runnable(path: &Path) -> std::io::Result<()> { + use std::os::unix::fs::PermissionsExt as _; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700)) +} + +/// The SHA-256 a `sha256sum` list names for `file`: one line `<64 hex> ` (or ` *`), and only one. +pub fn sum_of(list: &[u8], file: &str) -> Option { + let text = std::str::from_utf8(list).ok()?; + let mut found = text.lines().filter_map(|line| { + let (sum, name) = line.split_once(' ')?; + let name = name.strip_prefix(' ').or_else(|| name.strip_prefix('*'))?; + (name == file).then_some(sum) + }); + let sum = found.next()?; + if found.next().is_some() + || sum.len() != 64 + || !sum.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) + { + return None; + } + Some(sum.to_string()) +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} + +#[cfg(test)] +mod tests { + use std::collections::{HashMap, VecDeque}; + use std::sync::Arc; + + use cf_daemon::update::{MAX_STARTS, UpdateFile}; + use cf_fake_github::httpd::{Handler, Reply, Request, Server}; + use sha2::{Digest as _, Sha256}; + + use super::*; + use crate::update::{Started, guard}; + + const ASSET: &str = "codefusion-runner-linux-arm64"; + const OTHER: &str = "codefusion-runner-linux-amd64"; + const RUNNING: &str = "aaaaaaaaaaaa"; + const RUNNING_FULL: &str = "aaaaaaaaaaaa0000000000000000000000000000"; + const NEW: &str = "bbbbbbbbbbbb1111111111111111111111111111"; + const HEAD: &str = "cccccccccccc2222222222222222222222222222"; + + fn fits(_: &Path) -> Result<(), String> { + Ok(()) + } + + /// A program that says it is of `short`, as the Runner's `version` does. + fn program(short: &str) -> Vec { + format!("#!/bin/sh\necho \"codefusion-runner 0.1.0+{short}\"\n").into_bytes() + } + + fn sum(bytes: &[u8]) -> String { + hex(&Sha256::digest(bytes)) + } + + /// What the fake GitHub holds and answers. + #[derive(Default)] + struct Hub { + releases: Vec, + files: HashMap>, + compared: HashMap<(String, String), &'static str>, + /// Answers given before the usual one, by path, in turn. + faults: HashMap>, + seen: Vec, + } + + struct Bench { + _folder: tempfile::TempDir, + hub: Arc>, + _server: Server, + runtime: tokio::runtime::Runtime, + update: ReleaseUpdate, + } + + fn reply_for(hub: &mut Hub, request: &Request, base: &str) -> Reply { + let path = request.path.clone(); + hub.seen.push(path.clone()); + if let Some(reply) = hub.faults.get_mut(&path).and_then(VecDeque::pop_front) { + return reply; + } + let repo = format!("/repos/{REPO}/"); + if let Some(rest) = path.strip_prefix(&repo) { + if let Some(tag) = rest.strip_prefix("releases/tags/") { + return hub + .releases + .iter() + .find(|release| release["tag_name"] == tag) + .map_or_else( + || Reply::status_json(404, &serde_json::json!({"message": "Not Found"})), + |release| Reply::json(&with_urls(release, base)), + ); + } + if rest == "releases" { + let all: Vec<_> = hub + .releases + .iter() + .map(|release| with_urls(release, base)) + .collect(); + return Reply::json(&serde_json::Value::Array(all)); + } + if rest == "git/ref/heads/main" { + return Reply::json(&serde_json::json!({"ref": "refs/heads/main", "object": {"sha": HEAD}})); + } + if let Some((base, head)) = rest + .strip_prefix("compare/") + .and_then(|pair| pair.split_once("...")) + { + let status = hub + .compared + .get(&(base.to_string(), head.to_string())) + .copied() + .unwrap_or("diverged"); + return Reply::json(&serde_json::json!({"status": status})); + } + } + match hub.files.get(&path) { + Some(bytes) => Reply::Response { + status: 200, + headers: vec![], + body: bytes.clone(), + }, + None => Reply::empty(404), + } + } + + /// A release as GitHub lists it, its files' addresses on this server. + fn with_urls(release: &serde_json::Value, base: &str) -> serde_json::Value { + let mut release = release.clone(); + let tag = release["tag_name"].as_str().unwrap_or_default().to_string(); + if let Some(assets) = release["assets"].as_array_mut() { + for asset in assets { + let name = asset["name"].as_str().unwrap_or_default().to_string(); + asset["browser_download_url"] = format!("{base}/download/{tag}/{name}").into(); + } + } + release + } + + fn release(tag: &str, commit: &str, files: &[(&str, usize)]) -> serde_json::Value { + serde_json::json!({ + "tag_name": tag, + "target_commitish": commit, + "draft": false, + "author": {"login": PUBLISHER}, + "assets": files.iter().map(|(name, size)| serde_json::json!({"name": name, "size": size, "browser_download_url": ""})).collect::>(), + }) + } + + fn quick() -> RetryPolicy { + RetryPolicy { + retries: 3, + base: Duration::from_millis(10), + cap: Duration::from_millis(50), + max_retry_after: Duration::from_secs(5), + } + } + + /// GitHub with the release of `NEW` (its program for both architectures and their SHA256SUMS), main at `HEAD` + /// past `NEW` past the running commit, and a host whose installed program runs `RUNNING`. + fn bench() -> Bench { + let folder = tempfile::tempdir().expect("folder"); + let state = folder.path().canonicalize().expect("folder"); + std::fs::create_dir_all(state.join("bin")).expect("folder"); + let installed = state.join("bin/codefusion-runner"); + std::fs::write(&installed, "the one that runs").expect("program"); + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .enable_all() + .build() + .expect("a runtime"); + let hub = Arc::new(Mutex::new(Hub::default())); + let answers = hub.clone(); + let base = Arc::new(Mutex::new(String::new())); + let at = base.clone(); + let handler: Handler = Arc::new(move |request: Request| { + let base = at.lock().expect("base").clone(); + let reply = reply_for(&mut answers.lock().expect("hub"), &request, &base); + Box::pin(async move { reply }) + }); + let server = runtime.block_on(Server::start(handler)); + *base.lock().expect("base") = server.url(); + let installer = Installer { + installed, + records: state.join("run"), + state_dir: state, + fit: fits, + }; + let mut update = ReleaseUpdate::new(installer, &server.url(), ASSET, quick()).expect("an update"); + update.wait_for(Duration::ZERO, Duration::ZERO); + update.running = format!("0.1.0+{RUNNING}"); + let bench = Bench { + _folder: folder, + hub, + _server: server, + runtime, + update, + }; + bench.publish(NEW, &program(&NEW[..12]), &program(&NEW[..12])); + { + let mut hub = bench.hub.lock().expect("hub"); + hub.compared.insert((NEW.into(), HEAD.into()), "ahead"); + hub.compared.insert((RUNNING.into(), NEW.into()), "ahead"); + } + bench + } + + impl Bench { + fn tag(commit: &str) -> String { + tag_of(env!("CARGO_PKG_VERSION"), &commit[..12]) + } + + /// The release of `commit` with these programs for this host's architecture and the other, and the + /// SHA256SUMS of both. + fn publish(&self, commit: &str, ours: &[u8], other: &[u8]) { + let tag = Bench::tag(commit); + let sums = format!( + "{} {ASSET}\n{} {OTHER}\n{} host-setup.sh\n", + sum(ours), + sum(other), + sum(b"script") + ); + let mut hub = self.hub.lock().expect("hub"); + hub.releases.retain(|release| release["tag_name"] != tag.as_str()); + hub.releases.insert( + 0, + release( + &tag, + commit, + &[(ASSET, ours.len()), (OTHER, other.len()), (SUMS, sums.len())], + ), + ); + hub.files + .insert(format!("/download/{tag}/{ASSET}"), ours.to_vec()); + hub.files + .insert(format!("/download/{tag}/{OTHER}"), other.to_vec()); + hub.files + .insert(format!("/download/{tag}/{SUMS}"), sums.into_bytes()); + } + + fn change_release(&self, commit: &str, change: impl FnOnce(&mut serde_json::Value)) { + let tag = Bench::tag(commit); + let mut hub = self.hub.lock().expect("hub"); + let release = hub + .releases + .iter_mut() + .find(|release| release["tag_name"] == tag.as_str()) + .expect("the release"); + change(release); + } + + fn fault(&self, path: &str, replies: Vec) { + self.hub + .lock() + .expect("hub") + .faults + .insert(path.to_string(), replies.into()); + } + + fn file_path(commit: &str, file: &str) -> String { + format!("/download/{}/{file}", Bench::tag(commit)) + } + + fn seen(&self) -> Vec { + self.hub.lock().expect("hub").seen.clone() + } + + fn downloads(&self) -> Vec { + self.seen() + .into_iter() + .filter(|path| path.starts_with("/download/")) + .collect() + } + + /// What the host's bin folder holds besides the program that runs. + fn left(&self) -> Vec { + let mut names: Vec = + std::fs::read_dir(self.update.installer.installed.parent().expect("bin")) + .expect("bin") + .flatten() + .map(|entry| entry.file_name().into_string().expect("a name")) + .filter(|name| name != "codefusion-runner") + .collect(); + names.sort(); + names + } + + fn installed(&self) -> Vec { + std::fs::read(&self.update.installer.installed).expect("the program") + } + + fn pending(&self, to: &str) -> Pending { + Pending { + id: "cmd-1".into(), + from: self.update.running.clone(), + to: to.into(), + starts: 0, + failed: None, + } + } + } + + /// What a refused update leaves: the program that runs, nothing beside it, nothing written down. + fn untouched(bench: &Bench) { + assert_eq!(bench.installed(), b"the one that runs"); + assert_eq!(bench.left(), Vec::::new()); + assert_eq!( + UpdateFile::in_state(&bench.update.installer.state_dir).read(), + Ok(None) + ); + } + + #[test] + fn a_newer_commit_of_main_is_fetched_checked_and_installed_and_rolled_back_when_it_does_not_stay_up() { + let bench = bench(); + let version = format!("0.1.0+{}", &NEW[..12]); + assert_eq!(bench.update.build(Some(NEW)), Ok(version.clone())); + assert_eq!( + bench.left(), + ["codefusion-runner.next"], + "the download is gone, the program checked waits" + ); + assert_eq!( + bench.downloads(), + [Bench::file_path(NEW, SUMS), Bench::file_path(NEW, ASSET)], + "only this host's program" + ); + assert_eq!( + bench.installed(), + b"the one that runs", + "nothing in place before the install" + ); + + let pending = bench.pending(&version); + bench.update.install(&pending).expect("installed"); + assert_eq!(bench.installed(), program(&NEW[..12])); + assert_eq!(bench.left(), ["codefusion-runner.prev"]); + let installed = &bench.update.installer.installed; + let state = &bench.update.installer.state_dir; + // The new program fails its health check: it does not stay up, start after start. + for _ in 0..MAX_STARTS { + assert!(matches!( + guard(state, installed, &version), + Ok(Started::Trying(_)) + )); + } + let said = guard(state, installed, &version).expect_err("taken back"); + assert!(said.contains("is back in place"), "{said}"); + assert_eq!(bench.installed(), b"the one that runs"); + match guard(state, installed, &pending.from) { + Ok(Started::Failed { id, message }) => { + assert_eq!(id, "cmd-1"); + assert!( + message.contains("did not stay up and was taken back"), + "{message}" + ); + } + other => panic!("{other:?}"), + } + } + + #[test] + fn a_program_whose_sha256_is_not_the_one_listed_is_not_offered() { + let bench = bench(); + let listed = program(&NEW[..12]); + bench.publish(NEW, &listed, b"other"); + // The release's file is another than the one its list names. + let mut altered = listed.clone(); + altered.extend_from_slice(b"# and one more line\n"); + bench + .hub + .lock() + .expect("hub") + .files + .insert(Bench::file_path(NEW, ASSET), altered.clone()); + let said = bench.update.build(Some(NEW)).expect_err("refused"); + assert_eq!( + said, + format!( + "{ASSET} of {} has the SHA-256 {}, not the {} its SHA256SUMS names; nothing was installed", + Bench::tag(NEW), + sum(&altered), + sum(&listed) + ) + ); + untouched(&bench); + } + + #[test] + fn a_release_without_this_hosts_program_or_its_list_is_refused_before_any_download() { + let bench = bench(); + bench.change_release(NEW, |release| { + release["assets"] + .as_array_mut() + .expect("assets") + .retain(|asset| asset["name"] != ASSET); + }); + assert_eq!( + bench.update.build(Some(NEW)), + Err(format!( + "the release {} has no {ASSET} for this host's architecture; nothing was installed", + Bench::tag(NEW) + )) + ); + let bench2 = self::bench(); + bench2.change_release(NEW, |release| { + release["assets"] + .as_array_mut() + .expect("assets") + .retain(|asset| asset["name"] != SUMS); + }); + assert_eq!( + bench2.update.build(Some(NEW)), + Err(format!( + "the release {} has no SHA256SUMS; nothing was installed", + Bench::tag(NEW) + )) + ); + for bench in [bench, bench2] { + assert_eq!(bench.downloads(), Vec::::new()); + untouched(&bench); + } + } + + #[test] + fn a_list_that_does_not_name_this_hosts_program_once_is_refused() { + let bench = bench(); + let ours = program(&NEW[..12]); + for list in [ + format!("{} {OTHER}\n", sum(b"x")), + format!("{} {ASSET}\n{} {ASSET}\n", sum(&ours), sum(&ours)), + format!("{} {ASSET}\n", &sum(&ours)[..63]), + format!("{} {ASSET}\n", sum(&ours).to_uppercase()), + ] { + bench + .hub + .lock() + .expect("hub") + .files + .insert(Bench::file_path(NEW, SUMS), list.into_bytes()); + assert_eq!( + bench.update.build(Some(NEW)), + Err(format!( + "SHA256SUMS of {} names no single SHA-256 of {ASSET}; nothing was installed", + Bench::tag(NEW) + )) + ); + untouched(&bench); + } + } + + #[test] + fn a_commit_with_no_release_is_refused_and_one_that_comes_out_while_it_waits_is_taken() { + let mut bench = bench(); + let none = "dddddddddddd3333333333333333333333333333"; + let said = bench.update.build(Some(none)).expect_err("no release"); + assert!( + said.starts_with(&format!("dddddddddddd has no release of {REPO} (asked for ")), + "{said}" + ); + assert!(said.ends_with("; nothing was downloaded"), "{said}"); + assert_eq!( + bench.seen(), + [ + format!("/repos/{REPO}/releases/tags/{}", Bench::tag(none)), + format!("/repos/{REPO}/releases") + ], + "the tag of this version, then the newest releases" + ); + untouched(&bench); + + // Out after two looks: waited for. + bench + .update + .wait_for(Duration::from_secs(5), Duration::from_millis(100)); + let tags = format!("/repos/{REPO}/releases/tags/{}", Bench::tag(NEW)); + let not_found = Reply::status_json(404, &serde_json::json!({"message": "Not Found"})); + bench.fault(&tags, vec![not_found.clone(), not_found]); + bench.fault( + &format!("/repos/{REPO}/releases"), + vec![ + Reply::json(&serde_json::json!([])), + Reply::json(&serde_json::json!([])), + ], + ); + assert_eq!(bench.update.build(Some(NEW)), Ok(format!("0.1.0+{}", &NEW[..12]))); + } + + #[test] + fn a_release_of_another_package_version_is_found_among_the_newest() { + let bench = bench(); + bench.change_release(NEW, |release| { + release["tag_name"] = format!("runner-0.2.0-{}", &NEW[..12]).into(); + }); + { + let mut hub = bench.hub.lock().expect("hub"); + for file in [ASSET, SUMS] { + let bytes = hub.files.remove(&Bench::file_path(NEW, file)).expect("a file"); + hub.files + .insert(format!("/download/runner-0.2.0-{}/{file}", &NEW[..12]), bytes); + } + // Tags that only look like it are passed over. + for tag in [ + format!("runner-0.2-{}", &NEW[..12]), + format!("image-0.2.0-{}", &NEW[..12]), + ] { + let mut fake = release(&tag, NEW, &[]); + fake["assets"] = serde_json::json!([]); + hub.releases.insert(0, fake); + } + } + assert_eq!(bench.update.build(Some(NEW)), Ok(format!("0.1.0+{}", &NEW[..12]))); + } + + #[test] + fn github_s_rate_limit_and_server_errors_are_waited_out() { + let bench = bench(); + let tags = format!("/repos/{REPO}/releases/tags/{}", Bench::tag(NEW)); + bench.fault( + &tags, + vec![ + Reply::empty(429).with_header("retry-after", "1"), + Reply::empty(500), + ], + ); + bench.fault( + &format!("/repos/{REPO}/compare/{NEW}...{HEAD}"), + vec![Reply::empty(503)], + ); + bench.fault( + &format!("/repos/{REPO}/git/ref/heads/main"), + vec![Reply::empty(502)], + ); + bench.fault( + &Bench::file_path(NEW, ASSET), + vec![Reply::empty(503), Reply::empty(429)], + ); + assert_eq!(bench.update.build(Some(NEW)), Ok(format!("0.1.0+{}", &NEW[..12]))); + assert_eq!(bench.seen().iter().filter(|path| **path == tags).count(), 3); + + // One that lasts ends the update, said as GitHub's. + let bench = self::bench(); + bench.fault(&tags, vec![Reply::empty(503); 10]); + let said = bench.update.build(Some(NEW)).expect_err("GitHub is down"); + assert!( + said.starts_with(&format!("the releases of {REPO} could not be read: ")), + "{said}" + ); + assert!(said.contains("503"), "{said}"); + untouched(&bench); + } + + #[test] + fn an_interrupted_download_begins_again_and_one_that_breaks_off_every_time_installs_nothing() { + let bench = bench(); + let ours = program(&NEW[..12]); + let cut_short = || Reply::Truncated { + status: 200, + body: ours.clone(), + sent: ours.len() / 2, + }; + bench.fault(&Bench::file_path(NEW, ASSET), vec![cut_short()]); + assert_eq!(bench.update.build(Some(NEW)), Ok(format!("0.1.0+{}", &NEW[..12]))); + assert_eq!( + std::fs::read(bench.update.installer.next()).expect("offered"), + ours + ); + + let bench = self::bench(); + bench.fault(&Bench::file_path(NEW, ASSET), vec![cut_short(); 10]); + let said = bench.update.build(Some(NEW)).expect_err("broke off"); + assert!( + said.starts_with(&format!("{ASSET} of {} could not be fetched: ", Bench::tag(NEW))), + "{said}" + ); + assert!(said.ends_with("; nothing was installed"), "{said}"); + untouched(&bench); + } + + #[test] + fn a_program_that_does_not_run_here_or_says_another_commit_is_not_offered() { + // Another architecture's program under this one's name: it checks out, and does not run. + let bench = bench(); + let elf_of_another = b"\x7fELF\x02\x01\x01\x00not for this machine".to_vec(); + bench.publish(NEW, &elf_of_another, b"other"); + let said = bench.update.build(Some(NEW)).expect_err("does not run"); + assert!( + said.starts_with(&format!("{ASSET} does not run on this host: ")), + "{said}" + ); + untouched(&bench); + + let bench = self::bench(); + bench.publish(NEW, &program("eeeeeeeeeeee"), b"other"); + let said = bench.update.build(Some(NEW)).expect_err("another commit"); + assert_eq!( + said, + format!( + "{ASSET} says it is `codefusion-runner 0.1.0+eeeeeeeeeeee`, not of {}", + &NEW[..12] + ) + ); + untouched(&bench); + } + + #[test] + fn a_commit_older_than_the_running_one_or_off_main_is_refused_before_any_download() { + for (pair, status, said) in [ + ( + (RUNNING, NEW), + "behind", + format!( + "bbbbbbbbbbbb is not newer than {RUNNING}, which runs: a Runner is not taken back; nothing was downloaded" + ), + ), + ( + (RUNNING, NEW), + "diverged", + format!( + "bbbbbbbbbbbb does not have {RUNNING}, which runs and is not on main; update it by hand; nothing was downloaded" + ), + ), + ( + (NEW, HEAD), + "behind", + "bbbbbbbbbbbb is not on main (at cccccccccccc); nothing was downloaded".into(), + ), + ( + (NEW, HEAD), + "diverged", + "bbbbbbbbbbbb is not on main (at cccccccccccc); nothing was downloaded".into(), + ), + ] { + let bench = bench(); + bench + .hub + .lock() + .expect("hub") + .compared + .insert((pair.0.into(), pair.1.into()), status); + assert_eq!(bench.update.build(Some(NEW)), Err(said), "{pair:?} {status}"); + assert_eq!(bench.downloads(), Vec::::new()); + untouched(&bench); + } + } + + #[test] + fn a_release_not_made_by_the_workflow_or_of_another_commit_is_refused() { + let tag = Bench::tag(NEW); + for (change, said) in [ + ( + serde_json::json!({"author": {"login": "someone"}}), + format!( + "the release {tag} was made by someone, not by the release workflow; nothing was downloaded" + ), + ), + ( + serde_json::json!({"author": null}), + format!( + "the release {tag} was made by nobody, not by the release workflow; nothing was downloaded" + ), + ), + ( + serde_json::json!({"draft": true}), + format!("the release {tag} is a draft; nothing was downloaded"), + ), + ( + serde_json::json!({"target_commitish": "main"}), + format!("the release {tag} names the commit `main`, not {NEW}; nothing was downloaded"), + ), + ( + serde_json::json!({"target_commitish": "bbbbbbbbbbbb9999999999999999999999999999"}), + format!( + "the release {tag} names the commit `bbbbbbbbbbbb9999999999999999999999999999`, not {NEW}; nothing \ + was downloaded" + ), + ), + ] { + let bench = bench(); + bench.change_release(NEW, |release| { + for (key, value) in change.as_object().expect("an object") { + release[key] = value.clone(); + } + }); + assert_eq!(bench.update.build(Some(NEW)), Err(said)); + assert_eq!(bench.downloads(), Vec::::new()); + untouched(&bench); + } + } + + #[test] + fn main_s_head_is_not_taken_and_names_that_cannot_name_a_release_ask_nothing() { + let bench = bench(); + assert_eq!(bench.update.build(None), Err(NO_HEAD.into())); + assert!(!bench.update.takes_head()); + assert_eq!( + bench.update.build(Some("bbbbbbb")), + Err("bbbbbbb is too short to name a release: send 12 characters of it or more; nothing was downloaded".into()) + ); + for name in ["main", "--x", "BBBBBBBBBBBB", "bbbbbbbbbbbb~1"] { + let said = bench.update.build(Some(name)).expect_err(name); + assert!( + said.ends_with("is no commit's name; nothing was downloaded"), + "{name}: {said}" + ); + } + // The commit that runs: nothing to do. + assert_eq!( + bench.update.build(Some(RUNNING_FULL)), + Ok(bench.update.running.clone()) + ); + assert_eq!(bench.seen(), Vec::::new()); + let mut bench = bench; + bench.update.running = "0.1.0+unknown".into(); + assert!( + bench + .update + .build(Some(NEW)) + .expect_err("no commit") + .contains("names no commit") + ); + untouched(&bench); + } + + /// The service runs one update at a time; a second fetch at once is refused all the same, and fetches nothing. + #[test] + fn a_second_update_while_one_fetches_is_refused() { + let bench = bench(); + let tags = format!("/repos/{REPO}/releases/tags/{}", Bench::tag(NEW)); + let slow = Reply::After( + Duration::from_secs(2), + Box::new(Reply::status_json(404, &serde_json::json!({}))), + ); + bench.fault(&tags, vec![slow]); + bench.fault( + &format!("/repos/{REPO}/releases"), + vec![Reply::json(&serde_json::json!([]))], + ); + std::thread::scope(|scope| { + let first = scope.spawn(|| bench.update.build(Some(NEW))); + let began = Instant::now(); + while bench.seen().is_empty() { + assert!(began.elapsed() < Duration::from_secs(10), "the first asks"); + std::thread::sleep(Duration::from_millis(10)); + } + assert_eq!( + bench.update.build(Some(NEW)), + Err("a release is being fetched already; nothing was downloaded".into()) + ); + assert!( + first.join().expect("the first").is_err(), + "no release on the first look" + ); + }); + assert_eq!(bench.seen().len(), 2, "the second asked nothing"); + } + + /// The Runner ends while a release is waited for: the wait ends with it. + #[test] + fn a_wait_for_a_release_ends_when_the_runner_ends() { + let mut bench = bench(); + bench + .update + .wait_for(Duration::from_secs(600), Duration::from_secs(60)); + let began = Instant::now(); + std::thread::scope(|scope| { + let waits = scope.spawn(|| { + bench + .update + .build(Some("dddddddddddd3333333333333333333333333333")) + }); + while bench.seen().len() < 2 { + std::thread::sleep(Duration::from_millis(10)); + } + bench.update.end(); + assert!( + waits + .join() + .expect("the wait") + .expect_err("ended") + .contains("has no release") + ); + }); + assert!(began.elapsed() < Duration::from_secs(10), "{:?}", began.elapsed()); + } + + /// The service fetches on a thread of its runtime's blocking pool, whose runtime the download uses. + #[test] + fn a_fetch_on_the_service_s_blocking_pool_uses_its_runtime() { + let Bench { + _folder, + hub: _hub, + _server, + runtime, + update, + } = bench(); + let update = Arc::new(update); + let fetching = update.clone(); + let said = runtime + .block_on(runtime.spawn_blocking(move || fetching.build(Some(NEW)))) + .expect("the thread"); + assert_eq!(said, Ok(format!("0.1.0+{}", &NEW[..12]))); + } + + #[test] + fn a_sum_is_read_from_its_one_line() { + let a = "a".repeat(64); + let b = "b".repeat(64); + assert_eq!( + sum_of(format!("{a} x\n{b} y\n").as_bytes(), "y"), + Some(b.clone()) + ); + assert_eq!(sum_of(format!("{a} *y\n").as_bytes(), "y"), Some(a.clone())); + for list in [ + format!("{a} y\n{b} y\n"), + format!("{a} yy\n"), + format!("{a} y\n"), + format!("{a}\ty\n"), + format!("{} y\n", &a[..62]), + String::new(), + ] { + assert_eq!(sum_of(list.as_bytes(), "y"), None, "{list:?}"); + } + assert_eq!(sum_of(b"\xff\xfe y", "y"), None); + } + + #[test] + fn this_host_takes_its_own_architecture_s_program() { + let asset = asset_for_this_host(); + match (std::env::consts::OS, std::env::consts::ARCH) { + ("linux", "x86_64") => assert_eq!(asset, Some(OTHER)), + ("linux", "aarch64") => assert_eq!(asset, Some(ASSET)), + _ => assert_eq!(asset, None, "no release is made for this system"), + } + } +} diff --git a/crates/runner/src/service.rs b/crates/runner/src/service.rs index 499bfa0..5d58a29 100644 --- a/crates/runner/src/service.rs +++ b/crates/runner/src/service.rs @@ -20,7 +20,8 @@ use crate::commands::VERSION; use crate::config::{Class, Config}; use crate::hidden::{self, NoLine}; use crate::manager::ServiceManager; -use crate::update::{self, CheckoutUpdate, Started}; +use crate::release_update::ReleaseUpdate; +use crate::update::{self, CheckoutUpdate, Installer, Started}; /// How long GitHub and the console have to answer one call of a command. const ANSWER: Duration = Duration::from_secs(30); @@ -440,20 +441,11 @@ fn run_daemon(config: &Path) -> Result<(), String> { } // Only the installed program replaces itself: one run from elsewhere would replace another. let updates = (program == config.program()) - .then(|| { - CheckoutUpdate::of_this_program( - &config.state_dir, - &config.program(), - &config.records(), - config.build_folder.as_deref(), - This::fit, - ) - }) - .flatten() - .map(Arc::new); + .then(|| self_update(&config)) + .flatten(); let runtime = runtime(2)?; let ran = runtime.block_on(async { - let updates = updates.clone().map(|updates| updates as Arc); + let updates = updates.clone(); let (app, key) = app_and_key(&config)?; let held = key.map(|tokens| held(&config, &path_of_settings, tokens)); // A key that serves says why it is kept, in its own words. @@ -521,7 +513,7 @@ fn run_daemon(config: &Path) -> Result<(), String> { // An update's build is ended with the Runner (the console sends its command again to the one that starts // next), and a git call of one is not waited for. if let Some(updates) = &updates { - updates.end_build(); + updates.end(); } runtime.shutdown_timeout(Duration::from_secs(5)); if ran.is_ok() @@ -533,6 +525,34 @@ fn run_daemon(config: &Path) -> Result<(), String> { ran } +/// How the installed program updates itself: from the checkout it was built in when the configuration names that +/// as its build folder; else, on a host the Runner's releases are made for, from the release of the commit the +/// console names; else from a checkout, whose build says why it builds nothing. +fn self_update(config: &Config) -> Option> { + let installer = || Installer { + installed: config.program(), + records: config.records(), + state_dir: config.state_dir.clone(), + fit: This::fit, + }; + let checkout = || { + CheckoutUpdate::of_this_program( + &config.state_dir, + &config.program(), + &config.records(), + config.build_folder.as_deref(), + This::fit, + ) + .map(|update| Arc::new(update) as Arc) + }; + if config.build_folder.is_some() { + return checkout(); + } + ReleaseUpdate::of_this_program(installer(), &config.github_api) + .map(|update| Arc::new(update) as Arc) + .or_else(checkout) +} + /// What a Runner starts on: its settings, as [`startable`] took them, what became of the App's key folder they named, /// and the state folder's lock, which it holds for as long as it runs. struct Starting { diff --git a/crates/runner/src/update.rs b/crates/runner/src/update.rs index 98ab8c8..19126de 100644 --- a/crates/runner/src/update.rs +++ b/crates/runner/src/update.rs @@ -30,10 +30,17 @@ const PREVIOUS: &str = "prev"; const NEXT: &str = "next"; /// The commit part of a version: `0.1.0+446125c0a1b2` → `446125c0a1b2`. -fn commit_of(version: &str) -> Option<&str> { +pub(crate) fn commit_of(version: &str) -> Option<&str> { version.split_once('+').map(|(_, commit)| commit) } +/// The twelve characters of the commit `running` (a version) names, or why an update cannot go from it. +pub(crate) fn running_commit(running: &str) -> Result<&str, String> { + commit_of(running) + .filter(|running| running.len() == 12 && running.bytes().all(|b| b.is_ascii_hexdigit())) + .ok_or_else(|| format!("{running} names no commit it was built at; update it by hand")) +} + /// The update of a Runner built from a checkout on this host. pub struct CheckoutUpdate { /// The checkout the running program was built from. @@ -44,18 +51,12 @@ pub struct CheckoutUpdate { pub env: Vec<(String, OsString)>, /// Where the build writes what it says. pub log: PathBuf, - /// The installed program, which the service manager runs. - pub installed: PathBuf, - /// The folder of the machines' records: an install is refused while one is there. - pub records: PathBuf, - /// The state folder, where [`UpdateFile`] is. - pub state_dir: PathBuf, + /// Where the program built waits, and how it is put in place. + pub installer: Installer, /// The running version. pub running: String, /// How long a build may take. pub build_limit: Duration, - /// Whether a program can start machines on this host. - pub fit: fn(&Path) -> Result<(), String>, /// ` main`, for the console. source: String, /// The process group of the build that runs, which [`CheckoutUpdate::end_build`] ends. @@ -108,12 +109,14 @@ impl CheckoutUpdate { cargo: cargo_home.join("bin").join("cargo"), env, log: state_dir.join("logs").join("update.log"), - installed: installed.to_path_buf(), - records: records.to_path_buf(), - state_dir: state_dir.to_path_buf(), + installer: Installer { + installed: installed.to_path_buf(), + records: records.to_path_buf(), + state_dir: state_dir.to_path_buf(), + fit, + }, running: VERSION.into(), build_limit: BUILD_LIMIT, - fit, source: String::new(), building: Mutex::new(None), build_folder: build_folder.map(Path::to_path_buf), @@ -143,17 +146,6 @@ impl CheckoutUpdate { self.checkout.join("target/release/codefusion-runner") } - /// The program kept beside the installed one. - fn previous(&self) -> PathBuf { - self.installed.with_extension(PREVIOUS) - } - - /// The program built and checked, put beside the installed one: the checkout is the person's too, and a build - /// of theirs while the machines end does not go in. - fn next(&self) -> PathBuf { - self.installed.with_extension(NEXT) - } - fn git(&self, args: &[&str]) -> Result { git(&self.checkout, args) } @@ -307,14 +299,7 @@ impl SelfUpdate for CheckoutUpdate { } fn build(&self, commit: Option<&str>) -> Result { - let running = commit_of(&self.running) - .filter(|running| running.len() == 12 && running.bytes().all(|b| b.is_ascii_hexdigit())) - .ok_or_else(|| { - format!( - "{} names no commit it was built at; update it by hand", - self.running - ) - })?; + let running = running_commit(&self.running)?; if let Some(name) = commit.filter(|name| !is_commit(name)) { return Err(format!( "`{}` is no commit's name; nothing was fetched or built", @@ -356,9 +341,48 @@ impl SelfUpdate for CheckoutUpdate { // Peeled, the name is the commit's alone: `git checkout` takes a branch named as a whole commit before it. self.git(&["checkout", "-q", "--detach", &format!("{target}^{{commit}}")])?; self.cargo_build()?; - let built = self.built(); - let printed = run_at_most(Command::new(&built).arg("version"), GIT_LIMIT) - .map_err(|error| format!("{} version: {error}", built.display()))?; + self.installer.offer(&self.built(), short, "the program built") + } + + fn install(&self, pending: &Pending) -> Result<(), String> { + self.installer.install(pending) + } + + fn end(&self) { + self.end_build(); + } +} + +/// Where a new program waits until it is put in place, and how it is: the same for a program built here and one +/// taken from a release. +pub struct Installer { + /// The installed program, which the service manager runs. + pub installed: PathBuf, + /// The folder of the machines' records: an install is refused while one is there. + pub records: PathBuf, + /// The state folder, where [`UpdateFile`] is. + pub state_dir: PathBuf, + /// Whether a program can start machines on this host. + pub fit: fn(&Path) -> Result<(), String>, +} + +impl Installer { + /// The program kept beside the installed one. + pub fn previous(&self) -> PathBuf { + self.installed.with_extension(PREVIOUS) + } + + /// The program checked, put beside the installed one until it is installed: what it came from (a checkout that + /// is the person's too, a download) may change while the machines end, and that does not go in. + pub fn next(&self) -> PathBuf { + self.installed.with_extension(NEXT) + } + + /// Offers `program` (`what` names it) for the install once it says it is of the commit `short` and `fit` passes + /// a copy of it: the copy is the one installed. Answers the version it says. + pub fn offer(&self, program: &Path, short: &str, what: &str) -> Result { + let printed = run_at_most(Command::new(program).arg("version"), GIT_LIMIT) + .map_err(|error| format!("{what} does not run on this host: {error}"))?; let version = printed .trim() .strip_prefix("codefusion-runner ") @@ -366,16 +390,18 @@ impl SelfUpdate for CheckoutUpdate { .to_string(); if commit_of(&version) != Some(short) { return Err(format!( - "the program built says it is `{}`, not of {short}", + "{what} says it is `{}`, not of {short}", cut(printed.trim(), 100) )); } - copy_whole(&built, &self.next())?; + copy_whole(program, &self.next())?; (self.fit)(&self.next())?; Ok(version) } - fn install(&self, pending: &Pending) -> Result<(), String> { + /// Puts the program offered last in place of the running one, keeping the running one to go back to, with + /// `pending` written down first. Refused while a machine has a record. + pub fn install(&self, pending: &Pending) -> Result<(), String> { let records = standing(&self.records); if records > 0 { return Err(format!( @@ -501,7 +527,7 @@ fn redact_url(url: &str) -> String { } /// At most `chars` characters of `text`. -fn cut(text: &str, chars: usize) -> String { +pub(crate) fn cut(text: &str, chars: usize) -> String { text.chars().take(chars).collect() } @@ -621,7 +647,11 @@ fn copy_checked(from: &Path, to: &Path, check: fn(&Path) -> Result<(), String>) let mut name = to.file_name().ok_or("a copy needs a file name")?.to_os_string(); name.push(".new"); let fresh = to.with_file_name(name); - std::fs::copy(from, &fresh).map_err(|error| format!("{}: {error}", fresh.display()))?; + if let Err(error) = std::fs::copy(from, &fresh) { + // Half a copy (a disk with no room) is not left behind. + let _ = std::fs::remove_file(&fresh); + return Err(format!("{}: {error}", fresh.display())); + } if let Err(error) = check(&fresh) { let _ = std::fs::remove_file(&fresh); return Err(error); @@ -716,12 +746,14 @@ chmod +x target/release/codefusion-runner"#; ("MARKS".into(), root.join("marks").into_os_string()), ], log: state.join("logs/update.log"), - installed, - records: state.join("run"), - state_dir: state, + installer: Installer { + installed, + records: state.join("run"), + state_dir: state, + fit: signed, + }, running: format!("0.1.0+{}", &first[..12]), build_limit: Duration::from_secs(60), - fit: signed, source: String::new(), building: Mutex::new(None), build_folder: Some(checkout.clone()), @@ -996,7 +1028,8 @@ chmod +x target/release/codefusion-runner"#; assert_eq!(bench.head(), bench.second); assert_eq!(bench.builds(), 1); bench.update.install(&bench.pending()).expect("installed"); - let says = run_at_most(&mut Command::new(&bench.update.installed), GIT_LIMIT).expect("it runs"); + let says = + run_at_most(&mut Command::new(&bench.update.installer.installed), GIT_LIMIT).expect("it runs"); assert_eq!( says, format!("codefusion-runner {}", Bench::version_of(&bench.second)) @@ -1317,7 +1350,7 @@ chmod +x target/release/codefusion-runner"#; assert!(said.contains("0.1.0+aaaaaaaaaaaa-dirty"), "{said}"); let mut bench = self::bench(BUILDS); - bench.update.fit = unsigned; + bench.update.installer.fit = unsigned; let said = bench.update.build(None).expect_err("refused"); assert!(said.ends_with("is not signed"), "{said}"); } @@ -1328,13 +1361,16 @@ chmod +x target/release/codefusion-runner"#; bench.update.build(None).expect("built"); bench.update.install(&bench.pending()).expect("installed"); let read = |path: &Path| std::fs::read_to_string(path).expect("a file"); - assert_eq!(read(&bench.update.previous()), "the one that runs"); - assert_eq!(read(&bench.update.installed), read(&bench.update.built())); + assert_eq!(read(&bench.update.installer.previous()), "the one that runs"); assert_eq!( - UpdateFile::in_state(&bench.update.state_dir).read(), + read(&bench.update.installer.installed), + read(&bench.update.built()) + ); + assert_eq!( + UpdateFile::in_state(&bench.update.installer.state_dir).read(), Ok(Some(bench.pending())) ); - let left: Vec<_> = std::fs::read_dir(bench.update.installed.parent().expect("bin")) + let left: Vec<_> = std::fs::read_dir(bench.update.installer.installed.parent().expect("bin")) .expect("bin") .flatten() .map(|entry| entry.file_name().into_string().expect("a name")) @@ -1353,7 +1389,7 @@ chmod +x target/release/codefusion-runner"#; std::fs::write(bench.update.built(), "a build of another branch").expect("program"); bench.update.install(&bench.pending()).expect("installed"); assert_eq!( - std::fs::read_to_string(&bench.update.installed).expect("program"), + std::fs::read_to_string(&bench.update.installer.installed).expect("program"), checked ); } @@ -1362,31 +1398,37 @@ chmod +x target/release/codefusion-runner"#; fn an_install_is_refused_while_a_machine_has_a_record_and_changes_nothing() { let bench = bench(BUILDS); bench.update.build(None).expect("built"); - std::fs::create_dir_all(&bench.update.records).expect("folder"); - std::fs::write(bench.update.records.join("job.json"), "{}").expect("record"); + std::fs::create_dir_all(&bench.update.installer.records).expect("folder"); + std::fs::write(bench.update.installer.records.join("job.json"), "{}").expect("record"); assert_eq!( bench.update.install(&bench.pending()), Err("1 machine has a record; nothing was installed".into()) ); assert_eq!( - std::fs::read_to_string(&bench.update.installed).expect("program"), + std::fs::read_to_string(&bench.update.installer.installed).expect("program"), "the one that runs" ); - assert!(!bench.update.previous().exists()); - assert_eq!(UpdateFile::in_state(&bench.update.state_dir).read(), Ok(None)); + assert!(!bench.update.installer.previous().exists()); + assert_eq!( + UpdateFile::in_state(&bench.update.installer.state_dir).read(), + Ok(None) + ); } #[test] fn an_install_whose_program_cannot_be_placed_leaves_no_update_written_down() { let mut bench = bench(BUILDS); bench.update.build(None).expect("built"); - bench.update.fit = unsigned; + bench.update.installer.fit = unsigned; assert!(bench.update.install(&bench.pending()).is_err()); assert_eq!( - std::fs::read_to_string(&bench.update.installed).expect("program"), + std::fs::read_to_string(&bench.update.installer.installed).expect("program"), "the one that runs" ); - assert_eq!(UpdateFile::in_state(&bench.update.state_dir).read(), Ok(None)); + assert_eq!( + UpdateFile::in_state(&bench.update.installer.state_dir).read(), + Ok(None) + ); } /// The starts of an installed program, as the service manager makes them: the new one counts its starts, goes back after diff --git a/docs/MAP.md b/docs/MAP.md index b5b2b23..354540c 100644 --- a/docs/MAP.md +++ b/docs/MAP.md @@ -13,25 +13,25 @@ | `crates/nft` (`cf-nft`) | nf_tables rules without the `nft` program. `lib.rs` is the model (`Match`, `Rule` and `Rule::new`, `Chain` on a `Hook`, `Table` in a `Family`: inet, bridge, netdev) and `decide`, what a rule set does with a packet (pure). `encode.rs` turns tables into one netlink batch (`batch`, `Load::Replace` swaps the whole ruleset in one transaction) and refuses what the kernel would; `decode.rs` reads the kernel's messages back (`read`). `apply` and `list` (Linux) talk to the kernel through `cf-netlink`. `BLOCKED` is the one list of private and special ranges, used by the guest's firewall and by the host's in front of a Linux machine. Tests: `tests/nft.rs` (bytes `nft` 1.0.6 wrote, recorded in `tests/nft/`), `tests/model.rs` (a model of the kernel's expressions against `decide`, round trips), `tests/kernel.rs` (Linux: applied and listed back in a fresh user and network namespace). | | `crates/netlink` (`cf-netlink`) | Netlink itself, for any protocol: `message`, `attribute`, `attribute_str` (a NUL-ended text), `nested`, `messages` and `attributes` to read them back, and on Linux the `Socket` that sends requests and reads every answer (`request`, `dump`), with its own `Error`. Users: `cf-nft` (nf_tables), `cf-vmm-ch` (rtnetlink). Tests in `tests/wire.rs`. | | `crates/vmm-ch` (`cf-vmm-ch`) | `ChVmm`: machines on Linux, cloud-hypervisor under KVM, one process per machine (`machine.rs`, Linux). Each machine has a cgroup of its own under the part of the tree delegated to the Runner (`cgroup.rs`: memory at its size and `overhead`, no swap (a kernel without a swap account per cgroup only on a host that swaps to nothing), `pids.max`, the cpuset of its cores, `cgroup.kill`, `wait_empty` woken by `cgroup.events`; `memory.current` is `Vm::resident`; `own` and `enter_host` for a test suite that shares its delegated cgroup), its vCPUs on the host's fast cores that it may use (`cpus.rs` with `cf_host_linux::fast_cpus`, `--host-cpus` overrides). With a network it runs under the system's pasta (`programs::pasta`: found on `PATH`, refused when another user could change it or what it links to, tried when the first machine with a network starts; `apt install passt`, 2023-03 or newer) in a user and network namespace; the hidden `vm-sandbox` subcommand of the binary (`sandbox.rs`) makes the guest's tap, loads `firewall.rs` onto it (netdev ingress: ARP, DHCP and DNS to pasta on the gateway, IPv4 to the internet; drops `BLOCKED`, 127/8 and every network of the host, IPv6 and everything else), bridges it with pasta's device (`link.rs`, rtnetlink), and starts the monitor without capabilities and without user namespaces in a PID, cgroup, mount and empty network namespace of its own (the tap's descriptor its only way out), no file it writes growing past the largest it is shown, its root a read-only tmpfs with only what `Sandbox::for_machine` shows it (its program, the guest's files, its console and folder, `/dev/kvm`, `/dev/urandom`) and a `/proc` of its own, then reports how it ended (`ending.rs`). `command.rs` builds both command lines, `events.rs` reads `--event-monitor`, `vsock.rs` the CONNECT handshake. cloud-hypervisor is pinned in `pins.rs` for each architecture, the host's fetched into `target/vmm-programs` and embedded by `build.rs` (Linux on aarch64 or x86_64), unpacked into `/programs` and checked before each machine (`programs.rs`). Tests: unit tests of everything pure, `tests/isolation.rs` (Linux: a stand-in guest on a veth pair behind the real pasta, guard and firewall; each blocked target a test of its own), `tests/monitor_root.rs` (Linux, no harness: a stand-in monitor in the real sandbox tries the host's files, processes and mounts, and under pasta tries to send past its tap), `tests/kvm.rs` (Linux with KVM, no harness: how a real machine ends, a panic, a stop, a kill, its cgroup's OOM and a dead monitor; it skips without `/dev/kvm` and the guest, `CF_REQUIRE_KVM=1` makes that a failure). | -| `crates/runner` (`codefusion-runner`) | The binary. `commands.rs`: `version` (the package's version and the commit, which `build.rs` reads from git or `CF_COMMIT`), `image verify`, `image install`, `pause`, `resume` and `status` (`holds.rs`: a named hold on the machine, `--wait` until the Runner saw it and runs nothing, exit 75 when the wait runs out; `status --json`), `vm` (the vm-host, `src/vm.rs`), `boot-proof`, `vmm` (Linux: the cloud-hypervisor the build carries, which `install` and an update ask of a program), and the service's commands in `service.rs` (macOS and Linux): `daemon` (what the service manager runs; ends on SIGTERM; one per state folder, held by a lock on `/runner.lock`), `enroll` (the console's one-time code on standard input, read with the terminal's echo off by `hidden.rs`, which gives the terminal back on every way out, a signal included; the token to `/console-token`, 0600, never printed), `check [--github]` (what an install needs, and the memory at that moment against the budget's floor with the sizes that fit; with `--github` what GitHub shows, changing nothing there and nothing in the state folder), `install` (copies the program to `/bin`, writes the service manager's entry, refuses while a machine stands), `uninstall [--leave] [--purge]` (refused while a Runner started by hand holds the state folder; a Mac without a usable token goes without telling the console). `config.rs` is the configuration file (refused when others can write it; `console` required unless the file names `app_dir` (absolute, not through `..`), the folder of a GitHub App key the host held before the console held it: while its `app.pem` is there `service.rs` `app_and_key` signs with it (`held_key`: `KeyTokens`, and `cf_daemon::HeldKey` for the service), else the GitHub tokens come from the console (`ConsoleTokens`); `check` and `uninstall` sign with a held key and never take it out (`looking`); `use` required on a Mac; on Linux `dedicated` or left out; `max_jobs`, `idle_minutes` (a workstation's only), `min_free_memory_percent` and `vcpus` (a size's vCPUs by its label, which `Settings::limited` checks against the CPUs CI may use), `ci_memory_gb` (the memory CI's machines hold at once, each with its overhead, in GB of 2^30 bytes) and `ci_cpus` (the CPUs they take together) are the owner's `Limits`, which `budget_of` and the service lay over the host's settings; `ConfigFile::change` writes the console's `settings` command into the file through `.json.next`, which must start a Runner before it replaces the file) and, on a Mac, the refusal of the cf-runners manager's own name and of external volumes; `manager.rs` is `ServiceManager`, the seam to the service manager; `launchd.rs` the agent `cc.codefusion.vm-runner` in the person's session (`plist`, `Agent`; `stop` waits past `ExitTimeOut` and decides by whether launchd still has the agent, not by what `bootout` says); `systemd.rs` (Linux) the user unit `codefusion-runner.service` (`unit`, with every path quoted for systemd or refused; `UserUnit` over an injected `systemctl --user`, the real one ended after 100 s; `stop` waits past `TimeoutStopSec` and decides by `ActiveState`); `linux.rs` the lines `check` adds on Linux (`Facts`, `lines`: KVM, lingering, delegated controllers, user namespaces, pasta, `noexec`, each with the root command that fixes it). `update.rs`: `CheckoutUpdate`, the `SelfUpdate` of a host that built the Runner, for the console's `update` and `update-to`: in the checkout the program was built from (`build.rs` writes it down, with `CARGO_HOME`, `RUSTUP_HOME` and `CARGO_BUILD_JOBS`), only when it is the config's `build_folder`, clean and at a commit (`own`), fetches `origin refs/heads/main` (a tag named `main` is not it), takes main's head or the commit `update-to` names when that resolves to one commit of the main fetched (`on_main`), refuses a target that does not contain the running commit (both looked up among the objects alone by `commit_named`, so a branch or tag named like a commit never stands for it), checks out the peeled commit, builds with `cargo xtask build` at low priority into `logs/update.log` and ends a build that takes longer than 45 minutes or outlasts the daemon, and keeps the program it checked as `bin/codefusion-runner.next`; `install` places that one, keeps the running program as `bin/codefusion-runner.prev` and writes `update.json` first; `guard`, run before the daemon reads its configuration, counts the new program's starts and puts the one before back after `MAX_STARTS`, and `ended_in_order` takes back the count of a start that ended as asked. `app_key.rs`: `take_out`: a file that still names `app_dir` has `/app.pem` deleted (a symlink: the link alone, never what it points to), `app_dir` taken out (the file without it checked as a configuration before anything is deleted, then written through `.json.next`) and the deleted file remembered first in `/app-key-file` (0600; a link's target on a second line; `remembered`), which the report's `appKeyFile` names while something is still there; refused, changing nothing, while the file names no console or the host holds no token for it (`KEPT_NO_CONSOLE`, `KEPT_NOT_ENROLLED`). A key deleted whose file could not be written is no failure: `TakenOut::app_dir_kept` says why the file still names the folder. `one_change` is held from the read to the write, by the take-out and by `ConfigFile::change` (the console's settings). `When::Answered` is the service's, once a scale set call of the console this host made was answered (`HeldKey::take_out`); `When::KeyGone` is what `daemon` runs (`start_on`) after `guard` and the state folder's lock (`only_runner`) and before it checks its configuration: it deletes nothing, and takes out only a folder whose key file this Runner's own take-out remembered; a key file missing otherwise keeps the folder (`NOT_TAKEN_OUT_HERE`), and the report's `appKeyKept` says so (`app_key_said`). `runner.entitlements` is signed into it. `config.rs`: `class` (`shared` or `trusted`) and `trusted` (the rules); `check_class` refuses a trusted host that would serve more than one job of one repository, or that holds the GitHub App's key (`app_dir`) or has no `console` to register through. | +| `crates/runner` (`codefusion-runner`) | The binary. `commands.rs`: `version` (the package's version and the commit, which `build.rs` reads from git or `CF_COMMIT`), `image verify`, `image install`, `pause`, `resume` and `status` (`holds.rs`: a named hold on the machine, `--wait` until the Runner saw it and runs nothing, exit 75 when the wait runs out; `status --json`), `vm` (the vm-host, `src/vm.rs`), `boot-proof`, `vmm` (Linux: the cloud-hypervisor the build carries, which `install` and an update ask of a program), and the service's commands in `service.rs` (macOS and Linux): `daemon` (what the service manager runs; ends on SIGTERM; one per state folder, held by a lock on `/runner.lock`), `enroll` (the console's one-time code on standard input, read with the terminal's echo off by `hidden.rs`, which gives the terminal back on every way out, a signal included; the token to `/console-token`, 0600, never printed), `check [--github]` (what an install needs, and the memory at that moment against the budget's floor with the sizes that fit; with `--github` what GitHub shows, changing nothing there and nothing in the state folder), `install` (copies the program to `/bin`, writes the service manager's entry, refuses while a machine stands), `uninstall [--leave] [--purge]` (refused while a Runner started by hand holds the state folder; a Mac without a usable token goes without telling the console). `config.rs` is the configuration file (refused when others can write it; `console` required unless the file names `app_dir` (absolute, not through `..`), the folder of a GitHub App key the host held before the console held it: while its `app.pem` is there `service.rs` `app_and_key` signs with it (`held_key`: `KeyTokens`, and `cf_daemon::HeldKey` for the service), else the GitHub tokens come from the console (`ConsoleTokens`); `check` and `uninstall` sign with a held key and never take it out (`looking`); `use` required on a Mac; on Linux `dedicated` or left out; `max_jobs`, `idle_minutes` (a workstation's only), `min_free_memory_percent` and `vcpus` (a size's vCPUs by its label, which `Settings::limited` checks against the CPUs CI may use), `ci_memory_gb` (the memory CI's machines hold at once, each with its overhead, in GB of 2^30 bytes) and `ci_cpus` (the CPUs they take together) are the owner's `Limits`, which `budget_of` and the service lay over the host's settings; `ConfigFile::change` writes the console's `settings` command into the file through `.json.next`, which must start a Runner before it replaces the file) and, on a Mac, the refusal of the cf-runners manager's own name and of external volumes; `manager.rs` is `ServiceManager`, the seam to the service manager; `launchd.rs` the agent `cc.codefusion.vm-runner` in the person's session (`plist`, `Agent`; `stop` waits past `ExitTimeOut` and decides by whether launchd still has the agent, not by what `bootout` says); `systemd.rs` (Linux) the user unit `codefusion-runner.service` (`unit`, with every path quoted for systemd or refused; `UserUnit` over an injected `systemctl --user`, the real one ended after 100 s; `stop` waits past `TimeoutStopSec` and decides by `ActiveState`); `linux.rs` the lines `check` adds on Linux (`Facts`, `lines`: KVM, lingering, delegated controllers, user namespaces, pasta, `noexec`, each with the root command that fixes it). `update.rs`: `CheckoutUpdate`, the `SelfUpdate` of a host that built the Runner, for the console's `update` and `update-to`: in the checkout the program was built from (`build.rs` writes it down, with `CARGO_HOME`, `RUSTUP_HOME` and `CARGO_BUILD_JOBS`), only when it is the config's `build_folder`, clean and at a commit (`own`), fetches `origin refs/heads/main` (a tag named `main` is not it), takes main's head or the commit `update-to` names when that resolves to one commit of the main fetched (`on_main`), refuses a target that does not contain the running commit (both looked up among the objects alone by `commit_named`, so a branch or tag named like a commit never stands for it), checks out the peeled commit, builds with `cargo xtask build` at low priority into `logs/update.log` and ends a build that takes longer than 45 minutes or outlasts the daemon, and keeps the program it checked as `bin/codefusion-runner.next`; `install` places that one, keeps the running program as `bin/codefusion-runner.prev` and writes `update.json` first; `guard`, run before the daemon reads its configuration, counts the new program's starts and puts the one before back after `MAX_STARTS`, and `ended_in_order` takes back the count of a start that ended as asked. `Installer` is what both updates share: `offer` asks a program its `version` (it must run here and name the commit) and keeps it as `.next`, `install` places it. `release_update.rs` (Linux and macOS builds; Linux hosts only by asset): `ReleaseUpdate`, the `SelfUpdate` of a Runner with no `build_folder`, installed from a release: `takes_head` is false, so it lists only `update-to`; `build` finds the commit's release (`runner--<12 hex>` by tag, else the newest page), waits `RELEASE_WAIT` for one not out yet, asked every `ASK_EVERY`, ended by `end`; refuses a draft, another publisher than `github-actions[bot]`, one not made at the whole commit, not on main or not newer than the running commit (`cf_ghapp::Releases::compare`); fetches `SHA256SUMS` and this host's asset (`asset_for_this_host`) into `.download` (0600), checks its SHA-256 (`sum_of`), one fetch at a time, and removes the download whatever happens. `service.rs` `self_update` picks `CheckoutUpdate` with a `build_folder`, else `ReleaseUpdate`. `app_key.rs`: `take_out`: a file that still names `app_dir` has `/app.pem` deleted (a symlink: the link alone, never what it points to), `app_dir` taken out (the file without it checked as a configuration before anything is deleted, then written through `.json.next`) and the deleted file remembered first in `/app-key-file` (0600; a link's target on a second line; `remembered`), which the report's `appKeyFile` names while something is still there; refused, changing nothing, while the file names no console or the host holds no token for it (`KEPT_NO_CONSOLE`, `KEPT_NOT_ENROLLED`). A key deleted whose file could not be written is no failure: `TakenOut::app_dir_kept` says why the file still names the folder. `one_change` is held from the read to the write, by the take-out and by `ConfigFile::change` (the console's settings). `When::Answered` is the service's, once a scale set call of the console this host made was answered (`HeldKey::take_out`); `When::KeyGone` is what `daemon` runs (`start_on`) after `guard` and the state folder's lock (`only_runner`) and before it checks its configuration: it deletes nothing, and takes out only a folder whose key file this Runner's own take-out remembered; a key file missing otherwise keeps the folder (`NOT_TAKEN_OUT_HERE`), and the report's `appKeyKept` says so (`app_key_said`). `runner.entitlements` is signed into it. `config.rs`: `class` (`shared` or `trusted`) and `trusted` (the rules); `check_class` refuses a trusted host that would serve more than one job of one repository, or that holds the GitHub App's key (`app_dir`) or has no `console` to register through. | | `crates/budget` (`cf-budget`) | Admission, with no I/O: `Budget` (`observe`, `claim`, `confirm`, `assign` and `withdraw` for a job GitHub assigned without asking (`assign_in_turn` and `confirm_in_turn` with its `Turn`: `line` puts main-branch and deploy jobs first, repositories in turn, none passed over past `DEFER_LIMIT`; `set_turn` when what a waiting job stands for changed; a `Turn::refusable` job, a test run, gets room only as a claim would, under the limits in force, and waiting is passed over without holding up those behind it; `make_room` holds back every other size, `making-room`, and puts the size first in line), `adopt`, `usage`, `release`, `hold`, and `status`, which names every slot with how it started and every waiting job with what keeps it), the pure `admit`, `Settings` (every number, `for_host` has the defaults; `limited` lays the owner's `Limits` over them: `max_machines`, refused past it with `Refusal::Machines`, `idle_after`, the floor as a share of memory, which lowers the cap, `ci_bytes`, the memory CI's machines hold at once with each one's overhead (`admit` counts them; `tests/ci_memory.rs`), and `ci_cpus`, the CPUs they take together; `for_headless` those of a Linux host nobody sits at, which reserves what the host needs, `HEADLESS_HOST_USE`, and caps the sizes so that each machine fits with its overhead; `on_fast_cores` gives CI only the fast kind of core of a host whose cores differ; `one_at_a_time` those of a trusted host, which runs one machine on every core it has), `Size` and its labels, the `Host` trait with `HostReading`, `Battery`, what a host's page shows of its battery, and `FakeHost`, the host tests script (`read_takes` lets a read take time before and after the reading gets its time; `tests/fake_host.rs`). Tests: `admission`, `assigned`, `headless`, `limits` (the rule at 4, 7.75, 8, 16 and 32 GiB and its boundaries), `levels`, `making_room`, `races`, `properties`, `turns`. | | `crates/clock` (`cf-clock`) | `now`: the two clocks the budget counts time by, read from the host (Linux: `CLOCK_MONOTONIC` and `CLOCK_BOOTTIME`; macOS: `CLOCK_UPTIME_RAW` and `CLOCK_MONOTONIC_RAW`). Both host crates' `Host::now`. | | `crates/host-macos` (`cf-host-macos`) | `MacHost` reads macOS behind `cf_budget::Host` (page counts, pressure, swap, idle time, thermal, power, disk; what the volume frees on demand, Time Machine's local snapshots and purgeable caches, is its capacity for important use beyond the free space, asked at most every `IMPORTANT_EVERY`, 30 s, for the budget's disk hold and never for an eviction); `Raw` and `reading` are the arithmetic: available memory is what is ready now (free, file cache, purgeable) and, while the Mac is calm (pressure normal, `kern.memorystatus_level` at least `CALM_LEVEL`, the compressor under `HEAVY_COMPRESSOR_SHARE` of memory), what compressing its inactive anonymous pages would free, never past the memorystatus level's share (`tests/reclaim.rs`). For the console's page: `boot_time`, `model` (chip and model, `model_name`), `battery` (what `pmset -g batt` prints, read by `battery_in`, as `cf_budget::Battery`), `host_name`, `load`, `cpu_percent`. `performance_cores` (`hw.perflevel0.logicalcpu` of a chip with two kinds of core, for the budget's CPUs). `program_of` says which program a process runs (the daemon's `is_ours`, `Holder`). `backup`: `exclude_from_backup` marks a folder the Runner made as skipped by Time Machine (the sticky exclusion `tmutil addexclusion` sets, an extended attribute), `exclude_with` is the decision behind the `Attributes` edge; the daemon, `enroll`, `install` and `image install` mark the state folder (the daemon's `Mac::keep_state`, a failure is a logged line, never fatal), and a cache folder to come goes through the same call when it is made. `examples/host-probe.rs` prints a Mac's readings. | | `crates/host-linux` (`cf-host-linux`) | `LinuxHost` reads Linux behind `cf_budget::Host`: MemAvailable, swap, swap-outs (`swapouts_of`: `pswpout`, but with zram in `/proc/swaps` only what is in use on swap that is not zram, since a page in zram stays in memory and PSI says when memory is short; `pswpout` again when `/proc/swaps` cannot be read), PSI (`/proc/pressure/memory`, Normal when the kernel has none), the thermal zones by their own trip points, `/sys/class/power_supply` (a host with no battery runs on the mains; a device's own battery, such as a mouse's, is left out unread), the state folder's disk by `statvfs`; idle time is none on a headless host. Every reader takes a root folder, so tests run against trees of files on any OS. `parse` reads each file, never more than `MAX_FILE_BYTES` (`counter` is one value of a `name value` file, also a cgroup's); `Raw` and `reading` are the arithmetic, with what each field means in the doc comment of `reading`. `fast_cpus` (the kind of core, by `midr_el1`, of the highest `cpu_capacity`, for `Settings::for_headless`). For the console's page: `boot_session` (`boot_id`), `boot_time`, `model` (device tree, else DMI), `battery`, `host_name`, `load`, `cpu_percent`. `process` and `program_of` say which program a process runs and when it started, read twice around the program so a number that went to another process in between names none; `started_at` moves with a step of the wall clock, `started_ticks` with `boot_session` does not. Tests: `fixtures.rs` (an RK3588 board's files, broken in every way), `parse.rs` (with properties), `real.rs` (Linux only: this machine against `free`, `df`, `getconf`). `examples/linux-probe.rs` prints a Linux host's readings and the headless budget. | | `crates/lifecycle` (`cf-lifecycle`) | `Slot`: the state machine of one slot and its job (events in, effects out, watchdogs, `saved` and `restore` for a daemon restart; `on` answers every phase and event with no catch-all arm, keeps a word of a job in every phase, and ends a slot as `no-job` only on `Event::Unregistered`, GitHub's 204; `Asking` is the phase of a slot that asks for its registration's removal, entered at the idle limit or on `Retire`; `Slot::hear` replays GitHub's stored word on a saved slot), and `StartBreaker` (`admit` before a claim, `holds` for the budget's hold). `tests/model.rs` searches every state it can reach. | | `crates/scaleset` (`cf-scaleset`) | The scale set protocol: `ScaleSetClient` (scale sets, sessions, one-job JIT configs, `generate_plain_jit_config` for one outside any scale set, removal of an Actions runner by id and by name (`remove_runner_named`); every administrator's call is the console's `github/scaleset/*`, which holds the Runners App's key and the service's admin token, except on a host that still holds the key (`with_key`, `holds_key`; `key.rs` `KeyAdmin`: its own registration token and admin token, the admin token kept until 120 s before it ends and replaced once on 401, and only the sets named `--…` listed), and on a host whose console is older than those calls (a 404 without a code: `KeyAdmin` with the console's `github/registration` token, `Registrar::Console`, until the Runner starts again; a Runner never needs a console newer than the one that runs; `tests/older_console.rs`); the session keeps the `serviceUrl` the console gave), `runner_busy`, the REST view of whether an Actions runner holds a job, `MessageSession` (long poll, ack, acquire), `listen` with the `Scaler` trait (handle, then ack; duplicates acked only; shutdown interrupts only the wait), `Backoff` and `supervise`. A token goes only to the API's origin and to hosts of `ACTIONS_HOST`: an address the service hands back that is elsewhere is refused. A registration whose answer was lost, or that names no runner, is removed again. Every status and malformed body is an `Error`. `examples/github_probe.rs` is the real check against a throwaway repository (prints no token). Tests in `tests/`, run against `cf-fake-github`; `github_rules.rs` holds the fake to what GitHub was seen to do (`docs/github-protocol.md`). | -| `crates/daemon` (`cf-daemon`) | The service. `holds.rs`: holds set on the machine itself, one file each in `/holds` (`set`, `lift`, `read`, which holds on a damaged file or an unreadable folder), and `/status.json` (`LocalStatus`: the holds the Runner saw, its machines and the ones coming, written on a change and every `STATUS_EVERY`); `Service::look_at_holds`, each tick, takes no new job while any hold or the console's pause is set and reports them as `status.holds`. `machines.rs`: `Machines` joins the budget, the lifecycle of each slot and the vm-hosts: its scale sets share the room their polls offer GitHub (`offer_room`: a free slot one poll offers no other does until that poll ends, `OFFER_STANDS` at most, and sets with jobs queued, `set_demand`, go before those without), so GitHub never assigns more jobs than it can start; a job GitHub assigned is counted (`assign`), started when memory allows (a machine counts with what its vm-host read it holds of the Mac's memory, `count_held`, its job's peak only when that is not known; the log says once when it is not, or is more than the machine is counted with, `Budget::host_bytes`), registered just in time, watched every tick, and ended once with an `EndKind` (`ends.rs`); `restore` adopts what a Runner before left from the records (`records.rs`, `DiskStore`; a record stores facts, `up`, `ran_job`, `completed`, `unregistered`, and carries the `RULES` it was written under; `Record::agrees` says its facts agree, and `Record::upgrade` is the one place that reads an older Runner's record: its beliefs cleared, GitHub's stored word replayed; a current record that disagrees is an error) and the vm-hosts that still answer, in the same start of the Mac (`kern.bootsessionuuid`, not the boot time, which moves with the clock); one that is silent or wrote nothing yet is killed, and so is what has no record. A tick looks at the vm-hosts without the machines' lock, so a vm-host that hangs keeps no listener waiting. What the polls in flight may still offer GitHub is kept (`Offer`): every answer a listener got since all bells last rang, since many scale sets of one size each ask at their own moment. The last slot taken rings their bells at once, room that went or came back with the Mac's memory at a look, at most every 5 s, so no poll in flight goes on offering a slot that is gone, or none while there is one. An open Mac whose other programs leave no memory for a size it serves says so in the log and is closed for `memory` in the console's report (`Snapshot::short_of`); one that would carry the size without its own machines is full, not short. `Snapshot::rooms` is what one more machine of each size meets, for the report's `sizes` and each scale set's capacity: how many fit, the budget's word when none does, `never` for a size the settings never run, and for memory what it needs and lacks. `making.rs`: `Making`, room made for a larger size. A job that waits for the room the Mac's own machines hold, all of them smaller, while the empty Mac would run it, has every other size held back until it starts, for `MAKE_ROOM_LIMIT` (30 min) at most and then none for as long; its scale set offers a place in line while GitHub has the job back, for `TAKEN_BACK_WAIT` (2 min); an empty Mac whose other programs hold the room gives up after `NO_ROOM_WAIT`. A size whose polls found no room for `STARVED_AFTER` (10 min) is offered a place before any of its jobs is assigned; a size the empty Mac cannot run never is. A headless host, one of a pool, makes room and offers a place only for the size the console's answer names it (`makeRoomFor`, `Making::name`, `Machines::make_room_for`), at once. `vms.rs`: `Vms`, the seam to `codefusion-runner vm`, and `VmHosts`, the real one. It writes each vm-host's process and start time to `run/.pid` (0600) before it hands it the job, so the Runner that starts next can end one that hangs before its first state. A signal goes only to a child whose exit was not collected, to a process that answers on the job's socket, or to one that runs the Runner's binary, is not the Runner and started no later than the time written down for it (`is_ours`, through `Platform::process`): a number that went to a later process is sent nothing. `service.rs`: `Service` (`discover` repositories and labels and, on a host that still holds the App's key (`HeldKey`), asks the console's scale set calls about each served repository in turn (the next after a refusal of one, none after a console that cannot be reached or has no such calls): once one answered, `HeldKey::take_out` and a start again, else the key serves and the report's `appKeyKept` says why, one supervised listener per scale set, `sweep` of registrations nothing stands behind, `report` to the console with the job events of the outbox and how long jobs handed here waited while nothing ran, with a WARN after `STALL_WARN_AFTER`; the sweep is the one place that removes a registration: those of slots in `Asking`, and of machines that ended while theirs stands, after `RESULT_WAIT` for one whose guest came up with no word of a job; each answer goes to `Machines::answered(runner, Answer)`: `Removed` (204) is no job, `Gone` (404) under a guest that came up is a job over, `Refused` (422) a job that runs, a job implied so is stored as inferred, `discover` and `look_at_queue` run in a task beside `run`'s loop, so a slow GitHub holds up no report and no stop; `look_at_queue`, about once a minute, serves the label sets of queued and running jobs that the default branches' workflows do not name, under `queued.rs`'s `Queued::select`: each for `queued_keep` (30 min) after a job of it was last seen, `queued_bounds` (8 a repository, 32 a host) the newest first, none of a repository it does not serve, none of a fork's pull request on a public repository, and none dropped while that repository's queue cannot be read (for up to `LONGEST_WAIT`, 24 h); a change lists the scale sets of only the repositories whose sets changed; `survey` for `check`, `retire` for `uninstall`, `run`). Each look at the queue also finds the jobs GitHub holds back from a scale set with room (`held.rs`, pure: queued two minutes while every poll of the set offered room beyond GitHub's assigned count for as long, `SetState::room_since`), names them in the report's `heldJobs`, and gives the set one Actions runner of its own at a time, one in five minutes at most and none while a queued job of the repository asks for only some of its labels (`held::narrower_waits`), in a slot the budget gives: registered outside the scale set (`SetRegistry::own`, scale set number `held::OWN`, 0) with exactly its labels and started by `scaler::start_runner`, as a scale set's are; it is retired once its set's queue holds no job it could take, or after five minutes. What the Runner believes about each machine and its job, which evidence moves it and what is written before what is done, is `docs/job-state.md`. `scaler.rs`: `Scaler` (reserve before acquire, release for jobs taken elsewhere, converge on assigned jobs, each place in line with the turn of the assigned job it stands for (`Slots::set_turn`), offers taken overdue first, then main and deploy jobs; `capacity` offers a paused or held Mac's polls only what runs and what was promised (`Slots::held`), and no room to a job in line `PLACE_LIMIT` (10 min) after it was first assigned here, taken back and handed here again or not (one away as long is a re-run, and new), so GitHub's take-back sends it to another machine, retire surplus after `IDLE_GRACE` (`Hosts::retire`: idle machines ask for their registration's removal, which the sweep does); before a session opens, `reconcile` asks GitHub's REST view of each Actions runner it believes waits, since messages go with a closed session, and a busy one took a job (`Answer::Busy`); a session opens only at start, after GitHub forgot one, after a listener failed, or after a restart), `Roster` (an Actions runner belongs to a repository's scale set: GitHub numbers scale sets within a repository, so the number alone names none), `turns.rs`: `turn_of`, whether a job is the main branch's or a deploy, from its workflow's ref, name and event. `names` (runner and scale set names, `ServePolicy`; a scale set's name ends in a hash of its repository and labels, so two repositories that ask for the same labels each get their own). `update.rs`: `SettingsFile` (what the Runner's configuration file says now against what it started with, `SettingsLook`; `change` writes the console's `settings` command, read by `SettingsChange::from_args`, which names each of the cf-runners manager's settings the Runner does not take; `Service::look_at_settings` takes a new one up as an update does, waiting in `drained`, the same wait, for no machine to stand, then starting again), `SelfUpdate` (`build` takes the commit `update-to` names, or none for main's head), `commit_from_args`, the console's `update-to` arguments (`{"commit"}` alone, `is_commit`: 7 to 40 lowercase hex characters), `Pending` in `update.json`, `at_start` (`Trying`, `GoBack` after `MAX_STARTS` starts that did not stay up for `STAYS_UP`, `Failed`, also when a program before could not be put back: `Pending::failed`); the service builds while jobs run, then holds new jobs until no machine stands and no slot promised to GitHub is left (`Machines::coming`), installs and ends (`update`, `restart`; one at a time, the result said once by the Runner that starts next). `platform/`: `host_service`, the service on a real host, which the binary and `cargo xtask it` both build, over a `Platform` (`This` is this build's): what differs between systems and nothing else: the host's reader and the budget's settings (`Settings::for_host` on the performance cores, `on_fast_cores` with `cf_host_macos::performance_cores`; on Linux `for_headless` with the fast cores), the boot session, host name, model and what the console's page shows (`extras`), `keep_state` (Time Machine; nothing on Linux), `fit` (the virtualization entitlement; on Linux `codefusion-runner vmm` says the build carries cloud-hypervisor), `process` (program and start, for `is_ours`; on Linux `cf_host_linux::process`), `program` (the program alone, for a kill's wait), `vm_hosts` and `SERVICE_LOG`. `mac.rs` is macOS, `linux.rs` Linux, where each vm-host starts in a systemd scope of its own (`VmHosts::in_scopes`, `systemd-run --user --scope -p Delegate=yes`, `codefusion-runner-.scope`), whose delegated cgroup it claims for its machines, and `user_bus_env` finds the user's systemd under `sudo -iu`, and not another user's that `su` left in the environment (`this_user` names the user from the user database). `tests/scopes.rs` (Linux, no harness) starts a stand-in vm-host so and checks it claims the scope's cgroup. `test_run/`: the console's `test-run` command (`TestRun::from_args`, its arguments checked, the run's `url` on the Runner's own console; `admit`, an urgent claim of the budget's as `test-run` that goes before the CI jobs in line, busy while the host takes no new job, a test run is on it or a Docker run finds the Docker jobs full; `wait_turn` for `waitS`, withdrawn as soon as the console's answer no longer carries the command), started by `Machines::start_test_run` as a machine with no Actions runner: its record is `unregistered` from the start with `test_run` set, its job starts when its guest is up and ends with the machine, and `Service::test_run` answers `started tr-…`, `busy: …` or `never: …`, later for a run that waits. `agent.mjs` is the workload (`workload()`: `node --input-type=module -e `, env `CF_RUN_URL`, `CF_RUN_TOKEN`, `CF_RUN_TIMEOUT_S`), the agent that fetches the bundles, runs the suite in its own process group, spools its output decoded as UTF-8 (U+FFFD for what is not) and sends it to the console; `agent.test.mjs` its `node --test` tests against a fake console (CI's Linux job). Tests: `tests/test_run.rs`, and in `tests/service.rs` the command end to end. `docker_check.rs`: whether Docker starts in this host's job machines (`DockerCheck`: `of_end` reads the check machine's end and its console log's "Docker sandbox ended" line, `due` at the start and every `RECHECK_AFTER` (6 h) while it fails, `None` from `of_end` for a machine that ended for something else (slept, evicted, stopped), checked again after `AGAIN_AFTER_NO_ANSWER` (10 min, `Machines::docker_check_due`), `refusal` while it failed; a failure is kept across starts by the `Store`, `docker_failure` and `keep_docker_failure`, in `DiskStore`'s `docker-check.failed`, a pass is not); `Machines::start_docker_probe` boots the check's machine in a claimed slot (no Actions runner, `Record::probe`, nothing reported), `Service::look_at_docker` on every tick, and a failed check drops the `cf-docker` sets (`size_here`) and answers a test run with Docker `never:`; tests `tests/docker_check.rs` and in `tests/service.rs`. `log.rs`: `Lines`, the log file that is rolled over. `testing` has the fakes, and `CrashingStore`, records at which a test makes the daemon die between any two writes. Tests: `machines.rs` (every end kind, a crash at every step), `records.rs` (every record main and earlier builds could have written, read by `Record::upgrade`), `interleavings.rs` (the property test), `schedules.rs` (the property test of the job facts against a GitHub that keeps the truth: crashes at any write, every answer to a removal, lost and late messages, records left in older formats; with pinned schedules), `offers.rs` (the property test of what several listeners of one size offer), `making_room.rs` (the mini's cf-6g job against the cf-2g ones that keep coming), `calm.rs` (the calm minute against time that passes within a look, and against everything that is really wrong), `service.rs` (GitHub's and the console's failures; the label sets of queued jobs: served, kept 30 min, bounded, refused, unreadable, 304, a fork's on a public repository; the property test that no job gets two machines, a session opens only for a cause, a job that waits while a machine fits moves at GitHub's next look, and every job that ran is heard, start and end once each, however many messages went with a session, with a stale or failing REST view, a crash between any two writes, five minutes that take an assignment back, removals that fail until GitHub recovers, and an uninstall in the middle; machines that boot slowly, so a start or an end comes before the Runner saw one up; no VM whose job runs is stopped, and every single write's facts agree (`CrashingStore::disagreements`); a second property aims the same checks at surplus removals), `scaler.rs`, `listening.rs`, `docker.rs` (a Docker job's daemon flag, the cap of `docker_max_jobs` in `Machines`, and `ServePolicy::serves_docker`: listed and private only). A trusted host (`ServePolicy::trusted`) makes no scale set: `Service::ask_trusted` asks for one Actions runner with the label `trusted` at a time (`trusted_every`, 15 s), in the largest machine it runs, whose `JobRequest::trusted` carries the rules; a 409 with `Retry-After` (the console's `no_trusted_job`) is a wait. It takes no test run, boots no Docker check and reports `class: trusted`. | -| `crates/ghapp` (`cf-ghapp`) | The GitHub App client. Its tokens come from a `TokenSource` (`source.rs`), the console's (`cf_console_client::ConsoleTokens`), through `GitHubApp::with_source`: no machine holds the App's key, a registration token or the Actions service's admin token, except one that still holds the key until the console's scale set calls answered it (`KeyTokens`, `key.rs`: the App JWT of `jwt.rs`, installation tokens and `registration_token`; `AppCredentials::load` in `credentials.rs`, `app.json` and `app.pem`, which must be 0600; tests in `tests/key.rs`); a call about a repository passes it, so a source whose installation IDs differ finds the one that covers it; `Monotonic` is the clock a token's lifetime is counted on. `rest.rs` is the REST plumbing both share (a 401 tells the source and tries once more). The clock skew is learned from the `Date` header (`clock.rs`). Repositories, workflow files (the caller names the largest file to read), repository variables, `waiting_jobs` (the jobs of a repository's queued and running runs of the label sets served, `cf_workflows::LabelSet::served`, from the REST view, each saying whether its run is a fork's, under bounds: `WAITING_RUNS` of the newest `RUNS_LISTED` runs, queued ones before those under way and each the oldest first, `JOBS_PAGE` jobs of one, `MAX_RUN_SETS` distinct sets of one run; the listings asked for again with the ETag of their last answer, so a 304 reuses it, and each run's jobs asked for again every look with theirs (a run's `updated_at` stays put while its jobs start and end), and `forget_queues` drops what repositories no longer served last read as; each set names its queued jobs, the oldest first), `generate_jit_config` (an Actions runner for one job outside any scale set, with exactly the labels given, sent once); `Http::new(timeout, retry)` with retries (429 and 5xx with jitter, `Retry-After`), HTTPS only (or this machine itself), redirects only to the same origin, at most `MAX_BODY_BYTES` of an answer and 100 pages of a listing. No error and no `Debug` carries an answer's text. Tests in `tests/app.rs`, run against `cf-fake-github`. | +| `crates/daemon` (`cf-daemon`) | The service. `holds.rs`: holds set on the machine itself, one file each in `/holds` (`set`, `lift`, `read`, which holds on a damaged file or an unreadable folder), and `/status.json` (`LocalStatus`: the holds the Runner saw, its machines and the ones coming, written on a change and every `STATUS_EVERY`); `Service::look_at_holds`, each tick, takes no new job while any hold or the console's pause is set and reports them as `status.holds`. `machines.rs`: `Machines` joins the budget, the lifecycle of each slot and the vm-hosts: its scale sets share the room their polls offer GitHub (`offer_room`: a free slot one poll offers no other does until that poll ends, `OFFER_STANDS` at most, and sets with jobs queued, `set_demand`, go before those without), so GitHub never assigns more jobs than it can start; a job GitHub assigned is counted (`assign`), started when memory allows (a machine counts with what its vm-host read it holds of the Mac's memory, `count_held`, its job's peak only when that is not known; the log says once when it is not, or is more than the machine is counted with, `Budget::host_bytes`), registered just in time, watched every tick, and ended once with an `EndKind` (`ends.rs`); `restore` adopts what a Runner before left from the records (`records.rs`, `DiskStore`; a record stores facts, `up`, `ran_job`, `completed`, `unregistered`, and carries the `RULES` it was written under; `Record::agrees` says its facts agree, and `Record::upgrade` is the one place that reads an older Runner's record: its beliefs cleared, GitHub's stored word replayed; a current record that disagrees is an error) and the vm-hosts that still answer, in the same start of the Mac (`kern.bootsessionuuid`, not the boot time, which moves with the clock); one that is silent or wrote nothing yet is killed, and so is what has no record. A tick looks at the vm-hosts without the machines' lock, so a vm-host that hangs keeps no listener waiting. What the polls in flight may still offer GitHub is kept (`Offer`): every answer a listener got since all bells last rang, since many scale sets of one size each ask at their own moment. The last slot taken rings their bells at once, room that went or came back with the Mac's memory at a look, at most every 5 s, so no poll in flight goes on offering a slot that is gone, or none while there is one. An open Mac whose other programs leave no memory for a size it serves says so in the log and is closed for `memory` in the console's report (`Snapshot::short_of`); one that would carry the size without its own machines is full, not short. `Snapshot::rooms` is what one more machine of each size meets, for the report's `sizes` and each scale set's capacity: how many fit, the budget's word when none does, `never` for a size the settings never run, and for memory what it needs and lacks. `making.rs`: `Making`, room made for a larger size. A job that waits for the room the Mac's own machines hold, all of them smaller, while the empty Mac would run it, has every other size held back until it starts, for `MAKE_ROOM_LIMIT` (30 min) at most and then none for as long; its scale set offers a place in line while GitHub has the job back, for `TAKEN_BACK_WAIT` (2 min); an empty Mac whose other programs hold the room gives up after `NO_ROOM_WAIT`. A size whose polls found no room for `STARVED_AFTER` (10 min) is offered a place before any of its jobs is assigned; a size the empty Mac cannot run never is. A headless host, one of a pool, makes room and offers a place only for the size the console's answer names it (`makeRoomFor`, `Making::name`, `Machines::make_room_for`), at once. `vms.rs`: `Vms`, the seam to `codefusion-runner vm`, and `VmHosts`, the real one. It writes each vm-host's process and start time to `run/.pid` (0600) before it hands it the job, so the Runner that starts next can end one that hangs before its first state. A signal goes only to a child whose exit was not collected, to a process that answers on the job's socket, or to one that runs the Runner's binary, is not the Runner and started no later than the time written down for it (`is_ours`, through `Platform::process`): a number that went to a later process is sent nothing. `service.rs`: `Service` (`discover` repositories and labels and, on a host that still holds the App's key (`HeldKey`), asks the console's scale set calls about each served repository in turn (the next after a refusal of one, none after a console that cannot be reached or has no such calls): once one answered, `HeldKey::take_out` and a start again, else the key serves and the report's `appKeyKept` says why, one supervised listener per scale set, `sweep` of registrations nothing stands behind, `report` to the console with the job events of the outbox and how long jobs handed here waited while nothing ran, with a WARN after `STALL_WARN_AFTER`; the sweep is the one place that removes a registration: those of slots in `Asking`, and of machines that ended while theirs stands, after `RESULT_WAIT` for one whose guest came up with no word of a job; each answer goes to `Machines::answered(runner, Answer)`: `Removed` (204) is no job, `Gone` (404) under a guest that came up is a job over, `Refused` (422) a job that runs, a job implied so is stored as inferred, `discover` and `look_at_queue` run in a task beside `run`'s loop, so a slow GitHub holds up no report and no stop; `look_at_queue`, about once a minute, serves the label sets of queued and running jobs that the default branches' workflows do not name, under `queued.rs`'s `Queued::select`: each for `queued_keep` (30 min) after a job of it was last seen, `queued_bounds` (8 a repository, 32 a host) the newest first, none of a repository it does not serve, none of a fork's pull request on a public repository, and none dropped while that repository's queue cannot be read (for up to `LONGEST_WAIT`, 24 h); a change lists the scale sets of only the repositories whose sets changed; `survey` for `check`, `retire` for `uninstall`, `run`). Each look at the queue also finds the jobs GitHub holds back from a scale set with room (`held.rs`, pure: queued two minutes while every poll of the set offered room beyond GitHub's assigned count for as long, `SetState::room_since`), names them in the report's `heldJobs`, and gives the set one Actions runner of its own at a time, one in five minutes at most and none while a queued job of the repository asks for only some of its labels (`held::narrower_waits`), in a slot the budget gives: registered outside the scale set (`SetRegistry::own`, scale set number `held::OWN`, 0) with exactly its labels and started by `scaler::start_runner`, as a scale set's are; it is retired once its set's queue holds no job it could take, or after five minutes. What the Runner believes about each machine and its job, which evidence moves it and what is written before what is done, is `docs/job-state.md`. `scaler.rs`: `Scaler` (reserve before acquire, release for jobs taken elsewhere, converge on assigned jobs, each place in line with the turn of the assigned job it stands for (`Slots::set_turn`), offers taken overdue first, then main and deploy jobs; `capacity` offers a paused or held Mac's polls only what runs and what was promised (`Slots::held`), and no room to a job in line `PLACE_LIMIT` (10 min) after it was first assigned here, taken back and handed here again or not (one away as long is a re-run, and new), so GitHub's take-back sends it to another machine, retire surplus after `IDLE_GRACE` (`Hosts::retire`: idle machines ask for their registration's removal, which the sweep does); before a session opens, `reconcile` asks GitHub's REST view of each Actions runner it believes waits, since messages go with a closed session, and a busy one took a job (`Answer::Busy`); a session opens only at start, after GitHub forgot one, after a listener failed, or after a restart), `Roster` (an Actions runner belongs to a repository's scale set: GitHub numbers scale sets within a repository, so the number alone names none), `turns.rs`: `turn_of`, whether a job is the main branch's or a deploy, from its workflow's ref, name and event. `names` (runner and scale set names, `ServePolicy`; a scale set's name ends in a hash of its repository and labels, so two repositories that ask for the same labels each get their own). `update.rs`: `SettingsFile` (what the Runner's configuration file says now against what it started with, `SettingsLook`; `change` writes the console's `settings` command, read by `SettingsChange::from_args`, which names each of the cf-runners manager's settings the Runner does not take; `Service::look_at_settings` takes a new one up as an update does, waiting in `drained`, the same wait, for no machine to stand, then starting again), `SelfUpdate` (`build` takes the commit `update-to` names, or none for main's head; `takes_head` false lists no plain `update` and refuses one with `NO_HEAD`; `end` stops a build or a wait when the daemon ends), `commit_from_args`, the console's `update-to` arguments (`{"commit"}` alone, `is_commit`: 7 to 40 lowercase hex characters), `Pending` in `update.json`, `at_start` (`Trying`, `GoBack` after `MAX_STARTS` starts that did not stay up for `STAYS_UP`, `Failed`, also when a program before could not be put back: `Pending::failed`); the service builds while jobs run, then holds new jobs until no machine stands and no slot promised to GitHub is left (`Machines::coming`), installs and ends (`update`, `restart`; one at a time, the result said once by the Runner that starts next). `platform/`: `host_service`, the service on a real host, which the binary and `cargo xtask it` both build, over a `Platform` (`This` is this build's): what differs between systems and nothing else: the host's reader and the budget's settings (`Settings::for_host` on the performance cores, `on_fast_cores` with `cf_host_macos::performance_cores`; on Linux `for_headless` with the fast cores), the boot session, host name, model and what the console's page shows (`extras`), `keep_state` (Time Machine; nothing on Linux), `fit` (the virtualization entitlement; on Linux `codefusion-runner vmm` says the build carries cloud-hypervisor), `process` (program and start, for `is_ours`; on Linux `cf_host_linux::process`), `program` (the program alone, for a kill's wait), `vm_hosts` and `SERVICE_LOG`. `mac.rs` is macOS, `linux.rs` Linux, where each vm-host starts in a systemd scope of its own (`VmHosts::in_scopes`, `systemd-run --user --scope -p Delegate=yes`, `codefusion-runner-.scope`), whose delegated cgroup it claims for its machines, and `user_bus_env` finds the user's systemd under `sudo -iu`, and not another user's that `su` left in the environment (`this_user` names the user from the user database). `tests/scopes.rs` (Linux, no harness) starts a stand-in vm-host so and checks it claims the scope's cgroup. `test_run/`: the console's `test-run` command (`TestRun::from_args`, its arguments checked, the run's `url` on the Runner's own console; `admit`, an urgent claim of the budget's as `test-run` that goes before the CI jobs in line, busy while the host takes no new job, a test run is on it or a Docker run finds the Docker jobs full; `wait_turn` for `waitS`, withdrawn as soon as the console's answer no longer carries the command), started by `Machines::start_test_run` as a machine with no Actions runner: its record is `unregistered` from the start with `test_run` set, its job starts when its guest is up and ends with the machine, and `Service::test_run` answers `started tr-…`, `busy: …` or `never: …`, later for a run that waits. `agent.mjs` is the workload (`workload()`: `node --input-type=module -e `, env `CF_RUN_URL`, `CF_RUN_TOKEN`, `CF_RUN_TIMEOUT_S`), the agent that fetches the bundles, runs the suite in its own process group, spools its output decoded as UTF-8 (U+FFFD for what is not) and sends it to the console; `agent.test.mjs` its `node --test` tests against a fake console (CI's Linux job). Tests: `tests/test_run.rs`, and in `tests/service.rs` the command end to end. `docker_check.rs`: whether Docker starts in this host's job machines (`DockerCheck`: `of_end` reads the check machine's end and its console log's "Docker sandbox ended" line, `due` at the start and every `RECHECK_AFTER` (6 h) while it fails, `None` from `of_end` for a machine that ended for something else (slept, evicted, stopped), checked again after `AGAIN_AFTER_NO_ANSWER` (10 min, `Machines::docker_check_due`), `refusal` while it failed; a failure is kept across starts by the `Store`, `docker_failure` and `keep_docker_failure`, in `DiskStore`'s `docker-check.failed`, a pass is not); `Machines::start_docker_probe` boots the check's machine in a claimed slot (no Actions runner, `Record::probe`, nothing reported), `Service::look_at_docker` on every tick, and a failed check drops the `cf-docker` sets (`size_here`) and answers a test run with Docker `never:`; tests `tests/docker_check.rs` and in `tests/service.rs`. `log.rs`: `Lines`, the log file that is rolled over. `testing` has the fakes, and `CrashingStore`, records at which a test makes the daemon die between any two writes. Tests: `machines.rs` (every end kind, a crash at every step), `records.rs` (every record main and earlier builds could have written, read by `Record::upgrade`), `interleavings.rs` (the property test), `schedules.rs` (the property test of the job facts against a GitHub that keeps the truth: crashes at any write, every answer to a removal, lost and late messages, records left in older formats; with pinned schedules), `offers.rs` (the property test of what several listeners of one size offer), `making_room.rs` (the mini's cf-6g job against the cf-2g ones that keep coming), `calm.rs` (the calm minute against time that passes within a look, and against everything that is really wrong), `service.rs` (GitHub's and the console's failures; the label sets of queued jobs: served, kept 30 min, bounded, refused, unreadable, 304, a fork's on a public repository; the property test that no job gets two machines, a session opens only for a cause, a job that waits while a machine fits moves at GitHub's next look, and every job that ran is heard, start and end once each, however many messages went with a session, with a stale or failing REST view, a crash between any two writes, five minutes that take an assignment back, removals that fail until GitHub recovers, and an uninstall in the middle; machines that boot slowly, so a start or an end comes before the Runner saw one up; no VM whose job runs is stopped, and every single write's facts agree (`CrashingStore::disagreements`); a second property aims the same checks at surplus removals), `scaler.rs`, `listening.rs`, `docker.rs` (a Docker job's daemon flag, the cap of `docker_max_jobs` in `Machines`, and `ServePolicy::serves_docker`: listed and private only). A trusted host (`ServePolicy::trusted`) makes no scale set: `Service::ask_trusted` asks for one Actions runner with the label `trusted` at a time (`trusted_every`, 15 s), in the largest machine it runs, whose `JobRequest::trusted` carries the rules; a 409 with `Retry-After` (the console's `no_trusted_job`) is a wait. It takes no test run, boots no Docker check and reports `class: trusted`. | +| `crates/ghapp` (`cf-ghapp`) | `release.rs`: `Releases`, a public repository's releases read with no token (`by_tag`, `page`, `branch_head`, `compare`), and `download`, a release's file streamed into a `Sink` and hashed as it is written, followed across GitHub's redirect to its storage with no token, begun again from the first byte after a break, a 429 or a 5xx, ended at once by a full disk (`Error::Local`), a size over its limit or the stop flag (`tests/release.rs`). The GitHub App client. Its tokens come from a `TokenSource` (`source.rs`), the console's (`cf_console_client::ConsoleTokens`), through `GitHubApp::with_source`: no machine holds the App's key, a registration token or the Actions service's admin token, except one that still holds the key until the console's scale set calls answered it (`KeyTokens`, `key.rs`: the App JWT of `jwt.rs`, installation tokens and `registration_token`; `AppCredentials::load` in `credentials.rs`, `app.json` and `app.pem`, which must be 0600; tests in `tests/key.rs`); a call about a repository passes it, so a source whose installation IDs differ finds the one that covers it; `Monotonic` is the clock a token's lifetime is counted on. `rest.rs` is the REST plumbing both share (a 401 tells the source and tries once more). The clock skew is learned from the `Date` header (`clock.rs`). Repositories, workflow files (the caller names the largest file to read), repository variables, `waiting_jobs` (the jobs of a repository's queued and running runs of the label sets served, `cf_workflows::LabelSet::served`, from the REST view, each saying whether its run is a fork's, under bounds: `WAITING_RUNS` of the newest `RUNS_LISTED` runs, queued ones before those under way and each the oldest first, `JOBS_PAGE` jobs of one, `MAX_RUN_SETS` distinct sets of one run; the listings asked for again with the ETag of their last answer, so a 304 reuses it, and each run's jobs asked for again every look with theirs (a run's `updated_at` stays put while its jobs start and end), and `forget_queues` drops what repositories no longer served last read as; each set names its queued jobs, the oldest first), `generate_jit_config` (an Actions runner for one job outside any scale set, with exactly the labels given, sent once); `Http::new(timeout, retry)` with retries (429 and 5xx with jitter, `Retry-After`), HTTPS only (or this machine itself), redirects only to the same origin, at most `MAX_BODY_BYTES` of an answer and 100 pages of a listing. No error and no `Debug` carries an answer's text. Tests in `tests/app.rs`, run against `cf-fake-github`. | | `crates/workflows` (`cf-workflows`) | `self_hosted` / `self_hosted_with`: the label sets a repository's workflows ask for (port of the Go `workflows` package, tests in `tests/labels.rs`), `size_gb` for the `cf-2g/4g/6g/8g` labels, `self-hosted` = 4 GB, `DOCKER_LABEL` (`cf-docker`, served alone or with a size), `is_host_label` and `LabelSet::host_labels` (`cf-macos` and the like, served only by hosts whose config `labels` has them, and a host whose `only_labels` names some of them serves only the sets that ask for one: `ServePolicy::serves_labels` and `labels_refusal` in `crates/daemon/src/names.rs`), the `vars.*` expressions evaluated with the repository's variables by `expression.rs`, a reader of GitHub's expressions over variables and literals: `||`, parentheses, `format` (`{0}`, `{{`/`}}` escapes), `fromJSON` and `toJSON`, nested at most `MAX_DEPTH` deep, no value over `MAX_VALUE_BYTES` (a JSON list under `fromJSON`, with no `toJSON` anywhere in the expression, is several labels; what it cannot read stands for the pool label), `LabelSet::served`, the rules every set to serve keeps, a workflow's or a queued job's (one label of ours, no other platform, at most `MAX_SET_LABELS` labels of at most `MAX_LABEL_BYTES`, letters, digits, `-`, `_` and `.` only), and `MAX_FILE_BYTES`, the largest workflow file read. `LabelSet::served` refuses every set with `TRUSTED_LABEL` in any spelling GitHub's case-blind match takes for it (`is_trusted_label`: `TRUSTED`, `truſted`), from a list or a variable: no shared host takes a trusted job. | | `crates/secret` (`cf-secret`) | `Secret`: a value whose `Debug` prints `Secret(..)`. Every token and key is wrapped in it. `replace_private`: a file for this user alone (0600), put in place in one step; `replace_private_as` with a mode of the user's own bits (0500 for a program). `private_dir`: a folder for this user alone (0700). | | `crates/fake-github` (`cf-fake-github`) | A stateful fake GitHub over real sockets, for tests only: App JWT checks (`test_key`), installation tokens, repositories, scale sets, sessions, long poll, acquire, JIT configs, with `Options::auto_assign` (no kinder than the real service, each rule beside its evidence in `docs/github-protocol.md`: jobs go out only when a job is queued, ends, is cancelled or taken, or a test says GitHub looks (`github_looks`) or an Actions runner registered outside any scale set comes online (`runner_online`; `plain_runners` lists them, and `start_job` gives one the oldest queued job of its repository whose labels it has), against the smaller room of a scale set's last two polls; an opening hands out nothing; the counts never name a held job; a session that ends takes its pending messages; `offers_room` says when a job of a repository queued now goes out at once, `offers_room_for` when a job that waits goes out at GitHub's next look; only a session's newest poll is served), an assignment no Actions runner took is taken back after `TAKE_BACK_AFTER` on the queue's clock (`pass`), with faults (`fail`, `before`, `lose_answer`, `lose_session`, `expire_session_token`, `revoke_admin_tokens`, `send_the_service_to`, `send_the_queue_to`, clock offset, `set_rest_stale`: the REST view of an Actions runner one read behind, `set_down`: every request dropped, as for a client that died), and `assignments`, the hand-outs so far. What the console does with its own App's key, for `cf-fake-console`: `mint_read_token` (a token GitHub refuses every write, 403), `mint_registration_token`, `mint_plain_jit`, `revoke_token`, `token_works`, `revoke_installation_tokens`. `TestClock` is the wall clock and the monotonic one. `queue_job_at` queues a job at a ref, which its message's `jobWorkflowRef` and `eventName` show; `queue_run` the jobs of one run; `end_job` ends one as its run shows it; `from_fork` makes a job's run a pull request's from a fork. The REST view of runs and their jobs (`/actions/runs?status=`, `/actions/runs/{id}/jobs`) answers with an ETag and 304 to a request that names it (`not_modified` counts them). | | `crates/harness` (`cf-harness`) | What every crate's tests start from (dev-dependency only): `cf-fake-github` behind `cf-fake-console`, with the real clients in front of them as a machine has them (`enrolled`, `app`, `app_on`: `ConsoleTokens` and `ScaleSetClient`), `TestClock`, `fast` retries, `until`, `elsewhere` (a host no token may reach), and what a test asserts no machine did: `registrations` (requests for a registration token or traded one) and `admin_requests_from_machines` (service calls not made by the console, `CONSOLE_MARK`); `key_tokens` and `key_app` for a host that still holds the test key. | | `crates/console-client` (`cf-console-client`) | The client of the CodeFusion Console's runner protocol 1 (`/agent/runners/{enroll,report,leave}`): the wire types (`Enroll` and `Report` with the system they run on (`Os`, `macos` or `linux`, left out when none), `Status` with `commands` (the console's commands this Runner does: `pause` and `resume` (the answer's `paused`) on every system, `update` and `update-to` only with somewhere to update from, `test-run` with a console), `memoryForJobsBytes` (the cap of the limits in force), `idleWithJobsWaitingSeconds` and its reason, `heldJobs` (`HeldJobs`: per repository and label set, the jobs GitHub holds back from a scale set with room and the oldest one's wait, left out when none), `appKeyFile` (a GitHub App key file the host still holds, which `app_key::take_out` could not delete or found linked, or the one it signs with, left out when none), `appKeyKept` (why it still signs with it), `docker` (`DockerStatus`: the last Docker check, `ok`, `checkedAt`, `why`, left out before the first) and `sizes`, what one more machine of each size meets: `free` (-1 for a size the host never runs), `why`, `needBytes`, `shortBytes`, `JobLimits`, the report's `limits` that the `settings` command changes, `RunnerStatus` with its machine's `size` and `countedBytes` (the size the budget's cap counts it with; the vm-hosts' overhead is taken out of the cap), `JobEvent`, `Answer` (with `makeRoomFor`, the size the console names a pool host to make room for), `Command` ...), `enroll`, `Client` (`report`, `deliver`, `leave`; the token is a `Secret`), `Error` (`Unauthorized`, `Refused`, `GithubOff`, `Busy`, `Unavailable`, `Protocol`), GitHub through the console (`github.rs`: `github_read`, `github_jit`, each answer checked: tokens of visible ASCII, lifetimes of at least `MIN_TOKEN_LIFE` and held at most `MAX_TOKEN_LIFE`, unique installation IDs; what the console would refuse is refused unsent; a JIT runner is asked for once and never again by itself; `github_read` names at most `REFUSED_MAX` refused tokens; a 403 with the code `BROKER_OFF` is `Error::GithubOff`, a 429 or 5xx with Retry-After `Error::Busy`, held to `RETRY_AFTER_MAX`), `ConsoleTokens` (`tokens.rs`, the `TokenSource` of the App client: one read for all installations, renewed at half its life and not used `END_MARGIN` before its end on the monotonic clock, concurrent callers sharing one request; a console that fails is asked again after `FAILED_WAIT` doubling to `FAILED_WAIT_MAX` while the last token serves to its end, then every call fails at once with the reason, never asking sooner than the console's Retry-After; a token GitHub refuses is named in the next read so the console replaces it while the other installations' tokens serve, and one handed out again all the same, or new ones GitHub refuses too, are asked for again after a doubling wait; a runner the console answered with a wait is not asked for that repository until it passes (a 429 unsent); only the 403 `broker_off` (the owner's switch for the machine off) fails every call with the console's words for `GATE_RECHECK_ALONE`, and any other 403 is an error with the console's words; installation IDs are mapped by repository, and looked up again after a 404), the scale set calls (`scaleset.rs`: `scaleset_list`, `scaleset_create`, `scaleset_delete`, `scaleset_session_open`, `scaleset_session_refresh`, `scaleset_session_close`, `scaleset_jit`, `scaleset_remove`, each answer checked, the console's refusals by their codes: `not_yours`, `session_exists`, `session_lost`, `name_taken`, `runner_busy`, `rate_limited`; a 503 `github_busy` with its Retry-After is `Error::Busy`) and `Outbox`, the job events that wait for the console in one 0600 file: ids dedup (waiting, and the last 1000 sent), at most 500 kept with the oldest dropped and counted, 100 a report, marked sent only when a valid answer came. A report the console would refuse is refused here without sending. Tests: `tests/contract.rs` (every bound of the protocol against the fake), `client.rs`, `outbox.rs`, `delivery.rs`, `properties.rs`, `tokens.rs` (GitHub through the console end to end, with both fakes). | | `crates/fake-console` (`cf-fake-console`) | A fake CodeFusion Console for tests (dev-dependency only), over `cf_fake_github::httpd`: the checks of the console's `agent.ts` and `protocol.ts` in the same order with the same statuses and bounds (`schema.rs` is protocol.ts field by field), codes, tokens, commands, pause. `withdraw_command` (the console stops sending one). GitHub through the console (`github.rs`, the console's `github.ts`): `serve_github` attaches a `FakeGithub` that makes the tokens, `set_broker` is the owner's switch per machine (off at first; off forgets the kept tokens and revokes the read ones), `scaleset_answer` answers every scale set call with one reply (`no_scaleset_calls`: a console from before them), `set_installations` gives the console's App other installation IDs; kept tokens handed out again while more than `REUSE_MIN_S` is left, a read token named `refused` replaced and revoked (`ignore_refused`: a console that does not read the names), the switch's 403 with the code `broker_off`, `JIT_PER_REPO` and `JIT_PER_MACHINE` runners in `JIT_WINDOW_S` then 429 with Retry-After, a runner name not the machine's 403 without a code, before the Runners App is looked for. The scale set routes (`scaleset.rs`, `github/scaleset/{list,create,delete,session/open,session/refresh,session/close,jit,remove}`): the admin token made in-process and kept while more than `ADMIN_REUSE_S` is left (`forget_admin_tokens`), each call scoped to the machine's own scale sets and Actions runners (403 `not_yours`), rate limits per machine in `RATE_WINDOW_S`, GitHub's 5xx, 429, a drop or a timeout (`GITHUB_DEADLINE`) a 503 `github_busy` with Retry-After (30 s, or `github_busy_wait` for tests), and an audit row per call (`scaleset_audit`). Faults: `down`, `reports_down` (everything but `github/…`), `fail_next`, `fail_next_waiting` (with Retry-After), `drop_before_next`, `accept_then_drop_next`, `accept_then_drop_next_report`, `answer_next` (any answer to the next `github/…` call). What it saw: `reports`, `job_events`, `enrollments`, `tokens`, `calls`, `github_calls`, and `os_of`, the system the console keeps for a machine (`macos`, `linux`, none for anything else, which is never refused). Written from the console's code and not from the client's types. | -| `scripts` | `host-setup.sh`: a Linux host made a Runner of the console, unattended, as root, with no App key: `install` (packages, the Runner's user, user namespaces, the Runner built from its checkout or brought as `--binary`/`--version` with its `--sha256`, the newest image, the configuration with `--settings`; needs no code, runs any number of times, ends on `host-setup: changed` or `unchanged`, `--dry-run` says each change and a diff of the configuration; a new Runner on an enrolled host goes in under the hold `host-setup` once running jobs end, exit 75 when they outlast the wait), `enroll --code-file` (the enrolment and the service), or both at once as a cloud VM's startup script does; `merged_config` keeps the keys the Runner and the console set and takes out the ones a last `--settings` set and this one leaves out; `--class trusted` writes `"class": "trusted"` and no rules, so the Runner's defaults apply; `--sandbox container` needs no KVM, sets up the Runner user's rootless Podman (`place_engine`: subordinate ids, `podman.socket`), installs the newest container-image release into it and writes `sandbox` and `engine`; its reads of GitHub's API (`github_get`) send `GITHUB_TOKEN` when set and wait out a rate limit, a 5xx or no answer (`xtask/tests/host_setup.rs`, with the merge and the refusal of each option a step does not take); CI's `fresh-host` job runs it against the fakes, both at once from the source and `install`, again, `--dry-run` and `enroll` with a build brought along. `docker-rollout-mac.sh`: on one Mac, installs an image release and names it and the Docker repositories in the Runner's configuration file, then waits for the Runner to start again on it (or says why it did not). | +| `scripts` | `host-setup.sh`: a Linux host made a Runner of the console, unattended, as root, with no App key: `install` (packages, the Runner's user, user namespaces, the Runner built from its checkout or brought as `--binary`/`--version` with its `--sha256`, the newest image, the configuration with `--settings`; needs no code, runs any number of times, ends on `host-setup: changed` or `unchanged`, `--dry-run` says each change and a diff of the configuration; a new Runner on an enrolled host goes in under the hold `host-setup` once running jobs end, exit 75 when they outlast the wait), `enroll --code-file` (the enrolment and the service), or both at once as a cloud VM's startup script does; `merged_config` keeps the keys the Runner and the console set and takes out the ones a last `--settings` set and this one leaves out; `--class trusted` writes `"class": "trusted"` and no rules, so the Runner's defaults apply; `--sandbox container` needs no KVM, sets up the Runner user's rootless Podman (`place_engine`: subordinate ids, `podman.socket`), installs the newest container-image release into it and writes `sandbox` and `engine`; a `--version` is a minimum: `runs_newer` keeps an enrolled host's Runner that runs a newer commit of main (GitHub's compare), and keeps it when GitHub cannot say; its reads of GitHub's API (`github_get`) send `GITHUB_TOKEN` when set and wait out a rate limit, a 5xx or no answer (`xtask/tests/host_setup.rs`, with the merge and the refusal of each option a step does not take); CI's `fresh-host` job runs it against the fakes, both at once from the source and `install`, again, `--dry-run` and `enroll` with a build brought along. `release-gate.sh`: whether "Runner release" may put out a commit: on main, its push's newest CI, Image (when there is one) and Runner build runs green; `publish `, `wait`, `skip`, or a failure when GitHub cannot say (tests in `xtask/tests/host_setup.rs`). `docker-rollout-mac.sh`: on one Mac, installs an image release and names it and the Docker repositories in the Runner's configuration file, then waits for the Runner to start again on it (or says why it did not). | | `xtask` | `cargo xtask keygen | guest | release | build | it | fakes serve | kernel check | cloud-hypervisor [check]`. `fakes serve` (`xtask/src/fakes.rs`) runs the fake GitHub and console for `scripts/host-setup.sh` and ends once the host it set up serves. `it` (`xtask/src/it.rs`) runs the vm-host against real machines on this host, one small machine at a time, and writes its numbers to `target/it/report.txt`. Its scenarios: `boot`, `lifecycle`, `exit-code`, `oom`, `timeout`, `stop`, `sigterm`, `sigkill`, `firewall` (what a job reaches), `inside` (what a job's root tries against its own machine), `docker` (a `cf-docker` job: Compose, a build, a published port, and what its containers reach, privileged ones too, with a port open on the host as the target that shows the firewall at work), `docker-oom` (a container that takes all the job's memory ends the job `oom`; one stopped at its own limit, in `docker`, does not), `launchd`, `timers`, `full-memory` (what a machine whose job holds all its memory costs the host, against `Settings::VM_HOST_OVERHEAD`, and that its vm-host reads what its machine holds), and in `it/service.rs` `service` (three jobs through the whole service against the fake GitHub and console: to its end, across a Runner that dies, into an uninstall) and `commands` (the binary's `enroll`, `check --github` and `daemon`). A scenario that needs more memory than the host has free says `SKIPPED` and why. Kernel package pinned by version, size and SHA-256 in `xtask/src/kernel.rs`, read from our copy (the release `kernel-linux-virt-`) and from Alpine only when that is missing; `kernel check` goes red when our copy does not give the pinned bytes and warns when Alpine has a newer kernel. How to move the pin is at the top of `kernel.rs`. `cloud_hypervisor.rs` does the same for the Linux monitor pinned in `crates/vmm-ch/src/pins.rs` (our copy: the release `cloud-hypervisor-`), and `pinned.rs` is what both share: the HTTPS download that checks each redirect's host and the bytes, the check of our copy, and `newest`, the version a release or index names. | | `image/Dockerfile` | The root filesystem a job runs in: GitHub's Actions runner image plus tools, for arm64 and amd64 (`TARGETARCH`, each download pinned for each). For Docker jobs: iptables, pasta and the Compose plugin (the engine, CLI and buildx come with the Actions runner image). Packed by `.github/workflows/image.yml`. | -| `.github/workflows` | `ci.yml` (macos-15: fmt, guest, clippy, tests, release build; ubuntu-24.04-arm and ubuntu-24.04: clippy and tests of the crates in `LINUX_CRATES`, clippy for the guest's musl target of that architecture, and the Linux probe; ubuntu: cargo deny), `image.yml` (the image of each architecture, and its container-image release beside it: built and verified on every change to it, signed and published from main, weekly), `pins.yml` (`cargo xtask kernel check` and `cargo xtask cloud-hypervisor check`, weekly and on every change to a pin), `release.yml` (the Runner's Linux program of each architecture, built in Debian 12 from nothing on each change of main to it, released as `runner-` with the `host-setup.sh` of its commit and their SHA256SUMS, never replaced). Labelled pull requests are merged by CodeFusion Console. | +| `.github/workflows` | `ci.yml` (macos-15: fmt, guest, clippy, tests, release build; ubuntu-24.04-arm and ubuntu-24.04: clippy and tests of the crates in `LINUX_CRATES`, clippy for the guest's musl target of that architecture, and the Linux probe; ubuntu: cargo deny), `image.yml` (the image of each architecture, and its container-image release beside it: built and verified on every change to it, signed and published from main, weekly), `pins.yml` (`cargo xtask kernel check` and `cargo xtask cloud-hypervisor check`, weekly and on every change to a pin), `runner-build.yml` (the Runner's Linux program of each architecture, built in Debian 12 from nothing on every push to main, kept as artifacts for a week), `release.yml` (on each of CI, Image and Runner build ending for a push to main, `scripts/release-gate.sh`; once all are green, the build's programs released as `runner-` with the `host-setup.sh` of its commit and their SHA256SUMS, never replaced; by hand with `workflow_dispatch` and a commit). Labelled pull requests are merged by CodeFusion Console. | ## Run diff --git a/scripts/host-setup.sh b/scripts/host-setup.sh index c84d46b..8fcb879 100755 --- a/scripts/host-setup.sh +++ b/scripts/host-setup.sh @@ -22,7 +22,9 @@ # A build from a checkout updates itself from the console once the host is enrolled, so `install` leaves it be then. # A host given a --binary or --version gets that one: on an enrolled host `install` holds new jobs back, waits for the # running ones (at most an hour) and installs it; when they outlast the wait it exits 75, the hold lifted, and the -# Runner of before goes on. +# Runner of before goes on. A Runner from a release updates itself to newer releases from the console (`update-to`), so +# a --version is a minimum: an enrolled host whose Runner runs a newer commit of main, by GitHub's compare of main's +# history, keeps it, and so does one when GitHub cannot say. # # --class shared (the default) or trusted: a host that runs only main's deploy, release and signing jobs of # codefusion-cc/codefusion, by the Runner's default rules (crates/trusted), one job at a time @@ -39,7 +41,8 @@ # --github-api GitHub's API the Runner talks to, for a test against the fakes (`cargo xtask fakes serve`) # --check both steps' form only: does every step but the enrolment and the service, and needs no code # -# GITHUB_TOKEN, when set, goes with its reads of GitHub's API (the image's release), whose rate limit it waits out. +# GITHUB_TOKEN, when set, goes with its reads of GitHub's API (the image's release, the compare of a --version with the +# Runner that runs), whose rate limit it waits out. # # Each step is skipped when it is done already. Run again on an enrolled host with neither --binary nor --version (a # reboot's startup script), both steps only make sure the service runs. It says each step on standard output; a @@ -50,7 +53,8 @@ set -Eeuo pipefail USER_NAME=codefusion-runner GITHUB_REPO=https://github.com/codefusion-cc/runner -RELEASES_API=https://api.github.com/repos/codefusion-cc/runner/releases +REPO_API=https://api.github.com/repos/codefusion-cc/runner +RELEASES_API=$REPO_API/releases # A build needs the compiler and OpenSSL's headers; the Runner itself, passt (its machines' network) and curl's libssl. PACKAGES=(passt curl jq ca-certificates) # Container mode's engine: Podman, rootless, with the subordinate ids and the user's D-Bus it runs on. @@ -177,6 +181,51 @@ modules_wanted() { done } +# Whether the Runner that runs, of the commit $2 (the 12 characters its version names), should stay in place of the +# release of the commit $3 that a run asks for, by main's history as the repository's API at $1 compares them: `keep` +# when it runs a commit of main newer than $3, `install` when it runs $3, an older commit, one main does not have or +# no commit at all, `unknown` when GitHub could not say. +runs_newer() { + local api=$1 running=$2 wanted=$3 status head + if [[ ! $running =~ ^[0-9a-f]{12}$ || $running == "$wanted" ]]; then + echo install + return 0 + fi + status=$( + trap - ERR + github_get "$api/compare/$wanted...$running?per_page=1" | jq -r '.status // empty' + ) || status='' + case $status in + ahead) ;; + behind | diverged) + echo install + return 0 + ;; + *) + echo unknown + return 0 + ;; + esac + # Newer than the one asked for, and main's: main's branch by its ref, which no tag named main stands for. + head=$( + trap - ERR + github_get "$api/git/ref/heads/main" | jq -r '.object.sha // empty' + ) || head='' + if [[ ! $head =~ ^[0-9a-f]{40}$ ]]; then + echo unknown + return 0 + fi + status=$( + trap - ERR + github_get "$api/compare/$running...$head?per_page=1" | jq -r '.status // empty' + ) || status='' + case $status in + ahead | identical) echo keep ;; + behind | diverged) echo install ;; + *) echo unknown ;; + esac +} + # Sourced (the tests), it only defines the above. if (return 0 2>/dev/null); then return 0; fi case ${1-} in @@ -552,6 +601,25 @@ place_binary() { fi local named=${version:+release runner-$version} named=${named:-$binary} + # A release asked for is a minimum: a Runner that updated itself past it, to a newer commit of main, stays. + if ((enrolled)) && [[ -n $version && -x $installed && $(user_sha256 "$installed") != "$sha256" ]]; then + local runs + runs=$( + trap - ERR + as_user "'$installed' version" 2>/dev/null + ) || runs='' + runs=${runs#"$USER_NAME "} + case $(runs_newer "$REPO_API" "${runs##*+}" "${version##*-}") in + keep) + say "it runs $runs, newer on main than $named: it stays, and $named waits for none" + return + ;; + unknown) + say "it runs $runs, and GitHub could not say whether that is newer on main than $named: it stays; a run later asks again" + return + ;; + esac + fi if [[ $(user_sha256 "$staged") != "$sha256" ]]; then change "place the Runner of $named" if ((!dry)); then diff --git a/scripts/release-gate.sh b/scripts/release-gate.sh new file mode 100755 index 0000000..dd49b9c --- /dev/null +++ b/scripts/release-gate.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# Whether the Runner release of a commit may go out, by the rule the console rolls Runners out by: a commit of main +# whose push ran CI green, and Image green too when that push made an Image run, the newest run of each counting. Its +# programs are those the push's "Runner build" made, which must be green as well. +# +# scripts/release-gate.sh +# +# Its one line on standard output, with status 0: +# publish the release may go out, with the programs of that "Runner build" run +# wait a run the release waits for has not ended: the run that ends it starts this check again +# skip the commit gets no release: it is not on main, or a run it needs ended other than green +# A status other than 0 when GitHub could not say, after host-setup.sh's retries of 403, 408, 429, 5xx and no answer. +# GITHUB_TOKEN, when set, goes with each read. +set -Eeuo pipefail + +api=${1:?the repository API} +commit=${2:?the commit} + +# github_get, with its retries, from the script hosts run; sourced, it only defines it. +# shellcheck source=scripts/host-setup.sh +source "$(dirname "${BASH_SOURCE[0]}")/host-setup.sh" +trap - ERR EXIT +step=release-gate + +CI=ci.yml +IMAGE=image.yml +BUILD=runner-build.yml + +if [[ ! $commit =~ ^[0-9a-f]{40}$ ]]; then + echo "release-gate: $commit is no whole commit" >&2 + exit 2 +fi + +main=$(github_get "$api/git/ref/heads/main" | jq -r '.object.sha // empty') +if [[ ! $main =~ ^[0-9a-f]{40}$ ]]; then + echo "release-gate: main's ref names no commit" >&2 + exit 1 +fi +status=$(github_get "$api/compare/$commit...$main?per_page=1" | jq -r '.status // empty') +case $status in + ahead | identical) ;; + behind | diverged) + echo "skip $commit is not on main" + exit 0 + ;; + *) + echo "release-gate: the compare of $commit with main says '$status'" >&2 + exit 1 + ;; +esac + +# The newest run of a workflow for the push of the commit to main: ` `, or nothing for none. +newest_run() { + github_get "$api/actions/workflows/$1/runs?head_sha=$commit&event=push&branch=main&per_page=100" | + jq -r --arg commit "$commit" ' + [.workflow_runs[] | select(.head_sha == $commit and .event == "push" and .head_branch == "main")] + | if length == 0 then empty else max_by(.run_number, .run_attempt) + | "\(.id) \(.status) \(.conclusion // "none")" end' +} + +# `wait` or `skip` for a run that is not green; nothing for a green one. $1 names the workflow, $2 its run, $3 whether +# no run is needed. +judge() { + local name=$1 run=$2 optional=$3 id state conclusion + if [[ -z $run ]]; then + ((optional)) || echo "wait $name has not run for this push" + return 0 + fi + read -r id state conclusion <<<"$run" + if [[ ! $id =~ ^[0-9]+$ ]]; then + echo "release-gate: $name's run is not as GitHub lists runs" >&2 + return 1 + fi + if [[ $state != completed ]]; then + echo "wait $name run $id is $state" + elif [[ $conclusion != success ]]; then + echo "skip $name run $id ended $conclusion" + fi +} + +build='' +for workflow in "$CI" "$IMAGE" "$BUILD"; do + case $workflow in + "$CI") name=CI optional=0 ;; + "$IMAGE") name=Image optional=1 ;; + *) name='Runner build' optional=0 ;; + esac + run=$(newest_run "$workflow") + said=$(judge "$name" "$run" "$optional") + if [[ -n $said ]]; then + echo "$said" + exit 0 + fi + [[ $workflow != "$BUILD" ]] || build=${run%% *} +done +echo "publish $build" diff --git a/xtask/tests/host_setup.rs b/xtask/tests/host_setup.rs index da401e1..0423d31 100644 --- a/xtask/tests/host_setup.rs +++ b/xtask/tests/host_setup.rs @@ -223,6 +223,167 @@ fn a_server_error_or_no_answer_is_asked_again() { assert!(said.contains("GitHub answered 000 for"), "{said}"); } +/// `runs_newer` of the script, sourced, against the repository's API at `api`: the Runner that runs at `running`, +/// the release asked for at `wanted`. Its answer and what it said on standard error. +fn runs_newer(api: &str, running: &str, wanted: &str, env: &[(&str, &str)]) -> (String, String) { + let script = concat!(env!("CARGO_MANIFEST_DIR"), "/../scripts/host-setup.sh"); + let mut command = Command::new("bash"); + command + .args([ + "-c", + "source \"$1\" && runs_newer \"$2\" \"$3\" \"$4\"", + "bash", + script, + api, + running, + wanted, + ]) + .env_remove("GITHUB_TOKEN"); + for (name, value) in env { + command.env(name, value); + } + let out = command.output().expect("bash runs"); + assert!(out.status.success(), "it always answers: {out:?}"); + ( + String::from_utf8_lossy(&out.stdout).trim().to_string(), + String::from_utf8_lossy(&out.stderr).to_string(), + ) +} + +const RUNS: &str = "bbbbbbbbbbbb"; +const ASKED: &str = "aaaaaaaaaaaa"; +const MAIN: &str = "cccccccccccc0000000000000000000000000000"; + +fn compared(status: &str) -> Reply { + Reply::json(&serde_json::json!({ "status": status, "commits": [] })) +} + +fn main_at() -> Reply { + Reply::json(&serde_json::json!({ "ref": "refs/heads/main", "object": { "sha": MAIN, "type": "commit" } })) +} + +fn paths(api: &Api) -> Vec { + api.requests().iter().map(|r| r.path.clone()).collect() +} + +/// A Runner that updated itself past the release a run asks for, to a commit main has, stays: the version is a +/// minimum. +#[test] +fn a_runner_newer_on_main_than_the_version_asked_for_stays() { + for main_says in ["ahead", "identical"] { + let api = api(vec![compared("ahead"), main_at(), compared(main_says)]); + let (said, _) = runs_newer(&api.server.url(), RUNS, ASKED, &[]); + assert_eq!(said, "keep", "main is {main_says} of it"); + assert_eq!( + paths(&api), + [ + format!("/compare/{ASKED}...{RUNS}"), + "/git/ref/heads/main".to_string(), + format!("/compare/{RUNS}...{MAIN}"), + ] + ); + } +} + +#[test] +fn a_runner_older_than_the_version_asked_for_or_off_main_takes_it() { + for (answers, asked) in [ + (vec![compared("behind")], 1), + (vec![compared("diverged")], 1), + // Newer than the one asked for, but on a branch beside main, or a tag named main. + (vec![compared("ahead"), main_at(), compared("diverged")], 3), + (vec![compared("ahead"), main_at(), compared("behind")], 3), + ] { + let api = api(answers); + let (said, _) = runs_newer(&api.server.url(), RUNS, ASKED, &[]); + assert_eq!(said, "install"); + assert_eq!(api.requests().len(), asked); + } +} + +/// The same commit with other bytes, or a program that names no commit: the release asked for goes in, and GitHub is +/// not asked. +#[test] +fn a_runner_of_no_commit_or_of_the_one_asked_for_takes_it_without_asking_github() { + let api = api(vec![compared("ahead")]); + for running in [ + ASKED, + "", + "unknown", + "bbbbbbbbbbb", + "bbbbbbbbbbbbb", + "BBBBBBBBBBBB", + "bbbbbbbbbbb-", + ] { + let (said, _) = runs_newer(&api.server.url(), running, ASKED, &[]); + assert_eq!(said, "install", "{running:?}"); + } + assert_eq!(api.requests().len(), 0); +} + +/// GitHub's rate limit and server errors are waited out as every read of the script waits them out. +#[test] +fn a_429_or_a_5xx_is_asked_again_before_it_decides() { + let api = api(vec![ + rate_limited(429).with_header("Retry-After", "1"), + compared("ahead"), + Reply::empty(502), + main_at(), + Reply::empty(503), + compared("ahead"), + ]); + let (said, stderr) = runs_newer(&api.server.url(), RUNS, ASKED, &[]); + assert_eq!(said, "keep"); + assert_eq!(api.requests().len(), 6); + assert!( + stderr.contains("GitHub answered 429, asking again in 1s"), + "{stderr}" + ); +} + +/// When GitHub cannot say, at any of the three reads, the Runner that runs stays. +#[test] +fn what_github_cannot_say_keeps_the_runner_that_runs() { + let not_found = Reply::status_json(404, &serde_json::json!({ "message": "Not Found" })); + for answers in [ + vec![Reply::empty(503)], + vec![not_found.clone()], + vec![Reply::text(200, "a proxy")], + vec![compared("sideways")], + vec![compared("ahead"), Reply::empty(503)], + vec![ + compared("ahead"), + Reply::json(&serde_json::json!({ "object": { "sha": "main" } })), + ], + vec![compared("ahead"), main_at(), Reply::empty(503)], + vec![compared("ahead"), main_at(), not_found], + ] { + let api = api(answers.clone()); + let (said, _) = runs_newer( + &api.server.url(), + RUNS, + ASKED, + &[("GITHUB_TRIES", "2"), ("GITHUB_WAIT_MAX", "1")], + ); + assert_eq!(said, "unknown", "{answers:?}"); + } +} + +/// A ref that names no whole commit, such as a tag's or a short name, is not compared with: main is not known. +#[test] +fn a_main_that_names_no_whole_commit_is_not_compared_with() { + for sha in ["main", &MAIN[..12], &MAIN.to_uppercase(), ""] { + let api = api(vec![ + compared("ahead"), + Reply::json(&serde_json::json!({ "object": { "sha": sha } })), + compared("ahead"), + ]); + let (said, _) = runs_newer(&api.server.url(), RUNS, ASKED, &[]); + assert_eq!(said, "unknown", "{sha:?}"); + assert_eq!(api.requests().len(), 2, "{sha:?}"); + } +} + /// `merged_config` of the script, sourced: the configuration's text now, the last run's settings, the keys the script /// owns and the settings of this run. fn merged(now: &str, last: &str, owned: &str, settings: &str) -> Result { @@ -626,3 +787,238 @@ fn euid_is_root() -> bool { .output() .is_ok_and(|output| output.stdout.trim_ascii() == b"0") } + +/// `scripts/release-gate.sh` against the repository's API at `api`, for `commit`: its status, standard output and +/// standard error. +fn release_gate(api: &str, commit: &str, env: &[(&str, &str)]) -> (Option, String, String) { + let script = concat!(env!("CARGO_MANIFEST_DIR"), "/../scripts/release-gate.sh"); + let mut command = Command::new("bash"); + command.args([script, api, commit]).env_remove("GITHUB_TOKEN"); + for (name, value) in env { + command.env(name, value); + } + let out = command.output().expect("bash runs"); + ( + out.status.code(), + String::from_utf8_lossy(&out.stdout).trim().to_string(), + String::from_utf8_lossy(&out.stderr).to_string(), + ) +} + +const PUSHED: &str = "dddddddddddd0000000000000000000000000000"; + +/// A run of a workflow as GitHub lists it, of the push of `PUSHED` to main unless changed. +fn run(id: u64, number: u64, status: &str, conclusion: Option<&str>) -> serde_json::Value { + serde_json::json!({ + "id": id, "run_number": number, "run_attempt": 1, "status": status, "conclusion": conclusion, + "head_sha": PUSHED, "event": "push", "head_branch": "main", + }) +} + +fn runs(list: &[serde_json::Value]) -> Reply { + Reply::json(&serde_json::json!({ "total_count": list.len(), "workflow_runs": list })) +} + +fn green(id: u64) -> Reply { + runs(&[run(id, 1, "completed", Some("success"))]) +} + +/// The reads before the runs: main's ref, and main has the commit. +fn on_main() -> Vec { + vec![main_at(), compared("ahead")] +} + +fn gate_says(answers: Vec) -> (String, usize) { + let api = api(answers); + let (status, said, stderr) = release_gate(&api.server.url(), PUSHED, &[]); + assert_eq!(status, Some(0), "{said} {stderr}"); + (said, api.requests().len()) +} + +/// A push whose CI, Image when it made one, and Runner build are green gets its release, with that build's programs. +#[test] +fn a_green_push_of_main_is_released_with_its_build() { + let api = api([on_main(), vec![green(10), runs(&[]), green(30)]].concat()); + let (status, said, stderr) = release_gate(&api.server.url(), PUSHED, &[]); + assert_eq!((status, said.as_str()), (Some(0), "publish 30"), "{stderr}"); + let requests = api.requests(); + let asked: Vec<&str> = requests.iter().map(|r| r.path.as_str()).collect(); + assert_eq!( + asked, + [ + "/git/ref/heads/main", + &format!("/compare/{PUSHED}...{MAIN}"), + "/actions/workflows/ci.yml/runs", + "/actions/workflows/image.yml/runs", + "/actions/workflows/runner-build.yml/runs", + ] + ); + for request in &requests[2..] { + for pair in [("head_sha", PUSHED), ("event", "push"), ("branch", "main")] { + assert!( + request.query.contains(&(pair.0.to_string(), pair.1.to_string())), + "{:?}", + request.query + ); + } + } + // Main at the commit itself, and an Image run that is green. + let (said, _) = gate_says(vec![ + main_at(), + compared("identical"), + green(10), + green(20), + green(30), + ]); + assert_eq!(said, "publish 30"); +} + +/// The red commit that went out before: a run that ended other than green keeps its commit's release back, as does +/// a commit main does not have. +#[test] +fn a_push_with_a_red_run_or_off_main_gets_no_release() { + let red = |conclusion| runs(&[run(9, 1, "completed", Some(conclusion))]); + for (answers, starts) in [ + (vec![main_at(), compared("behind")], "skip "), + (vec![main_at(), compared("diverged")], "skip "), + ( + [on_main(), vec![red("failure")]].concat(), + "skip CI run 9 ended failure", + ), + ( + [on_main(), vec![red("cancelled")]].concat(), + "skip CI run 9 ended cancelled", + ), + ( + [on_main(), vec![red("timed_out")]].concat(), + "skip CI run 9 ended timed_out", + ), + ( + [on_main(), vec![green(10), red("failure")]].concat(), + "skip Image run 9", + ), + ( + [on_main(), vec![green(10), runs(&[]), red("failure")]].concat(), + "skip Runner build run 9", + ), + ] { + let (said, _) = gate_says(answers); + assert!(said.starts_with(starts), "{said}"); + } +} + +/// A run not ended, or not started, makes it wait: the workflow_run of the run that ends starts the check again. +#[test] +fn a_push_whose_runs_have_not_ended_waits() { + let going = |status| runs(&[run(9, 1, status, None)]); + for (answers, says) in [ + ( + [on_main(), vec![going("in_progress")]].concat(), + "wait CI run 9 is in_progress", + ), + ( + [on_main(), vec![going("queued")]].concat(), + "wait CI run 9 is queued", + ), + ( + [on_main(), vec![runs(&[])]].concat(), + "wait CI has not run for this push", + ), + ( + [on_main(), vec![green(10), going("in_progress")]].concat(), + "wait Image run 9 is in_progress", + ), + ( + [on_main(), vec![green(10), runs(&[]), runs(&[])]].concat(), + "wait Runner build has not run for this push", + ), + ] { + let (said, _) = gate_says(answers); + assert_eq!(said, says); + } +} + +/// The newest run of a workflow counts, as on the console: a re-run that went green releases, a newer red one does +/// not; runs of other commits, events or branches in the answer are none of this push's. +#[test] +fn the_newest_run_of_the_push_counts() { + let old_red = run(9, 4, "completed", Some("failure")); + let new_green = run(10, 5, "completed", Some("success")); + let mut rerun_green = run(9, 4, "completed", Some("success")); + rerun_green["run_attempt"] = 2.into(); + let mut first_attempt_red = run(9, 4, "completed", Some("failure")); + first_attempt_red["run_attempt"] = 1.into(); + for (list, says) in [ + (vec![old_red.clone(), new_green.clone()], "publish 30"), + (vec![new_green.clone(), old_red.clone()], "publish 30"), + (vec![first_attempt_red, rerun_green], "publish 30"), + ( + vec![run(11, 6, "completed", Some("failure")), new_green.clone()], + "skip CI run 11 ended failure", + ), + ] { + let (said, _) = gate_says([on_main(), vec![runs(&list), runs(&[]), green(30)]].concat()); + assert_eq!(said, says, "{list:?}"); + } + for (field, value) in [ + ("head_sha", "eeeeeeeeeeee0000000000000000000000000000"), + ("event", "pull_request"), + ("head_branch", "main-copy"), + ] { + let mut other = run(12, 7, "completed", Some("success")); + other[field] = value.into(); + let (said, _) = gate_says([on_main(), vec![runs(&[other])]].concat()); + assert_eq!(said, "wait CI has not run for this push", "{field}"); + } +} + +/// GitHub's rate limit and server errors are waited out; when GitHub cannot say, the gate fails rather than release. +#[test] +fn the_gate_waits_out_429_and_5xx_and_fails_when_github_cannot_say() { + let (said, asked) = gate_says(vec![ + rate_limited(429).with_header("Retry-After", "1"), + main_at(), + Reply::empty(502), + compared("ahead"), + green(10), + Reply::empty(503), + runs(&[]), + green(30), + ]); + assert_eq!((said.as_str(), asked), ("publish 30", 8)); + + let limits = [("GITHUB_TRIES", "2"), ("GITHUB_WAIT_MAX", "1")]; + for answers in [ + vec![Reply::empty(503)], + vec![Reply::json(&serde_json::json!({ "object": { "sha": "main" } }))], + vec![main_at(), compared("sideways")], + vec![main_at(), Reply::status_json(404, &serde_json::json!({ "message": "Not Found" }))], + [on_main(), vec![Reply::empty(503)]].concat(), + [on_main(), vec![Reply::text(200, "a proxy")]].concat(), + [on_main(), vec![runs(&[serde_json::json!({ "id": "x", "run_number": 1, "run_attempt": 1, "status": "completed", "conclusion": "success", "head_sha": PUSHED, "event": "push", "head_branch": "main" })])]].concat(), + [on_main(), vec![green(10), runs(&[]), Reply::empty(500)]].concat(), + ] { + let api = api(answers.clone()); + let (status, said, _) = release_gate(&api.server.url(), PUSHED, &limits); + assert_ne!(status, Some(0), "{answers:?}"); + assert!(!said.starts_with("publish"), "{said}"); + } +} + +#[test] +fn the_gate_takes_only_a_whole_commit() { + let api = api(vec![main_at()]); + for commit in [ + "", + "main", + &PUSHED[..12], + &PUSHED.to_uppercase(), + &format!("{PUSHED}0"), + "../../x", + ] { + let (status, said, _) = release_gate(&api.server.url(), commit, &[]); + assert!(status.is_some_and(|code| code != 0), "{commit:?}"); + assert_eq!(said, "", "{commit:?}"); + } + assert_eq!(api.requests().len(), 0); +} From 4b75f1615bf78f0b7d1184f26882f48e0c509626 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krystian=20Paw=C5=82ow?= Date: Thu, 12 Nov 2026 21:07:03 +0100 Subject: [PATCH 2/6] Wait out a used-up rate limit, stop stalled downloads, and read releases more sparingly The review of the release update found: - a 403 with x-ratelimit-remaining 0 and no Retry-After was final; it is now waited out until x-ratelimit-reset, for every GitHub call; - a stalled download or a retry wait outlived the Runner's end; - the 16-minute wait could run out on a slow Runner build: 45 minutes now, asked every 3, the newest releases read only every fifth look and at the last, 30 a page; - SHA256SUMS had a second, looser parser: cf_image::parse_sums now; - the console showed 'building' for a release: 'fetching its release'; - the fetch lock repeated the daemon's one-update guard. --- README.md | 2 +- crates/daemon/src/service.rs | 5 +- crates/daemon/src/testing.rs | 8 ++ crates/daemon/src/update.rs | 5 + crates/daemon/tests/service.rs | 5 + crates/fake-github/src/httpd.rs | 13 ++ crates/ghapp/src/http.rs | 28 ++-- crates/ghapp/src/release.rs | 37 ++++-- crates/ghapp/tests/release.rs | 86 +++++++++++- crates/image/src/lib.rs | 2 +- crates/image/src/verify.rs | 3 +- crates/runner/src/release_update.rs | 196 +++++++++++++--------------- docs/MAP.md | 6 +- scripts/host-setup.sh | 2 +- 14 files changed, 266 insertions(+), 132 deletions(-) diff --git a/README.md b/README.md index bfdc862..394e050 100644 --- a/README.md +++ b/README.md @@ -180,7 +180,7 @@ A Linux Runner installed from a release (`host-setup.sh --version`, no `build_fo the whole commit, on `main` and newer than the commit that runs (GitHub's compare). It fetches `SHA256SUMS` and `codefusion-runner-linux-amd64` or `-arm64` for its own architecture, checks the program against its line, asks it its `version`, which must run on this host and name the commit, and then drains, installs and checks the start as a build -does. A commit whose release is not out yet is waited for up to 16 minutes, asked every 2; a commit with no release, +does. A commit whose release is not out yet is waited for up to 45 minutes, asked every 3; a commit with no release, an older one, a checksum that does not match, a missing file, a program of another architecture or a disk with no room leaves the running program as it was and says why. Releases are not signed: the checksum guards against a broken download, and HTTPS to GitHub and the publisher's name against anyone else. "Runner release" puts out a commit diff --git a/crates/daemon/src/service.rs b/crates/daemon/src/service.rs index 6bdd758..72b5d44 100644 --- a/crates/daemon/src/service.rs +++ b/crates/daemon/src/service.rs @@ -2711,9 +2711,10 @@ impl Service { return Some((false, "an update is under way already".into())); } update.command = Some(id.to_string()); + let step = updates.first_step(); update.doing = Some(match &commit { - Some(commit) => format!("updating to {commit}: building"), - None => "updating: building".into(), + Some(commit) => format!("updating to {commit}: {step}"), + None => format!("updating: {step}"), }); } let service = self.handle(); diff --git a/crates/daemon/src/testing.rs b/crates/daemon/src/testing.rs index c542acd..db5d2bc 100644 --- a/crates/daemon/src/testing.rs +++ b/crates/daemon/src/testing.rs @@ -954,4 +954,12 @@ impl crate::update::SelfUpdate for FakeUpdate { fn takes_head(&self) -> bool { !self.0.lock().expect("update").commits_only } + + fn first_step(&self) -> &'static str { + if self.0.lock().expect("update").commits_only { + "fetching its release" + } else { + "building" + } + } } diff --git a/crates/daemon/src/update.rs b/crates/daemon/src/update.rs index fee4858..154eef6 100644 --- a/crates/daemon/src/update.rs +++ b/crates/daemon/src/update.rs @@ -48,6 +48,11 @@ pub trait SelfUpdate: Send + Sync { /// Ends what a build or a download under way started, as the Runner ends: the console sends its command again /// to the Runner that starts next. Nothing by default. fn end(&self) {} + + /// What the console shows while [`SelfUpdate::build`] runs. + fn first_step(&self) -> &'static str { + "building" + } } /// What the console hears of a plain `update` on a host whose updates come as releases. diff --git a/crates/daemon/tests/service.rs b/crates/daemon/tests/service.rs index 5a10d39..04fbc95 100644 --- a/crates/daemon/tests/service.rs +++ b/crates/daemon/tests/service.rs @@ -3939,9 +3939,14 @@ async fn a_runner_from_a_release_updates_only_to_a_commit_and_waits_for_its_mach let runner = rig.job_running().await; rig.updates.build_gives(Ok(NEWER)); + rig.updates.build_takes(Duration::from_millis(1500)); let to = rig .console .queue_command("update-to", Some(json!({"commit": "5c0ffee0a1b2"}))); + rig.settles("the console hears it fetches, not builds", || { + last_status(&rig.console)["reason"] == json!("updating to 5c0ffee0a1b2: fetching its release") + }) + .await; rig.settles("no new job while the machine stands", || { last_status(&rig.console)["reason"] == json!(format!("updating to {NEWER}: waiting for 1 machine to end")) diff --git a/crates/fake-github/src/httpd.rs b/crates/fake-github/src/httpd.rs index ba11dad..46002d3 100644 --- a/crates/fake-github/src/httpd.rs +++ b/crates/fake-github/src/httpd.rs @@ -64,6 +64,8 @@ pub enum Reply { Drop, /// Promise the whole body, send `sent` bytes of it, close. Truncated { status: u16, body: Vec, sent: usize }, + /// Promise the whole body, send `sent` bytes of it, then keep the connection open and send nothing more. + Stalled { body: Vec, sent: usize }, /// Wait, then do the inner reply. After(Duration, Box), } @@ -202,6 +204,17 @@ async fn serve_connection(stream: TcpStream, handler: Handler) -> std::io::Resul write.flush().await?; return Ok(()); } + Reply::Stalled { body, sent } => { + let head = format!( + "HTTP/1.1 200 OK\r\ncontent-type: application/octet-stream\r\ncontent-length: {}\r\n\r\n", + body.len() + ); + write.write_all(head.as_bytes()).await?; + write.write_all(&body[..sent.min(body.len())]).await?; + write.flush().await?; + std::future::pending::<()>().await; + return Ok(()); + } Reply::Response { status, headers, diff --git a/crates/ghapp/src/http.rs b/crates/ghapp/src/http.rs index fb3781b..9741752 100644 --- a/crates/ghapp/src/http.rs +++ b/crates/ghapp/src/http.rs @@ -237,16 +237,26 @@ impl Http { } } -/// The wait `Retry-After` asks for, in whole seconds. (GitHub sends seconds; a date is not read.) +/// The wait an answer asks for: `Retry-After` in whole seconds (GitHub sends seconds; a date is not read), else, for +/// a rate limit used up (`x-ratelimit-remaining: 0`), until `x-ratelimit-reset` by the answer's own clock (`Date`), +/// at least a second. pub(crate) fn retry_after(headers: &HeaderMap) -> Option { - headers - .get(RETRY_AFTER)? - .to_str() - .ok()? - .trim() - .parse::() - .ok() - .map(Duration::from_secs) + let number = |name: &str| headers.get(name)?.to_str().ok()?.trim().parse::().ok(); + if let Some(seconds) = number(RETRY_AFTER.as_str()) { + return Some(Duration::from_secs(seconds)); + } + if number("x-ratelimit-remaining")? != 0 { + return None; + } + let reset = i64::try_from(number("x-ratelimit-reset")?).ok()?; + let now = server_time(headers).unwrap_or_else(|| { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |since| i64::try_from(since.as_secs()).unwrap_or(i64::MAX)) + }); + Some(Duration::from_secs( + u64::try_from(reset.saturating_sub(now)).unwrap_or(0).max(1), + )) } /// An exponential wait with jitter: between half and the whole of `base * 2^attempt`, at most `cap`. diff --git a/crates/ghapp/src/release.rs b/crates/ghapp/src/release.rs index dc684a7..34ca50d 100644 --- a/crates/ghapp/src/release.rs +++ b/crates/ghapp/src/release.rs @@ -20,8 +20,11 @@ const MAX_REDIRECTS: usize = 5; /// How long a download may go without a byte before it counts as dropped. const READ_TIMEOUT: Duration = Duration::from_secs(60); -/// How many releases one page lists, GitHub's most. -pub const RELEASES_PAGE: u32 = 100; +/// How often a download that waits, for its next bytes or to try again, looks whether it was stopped. +const STOP_LOOK: Duration = Duration::from_millis(200); + +/// How many releases one page lists: a new release is among the first, and an image release lists many files. +pub const RELEASES_PAGE: u32 = 30; /// A release as GitHub's API lists it, with the fields an update reads. #[derive(Debug, Clone, PartialEq, Eq, Deserialize)] @@ -277,7 +280,14 @@ impl Releases { None => crate::http::backoff(self.retry.base, self.retry.cap, attempt), }; tracing::debug!(attempt, ?wait, "fetching a release's file again"); - tokio::time::sleep(wait).await; + // A wait that `stop` ends early: a Retry-After may ask for minutes. + let until = tokio::time::Instant::now() + wait; + while tokio::time::Instant::now() < until { + if stop.load(Ordering::SeqCst) { + return Err(stopped()); + } + tokio::time::sleep_until(until.min(tokio::time::Instant::now() + STOP_LOOK)).await; + } attempt += 1; } } @@ -314,14 +324,19 @@ impl Releases { } let mut hash = Sha256::new(); let mut bytes = 0u64; - while let Some(chunk) = response - .chunk() - .await - .map_err(|e| Error::Transport(format!("the file ended early: {}", e.without_url())))? - { + loop { if stop.load(Ordering::SeqCst) { - return Err(Error::Local("the download was stopped".into())); + return Err(stopped()); } + // A connection that stalls is looked at for `stop` while it waits for its next bytes. + let Ok(next) = tokio::time::timeout(STOP_LOOK, response.chunk()).await else { + continue; + }; + let Some(chunk) = + next.map_err(|e| Error::Transport(format!("the file ended early: {}", e.without_url())))? + else { + break; + }; bytes += chunk.len() as u64; if bytes > max { return Err(too_long()); @@ -337,6 +352,10 @@ impl Releases { } } +fn stopped() -> Error { + Error::Local("the download was stopped".into()) +} + /// A write on this host that failed: no room on its disk, most often. fn local(error: &std::io::Error) -> Error { if error.kind() == std::io::ErrorKind::StorageFull { diff --git a/crates/ghapp/tests/release.rs b/crates/ghapp/tests/release.rs index 669517d..8f18dc8 100644 --- a/crates/ghapp/tests/release.rs +++ b/crates/ghapp/tests/release.rs @@ -338,6 +338,90 @@ async fn a_download_is_stopped_when_the_runner_ends() { assert_eq!(fake.requests().len(), 1, "not asked again"); } +/// The Runner ends while the connection stalls, or while a download waits to be tried again: it ends at once, not +/// after the read timeout or the wait GitHub asked for. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_download_that_stalls_or_waits_ends_when_the_runner_ends() { + let body = program(); + for reply in [ + Reply::Stalled { + body: body.clone(), + sent: 1000, + }, + Reply::empty(429).with_header("retry-after", "4"), + ] { + let fake = fake(move |_, _| reply.clone()); + let mut target = target(); + let stop = Arc::new(AtomicBool::new(false)); + let stopping = stop.clone(); + let began = std::time::Instant::now(); + let ends = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(500)); + stopping.store(true, std::sync::atomic::Ordering::SeqCst); + }); + let error = releases(&fake) + .download( + &format!("{}/f", fake.server.url()), + &mut target.file, + 2 << 20, + &stop, + ) + .await + .expect_err("stopped"); + ends.join().unwrap(); + assert_eq!(error, Error::Local("the download was stopped".into())); + assert!(began.elapsed() < Duration::from_secs(2), "{:?}", began.elapsed()); + assert_eq!(fake.requests().len(), 1); + } +} + +/// GitHub's rate limit used up (403, no Retry-After, `x-ratelimit-remaining: 0`) is waited out until its reset; any +/// other 403, or a reset later than the longest wait, is not asked again. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_used_up_rate_limit_is_waited_out_until_its_reset() { + let now = || { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs() + }; + let limited = |reset: u64| { + Reply::status_json(403, &serde_json::json!({"message": "API rate limit exceeded"})) + .with_header("x-ratelimit-remaining", "0") + .with_header("x-ratelimit-reset", &reset.to_string()) + }; + let release = serde_json::json!({"tag_name": "runner-0.1.0-aaaaaaaaaaaa", "target_commitish": "x"}); + for (first, asked, found) in [ + (limited(now() + 1), 2, true), + (limited(now() - 30), 2, true), + (limited(now() + 3600), 1, false), + ( + Reply::status_json(403, &serde_json::json!({"message": "Resource not accessible"})), + 1, + false, + ), + ( + Reply::status_json(403, &serde_json::json!({"message": "x"})) + .with_header("x-ratelimit-remaining", "12") + .with_header("x-ratelimit-reset", &(now() + 1).to_string()), + 1, + false, + ), + ] { + let first = Mutex::new(Some(first)); + let ok = Reply::json(&release); + let fake = fake(move |_, _| first.lock().unwrap().take().unwrap_or_else(|| ok.clone())); + let began = std::time::Instant::now(); + let got = releases(&fake).by_tag("runner-0.1.0-aaaaaaaaaaaa").await; + assert_eq!(got.is_ok(), found, "{got:?}"); + if !found { + assert_eq!(got.expect_err("refused").status(), Some(403)); + } + assert_eq!(fake.requests().len(), asked); + assert!(began.elapsed() < Duration::from_secs(4), "{:?}", began.elapsed()); + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn a_compare_says_how_two_commits_stand_and_nothing_else() { for (status, said) in [ @@ -384,7 +468,7 @@ async fn a_page_of_releases_is_read_with_its_files() { let fake = fake(|request, _| { assert_eq!( (request.query("per_page"), request.query("page")), - (Some("100"), Some("2")) + (Some("30"), Some("2")) ); Reply::json(&serde_json::json!([{ "tag_name": "runner-0.1.0-aaaaaaaaaaaa", diff --git a/crates/image/src/lib.rs b/crates/image/src/lib.rs index 2992e87..c2640e8 100644 --- a/crates/image/src/lib.rs +++ b/crates/image/src/lib.rs @@ -27,7 +27,7 @@ pub use manifest::{ Asset, FORMAT, HOST_ARCH, MANIFEST_NAME, Manifest, Rootfs, RootfsKind, SIGNATURE_NAME, SUMS_NAME, }; pub use release::{ReleaseInputs, parse_signing_key, public_key_text, write_release}; -pub use verify::{Verified, parse_public_key, verify}; +pub use verify::{Verified, parse_public_key, parse_sums, verify}; /// The public key every downloaded image must be signed with (unpadded base64url), committed in `trusted-key.txt`. pub const TRUSTED_PUBLIC_KEY: &str = include_str!("../trusted-key.txt"); diff --git a/crates/image/src/verify.rs b/crates/image/src/verify.rs index b3f1c13..701da1a 100644 --- a/crates/image/src/verify.rs +++ b/crates/image/src/verify.rs @@ -168,7 +168,8 @@ fn hash_asset(files: &dyn Files, asset: &Asset, also: &mut [&mut Sha256]) -> Res Ok(()) } -fn parse_sums(sums: &[u8]) -> Result, ImageError> { +/// A `sha256sum` list (`<64 lowercase hex> ` a line, each name once and a plain file name), by name. +pub fn parse_sums(sums: &[u8]) -> Result, ImageError> { let text = std::str::from_utf8(sums).map_err(|_| ImageError::SumsMalformed { line: 1 })?; let mut listed = BTreeMap::new(); for (index, line) in text.lines().enumerate() { diff --git a/crates/runner/src/release_update.rs b/crates/runner/src/release_update.rs index d7a9381..2700aad 100644 --- a/crates/runner/src/release_update.rs +++ b/crates/runner/src/release_update.rs @@ -9,7 +9,6 @@ //! that commit. Plain `update` (main's head) is a build's: it is not offered here. use std::path::{Path, PathBuf}; -use std::sync::Mutex; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{Duration, Instant}; @@ -33,10 +32,12 @@ const MAX_SUMS: u64 = 64 * 1024; /// The largest program taken: the Runner with its guest and cloud-hypervisor is some tens of megabytes. const MAX_PROGRAM: u64 = 512 * 1024 * 1024; /// How long a release that is not out is waited for: the console sends a commit once its CI is green, and the -/// release workflow publishes it minutes after, from what it built while CI ran. -pub const RELEASE_WAIT: Duration = Duration::from_secs(16 * 60); +/// release workflow publishes it minutes after, from what it built while CI ran; as long as a build may take. +pub const RELEASE_WAIT: Duration = Duration::from_secs(45 * 60); /// How often GitHub is asked for it meanwhile: GitHub answers 60 reads an hour to a host without a token. -pub const ASK_EVERY: Duration = Duration::from_secs(2 * 60); +pub const ASK_EVERY: Duration = Duration::from_secs(3 * 60); +/// Of those looks, every how many also reads the newest releases, for a tag of another package version. +const PAGE_EVERY: u32 = 5; /// The name a release's program is fetched to beside the installed one. const DOWNLOAD: &str = "download"; @@ -71,10 +72,8 @@ pub struct ReleaseUpdate { package: String, /// How long a release that is not out is waited for, asked for every so often. wait: (Duration, Duration), - /// Set as the Runner ends: a download or a wait under way ends with it. + /// Set as the Runner ends: a download or a wait under way ends with it. The service runs one update at a time. stop: AtomicBool, - /// Held while a release is fetched: one at a time. - fetching: Mutex<()>, } impl ReleaseUpdate { @@ -112,7 +111,6 @@ impl ReleaseUpdate { package: env!("CARGO_PKG_VERSION").into(), wait: (RELEASE_WAIT, ASK_EVERY), stop: AtomicBool::new(false), - fetching: Mutex::new(()), }) } @@ -128,15 +126,17 @@ impl ReleaseUpdate { } /// The release of the commit `short`, its tag that of the running package's version or, after a version - /// changed, any `runner--` of the newest releases. Waits for one that is not out yet. + /// changed, any `runner--` of the newest releases (read at the first look and every + /// [`PAGE_EVERY`]th). Waits for one that is not out yet. async fn release_of(&self, short: &str) -> Result { let (wait, every) = self.wait; let began = Instant::now(); - loop { - if let Some(release) = self.look_for(short).await? { + for look in 0u32.. { + let last = began.elapsed() + every > wait || self.stop.load(Ordering::SeqCst); + if let Some(release) = self.look_for(short, last || look % PAGE_EVERY == 0).await? { return Ok(release); } - if began.elapsed() + every > wait || self.stop.load(Ordering::SeqCst) { + if last { return Err(format!( "{short} has no release of {REPO} (asked for {}); nothing was downloaded", cf_daemon::update::spoken(began.elapsed()) @@ -145,9 +145,10 @@ impl ReleaseUpdate { tracing::info!(commit = short, "its release is not out yet; asking again"); self.sleep(every).await; } + unreachable!("the looks end by the wait") } - async fn look_for(&self, short: &str) -> Result, String> { + async fn look_for(&self, short: &str, with_page: bool) -> Result, String> { let unread = |error: cf_ghapp::Error| { format!("the releases of {REPO} could not be read: {error}; nothing was downloaded") }; @@ -159,6 +160,9 @@ impl ReleaseUpdate { { return Ok(Some(release)); } + if !with_page { + return Ok(None); + } let ends = format!("-{short}"); let page = self.releases.page(1).await.map_err(unread)?; Ok(page.into_iter().find(|release| { @@ -252,8 +256,10 @@ impl ReleaseUpdate { .download(&sums.browser_download_url, &mut listed, MAX_SUMS, &self.stop) .await .map_err(|error| fetched(SUMS, error))?; - let want = sum_of(&listed, &self.asset) - .ok_or_else(|| format!("{SUMS} of {tag} names no single SHA-256 of {}; {not}", self.asset))?; + let want = cf_image::parse_sums(&listed) + .map_err(|error| format!("{SUMS} of {tag} is no list of SHA-256s: {error}; {not}"))? + .remove(&self.asset) + .ok_or_else(|| format!("{SUMS} of {tag} names no SHA-256 of {}; {not}", self.asset))?; let path = self.download(); let mut file = new_file(&path).map_err(|error| format!("{}: {error}; {not}", path.display()))?; let got = self @@ -296,9 +302,6 @@ impl SelfUpdate for ReleaseUpdate { if short == running { return Ok(self.running.clone()); } - let Ok(_one) = self.fetching.try_lock() else { - return Err(format!("a release is being fetched already; {not}")); - }; let path = self.download(); let fetched = block_on(self.fetch(name, short, running)); let offered = match fetched { @@ -323,6 +326,10 @@ impl SelfUpdate for ReleaseUpdate { fn end(&self) { self.stop.store(true, Ordering::SeqCst); } + + fn first_step(&self) -> &'static str { + "fetching its release" + } } /// Runs `future` to its end from a thread that may wait: the service's runtime when the thread is one of its @@ -354,24 +361,6 @@ fn make_runnable(path: &Path) -> std::io::Result<()> { std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700)) } -/// The SHA-256 a `sha256sum` list names for `file`: one line `<64 hex> ` (or ` *`), and only one. -pub fn sum_of(list: &[u8], file: &str) -> Option { - let text = std::str::from_utf8(list).ok()?; - let mut found = text.lines().filter_map(|line| { - let (sum, name) = line.split_once(' ')?; - let name = name.strip_prefix(' ').or_else(|| name.strip_prefix('*'))?; - (name == file).then_some(sum) - }); - let sum = found.next()?; - if found.next().is_some() - || sum.len() != 64 - || !sum.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) - { - return None; - } - Some(sum.to_string()) -} - fn hex(bytes: &[u8]) -> String { bytes.iter().map(|byte| format!("{byte:02x}")).collect() } @@ -379,7 +368,7 @@ fn hex(bytes: &[u8]) -> String { #[cfg(test)] mod tests { use std::collections::{HashMap, VecDeque}; - use std::sync::Arc; + use std::sync::{Arc, Mutex}; use cf_daemon::update::{MAX_STARTS, UpdateFile}; use cf_fake_github::httpd::{Handler, Reply, Request, Server}; @@ -780,25 +769,42 @@ mod tests { fn a_list_that_does_not_name_this_hosts_program_once_is_refused() { let bench = bench(); let ours = program(&NEW[..12]); - for list in [ - format!("{} {OTHER}\n", sum(b"x")), - format!("{} {ASSET}\n{} {ASSET}\n", sum(&ours), sum(&ours)), - format!("{} {ASSET}\n", &sum(&ours)[..63]), - format!("{} {ASSET}\n", sum(&ours).to_uppercase()), + let tag = Bench::tag(NEW); + for (list, malformed) in [ + (format!("{} {OTHER}\n", sum(b"x")), false), + (String::new(), false), + ( + format!("{} {ASSET}\n{} {ASSET}\n", sum(&ours), sum(&ours)), + true, + ), + (format!("{} {ASSET}\n", &sum(&ours)[..63]), true), + (format!("{} {ASSET}\n", sum(&ours).to_uppercase()), true), + (format!("{} *{ASSET}\n", sum(&ours)), true), + ( + format!("{} {OTHER}\nnot a line\n{} {ASSET}\n", sum(b"x"), sum(&ours)), + true, + ), ] { bench .hub .lock() .expect("hub") .files - .insert(Bench::file_path(NEW, SUMS), list.into_bytes()); - assert_eq!( - bench.update.build(Some(NEW)), - Err(format!( - "SHA256SUMS of {} names no single SHA-256 of {ASSET}; nothing was installed", - Bench::tag(NEW) - )) - ); + .insert(Bench::file_path(NEW, SUMS), list.clone().into_bytes()); + let said = bench.update.build(Some(NEW)).expect_err("refused"); + if malformed { + assert!( + said.starts_with(&format!("SHA256SUMS of {tag} is no list of SHA-256s: ")), + "{list:?}: {said}" + ); + assert!(said.ends_with("; nothing was installed"), "{said}"); + } else { + assert_eq!( + said, + format!("SHA256SUMS of {tag} names no SHA-256 of {ASSET}; nothing was installed"), + "{list:?}" + ); + } untouched(&bench); } } @@ -838,6 +844,42 @@ mod tests { ], ); assert_eq!(bench.update.build(Some(NEW)), Ok(format!("0.1.0+{}", &NEW[..12]))); + let seen = bench.seen(); + let count = |path: &str| seen.iter().filter(|seen| *seen == path).count(); + assert_eq!((count(&tags), count(&format!("/repos/{REPO}/releases"))), (3, 2)); + } + + /// While it waits, a look reads the tag alone; the newest releases only every few looks, and at the last. + #[test] + fn a_wait_reads_the_newest_releases_only_now_and_then() { + let mut bench = bench(); + bench + .update + .wait_for(Duration::from_millis(1500), Duration::from_millis(100)); + let none = "dddddddddddd3333333333333333333333333333"; + bench.update.build(Some(none)).expect_err("no release"); + let seen = bench.seen(); + let tags = seen + .iter() + .filter(|path| path.contains("/releases/tags/")) + .count(); + let mut page_after = Vec::new(); + let mut look = 0usize; + for path in &seen { + if path.contains("/releases/tags/") { + look += 1; + } else { + page_after.push(look); + } + } + assert!(tags >= 10, "{seen:?}"); + let mut wanted: Vec = (1..=tags) + .filter(|look| (look - 1) % PAGE_EVERY as usize == 0) + .collect(); + if !wanted.contains(&tags) { + wanted.push(tags); + } + assert_eq!(page_after, wanted, "{seen:?}"); } #[test] @@ -1077,38 +1119,6 @@ mod tests { } /// The service runs one update at a time; a second fetch at once is refused all the same, and fetches nothing. - #[test] - fn a_second_update_while_one_fetches_is_refused() { - let bench = bench(); - let tags = format!("/repos/{REPO}/releases/tags/{}", Bench::tag(NEW)); - let slow = Reply::After( - Duration::from_secs(2), - Box::new(Reply::status_json(404, &serde_json::json!({}))), - ); - bench.fault(&tags, vec![slow]); - bench.fault( - &format!("/repos/{REPO}/releases"), - vec![Reply::json(&serde_json::json!([]))], - ); - std::thread::scope(|scope| { - let first = scope.spawn(|| bench.update.build(Some(NEW))); - let began = Instant::now(); - while bench.seen().is_empty() { - assert!(began.elapsed() < Duration::from_secs(10), "the first asks"); - std::thread::sleep(Duration::from_millis(10)); - } - assert_eq!( - bench.update.build(Some(NEW)), - Err("a release is being fetched already; nothing was downloaded".into()) - ); - assert!( - first.join().expect("the first").is_err(), - "no release on the first look" - ); - }); - assert_eq!(bench.seen().len(), 2, "the second asked nothing"); - } - /// The Runner ends while a release is waited for: the wait ends with it. #[test] fn a_wait_for_a_release_ends_when_the_runner_ends() { @@ -1156,28 +1166,6 @@ mod tests { assert_eq!(said, Ok(format!("0.1.0+{}", &NEW[..12]))); } - #[test] - fn a_sum_is_read_from_its_one_line() { - let a = "a".repeat(64); - let b = "b".repeat(64); - assert_eq!( - sum_of(format!("{a} x\n{b} y\n").as_bytes(), "y"), - Some(b.clone()) - ); - assert_eq!(sum_of(format!("{a} *y\n").as_bytes(), "y"), Some(a.clone())); - for list in [ - format!("{a} y\n{b} y\n"), - format!("{a} yy\n"), - format!("{a} y\n"), - format!("{a}\ty\n"), - format!("{} y\n", &a[..62]), - String::new(), - ] { - assert_eq!(sum_of(list.as_bytes(), "y"), None, "{list:?}"); - } - assert_eq!(sum_of(b"\xff\xfe y", "y"), None); - } - #[test] fn this_host_takes_its_own_architecture_s_program() { let asset = asset_for_this_host(); diff --git a/docs/MAP.md b/docs/MAP.md index 354540c..bce5e73 100644 --- a/docs/MAP.md +++ b/docs/MAP.md @@ -13,15 +13,15 @@ | `crates/nft` (`cf-nft`) | nf_tables rules without the `nft` program. `lib.rs` is the model (`Match`, `Rule` and `Rule::new`, `Chain` on a `Hook`, `Table` in a `Family`: inet, bridge, netdev) and `decide`, what a rule set does with a packet (pure). `encode.rs` turns tables into one netlink batch (`batch`, `Load::Replace` swaps the whole ruleset in one transaction) and refuses what the kernel would; `decode.rs` reads the kernel's messages back (`read`). `apply` and `list` (Linux) talk to the kernel through `cf-netlink`. `BLOCKED` is the one list of private and special ranges, used by the guest's firewall and by the host's in front of a Linux machine. Tests: `tests/nft.rs` (bytes `nft` 1.0.6 wrote, recorded in `tests/nft/`), `tests/model.rs` (a model of the kernel's expressions against `decide`, round trips), `tests/kernel.rs` (Linux: applied and listed back in a fresh user and network namespace). | | `crates/netlink` (`cf-netlink`) | Netlink itself, for any protocol: `message`, `attribute`, `attribute_str` (a NUL-ended text), `nested`, `messages` and `attributes` to read them back, and on Linux the `Socket` that sends requests and reads every answer (`request`, `dump`), with its own `Error`. Users: `cf-nft` (nf_tables), `cf-vmm-ch` (rtnetlink). Tests in `tests/wire.rs`. | | `crates/vmm-ch` (`cf-vmm-ch`) | `ChVmm`: machines on Linux, cloud-hypervisor under KVM, one process per machine (`machine.rs`, Linux). Each machine has a cgroup of its own under the part of the tree delegated to the Runner (`cgroup.rs`: memory at its size and `overhead`, no swap (a kernel without a swap account per cgroup only on a host that swaps to nothing), `pids.max`, the cpuset of its cores, `cgroup.kill`, `wait_empty` woken by `cgroup.events`; `memory.current` is `Vm::resident`; `own` and `enter_host` for a test suite that shares its delegated cgroup), its vCPUs on the host's fast cores that it may use (`cpus.rs` with `cf_host_linux::fast_cpus`, `--host-cpus` overrides). With a network it runs under the system's pasta (`programs::pasta`: found on `PATH`, refused when another user could change it or what it links to, tried when the first machine with a network starts; `apt install passt`, 2023-03 or newer) in a user and network namespace; the hidden `vm-sandbox` subcommand of the binary (`sandbox.rs`) makes the guest's tap, loads `firewall.rs` onto it (netdev ingress: ARP, DHCP and DNS to pasta on the gateway, IPv4 to the internet; drops `BLOCKED`, 127/8 and every network of the host, IPv6 and everything else), bridges it with pasta's device (`link.rs`, rtnetlink), and starts the monitor without capabilities and without user namespaces in a PID, cgroup, mount and empty network namespace of its own (the tap's descriptor its only way out), no file it writes growing past the largest it is shown, its root a read-only tmpfs with only what `Sandbox::for_machine` shows it (its program, the guest's files, its console and folder, `/dev/kvm`, `/dev/urandom`) and a `/proc` of its own, then reports how it ended (`ending.rs`). `command.rs` builds both command lines, `events.rs` reads `--event-monitor`, `vsock.rs` the CONNECT handshake. cloud-hypervisor is pinned in `pins.rs` for each architecture, the host's fetched into `target/vmm-programs` and embedded by `build.rs` (Linux on aarch64 or x86_64), unpacked into `/programs` and checked before each machine (`programs.rs`). Tests: unit tests of everything pure, `tests/isolation.rs` (Linux: a stand-in guest on a veth pair behind the real pasta, guard and firewall; each blocked target a test of its own), `tests/monitor_root.rs` (Linux, no harness: a stand-in monitor in the real sandbox tries the host's files, processes and mounts, and under pasta tries to send past its tap), `tests/kvm.rs` (Linux with KVM, no harness: how a real machine ends, a panic, a stop, a kill, its cgroup's OOM and a dead monitor; it skips without `/dev/kvm` and the guest, `CF_REQUIRE_KVM=1` makes that a failure). | -| `crates/runner` (`codefusion-runner`) | The binary. `commands.rs`: `version` (the package's version and the commit, which `build.rs` reads from git or `CF_COMMIT`), `image verify`, `image install`, `pause`, `resume` and `status` (`holds.rs`: a named hold on the machine, `--wait` until the Runner saw it and runs nothing, exit 75 when the wait runs out; `status --json`), `vm` (the vm-host, `src/vm.rs`), `boot-proof`, `vmm` (Linux: the cloud-hypervisor the build carries, which `install` and an update ask of a program), and the service's commands in `service.rs` (macOS and Linux): `daemon` (what the service manager runs; ends on SIGTERM; one per state folder, held by a lock on `/runner.lock`), `enroll` (the console's one-time code on standard input, read with the terminal's echo off by `hidden.rs`, which gives the terminal back on every way out, a signal included; the token to `/console-token`, 0600, never printed), `check [--github]` (what an install needs, and the memory at that moment against the budget's floor with the sizes that fit; with `--github` what GitHub shows, changing nothing there and nothing in the state folder), `install` (copies the program to `/bin`, writes the service manager's entry, refuses while a machine stands), `uninstall [--leave] [--purge]` (refused while a Runner started by hand holds the state folder; a Mac without a usable token goes without telling the console). `config.rs` is the configuration file (refused when others can write it; `console` required unless the file names `app_dir` (absolute, not through `..`), the folder of a GitHub App key the host held before the console held it: while its `app.pem` is there `service.rs` `app_and_key` signs with it (`held_key`: `KeyTokens`, and `cf_daemon::HeldKey` for the service), else the GitHub tokens come from the console (`ConsoleTokens`); `check` and `uninstall` sign with a held key and never take it out (`looking`); `use` required on a Mac; on Linux `dedicated` or left out; `max_jobs`, `idle_minutes` (a workstation's only), `min_free_memory_percent` and `vcpus` (a size's vCPUs by its label, which `Settings::limited` checks against the CPUs CI may use), `ci_memory_gb` (the memory CI's machines hold at once, each with its overhead, in GB of 2^30 bytes) and `ci_cpus` (the CPUs they take together) are the owner's `Limits`, which `budget_of` and the service lay over the host's settings; `ConfigFile::change` writes the console's `settings` command into the file through `.json.next`, which must start a Runner before it replaces the file) and, on a Mac, the refusal of the cf-runners manager's own name and of external volumes; `manager.rs` is `ServiceManager`, the seam to the service manager; `launchd.rs` the agent `cc.codefusion.vm-runner` in the person's session (`plist`, `Agent`; `stop` waits past `ExitTimeOut` and decides by whether launchd still has the agent, not by what `bootout` says); `systemd.rs` (Linux) the user unit `codefusion-runner.service` (`unit`, with every path quoted for systemd or refused; `UserUnit` over an injected `systemctl --user`, the real one ended after 100 s; `stop` waits past `TimeoutStopSec` and decides by `ActiveState`); `linux.rs` the lines `check` adds on Linux (`Facts`, `lines`: KVM, lingering, delegated controllers, user namespaces, pasta, `noexec`, each with the root command that fixes it). `update.rs`: `CheckoutUpdate`, the `SelfUpdate` of a host that built the Runner, for the console's `update` and `update-to`: in the checkout the program was built from (`build.rs` writes it down, with `CARGO_HOME`, `RUSTUP_HOME` and `CARGO_BUILD_JOBS`), only when it is the config's `build_folder`, clean and at a commit (`own`), fetches `origin refs/heads/main` (a tag named `main` is not it), takes main's head or the commit `update-to` names when that resolves to one commit of the main fetched (`on_main`), refuses a target that does not contain the running commit (both looked up among the objects alone by `commit_named`, so a branch or tag named like a commit never stands for it), checks out the peeled commit, builds with `cargo xtask build` at low priority into `logs/update.log` and ends a build that takes longer than 45 minutes or outlasts the daemon, and keeps the program it checked as `bin/codefusion-runner.next`; `install` places that one, keeps the running program as `bin/codefusion-runner.prev` and writes `update.json` first; `guard`, run before the daemon reads its configuration, counts the new program's starts and puts the one before back after `MAX_STARTS`, and `ended_in_order` takes back the count of a start that ended as asked. `Installer` is what both updates share: `offer` asks a program its `version` (it must run here and name the commit) and keeps it as `.next`, `install` places it. `release_update.rs` (Linux and macOS builds; Linux hosts only by asset): `ReleaseUpdate`, the `SelfUpdate` of a Runner with no `build_folder`, installed from a release: `takes_head` is false, so it lists only `update-to`; `build` finds the commit's release (`runner--<12 hex>` by tag, else the newest page), waits `RELEASE_WAIT` for one not out yet, asked every `ASK_EVERY`, ended by `end`; refuses a draft, another publisher than `github-actions[bot]`, one not made at the whole commit, not on main or not newer than the running commit (`cf_ghapp::Releases::compare`); fetches `SHA256SUMS` and this host's asset (`asset_for_this_host`) into `.download` (0600), checks its SHA-256 (`sum_of`), one fetch at a time, and removes the download whatever happens. `service.rs` `self_update` picks `CheckoutUpdate` with a `build_folder`, else `ReleaseUpdate`. `app_key.rs`: `take_out`: a file that still names `app_dir` has `/app.pem` deleted (a symlink: the link alone, never what it points to), `app_dir` taken out (the file without it checked as a configuration before anything is deleted, then written through `.json.next`) and the deleted file remembered first in `/app-key-file` (0600; a link's target on a second line; `remembered`), which the report's `appKeyFile` names while something is still there; refused, changing nothing, while the file names no console or the host holds no token for it (`KEPT_NO_CONSOLE`, `KEPT_NOT_ENROLLED`). A key deleted whose file could not be written is no failure: `TakenOut::app_dir_kept` says why the file still names the folder. `one_change` is held from the read to the write, by the take-out and by `ConfigFile::change` (the console's settings). `When::Answered` is the service's, once a scale set call of the console this host made was answered (`HeldKey::take_out`); `When::KeyGone` is what `daemon` runs (`start_on`) after `guard` and the state folder's lock (`only_runner`) and before it checks its configuration: it deletes nothing, and takes out only a folder whose key file this Runner's own take-out remembered; a key file missing otherwise keeps the folder (`NOT_TAKEN_OUT_HERE`), and the report's `appKeyKept` says so (`app_key_said`). `runner.entitlements` is signed into it. `config.rs`: `class` (`shared` or `trusted`) and `trusted` (the rules); `check_class` refuses a trusted host that would serve more than one job of one repository, or that holds the GitHub App's key (`app_dir`) or has no `console` to register through. | +| `crates/runner` (`codefusion-runner`) | The binary. `commands.rs`: `version` (the package's version and the commit, which `build.rs` reads from git or `CF_COMMIT`), `image verify`, `image install`, `pause`, `resume` and `status` (`holds.rs`: a named hold on the machine, `--wait` until the Runner saw it and runs nothing, exit 75 when the wait runs out; `status --json`), `vm` (the vm-host, `src/vm.rs`), `boot-proof`, `vmm` (Linux: the cloud-hypervisor the build carries, which `install` and an update ask of a program), and the service's commands in `service.rs` (macOS and Linux): `daemon` (what the service manager runs; ends on SIGTERM; one per state folder, held by a lock on `/runner.lock`), `enroll` (the console's one-time code on standard input, read with the terminal's echo off by `hidden.rs`, which gives the terminal back on every way out, a signal included; the token to `/console-token`, 0600, never printed), `check [--github]` (what an install needs, and the memory at that moment against the budget's floor with the sizes that fit; with `--github` what GitHub shows, changing nothing there and nothing in the state folder), `install` (copies the program to `/bin`, writes the service manager's entry, refuses while a machine stands), `uninstall [--leave] [--purge]` (refused while a Runner started by hand holds the state folder; a Mac without a usable token goes without telling the console). `config.rs` is the configuration file (refused when others can write it; `console` required unless the file names `app_dir` (absolute, not through `..`), the folder of a GitHub App key the host held before the console held it: while its `app.pem` is there `service.rs` `app_and_key` signs with it (`held_key`: `KeyTokens`, and `cf_daemon::HeldKey` for the service), else the GitHub tokens come from the console (`ConsoleTokens`); `check` and `uninstall` sign with a held key and never take it out (`looking`); `use` required on a Mac; on Linux `dedicated` or left out; `max_jobs`, `idle_minutes` (a workstation's only), `min_free_memory_percent` and `vcpus` (a size's vCPUs by its label, which `Settings::limited` checks against the CPUs CI may use), `ci_memory_gb` (the memory CI's machines hold at once, each with its overhead, in GB of 2^30 bytes) and `ci_cpus` (the CPUs they take together) are the owner's `Limits`, which `budget_of` and the service lay over the host's settings; `ConfigFile::change` writes the console's `settings` command into the file through `.json.next`, which must start a Runner before it replaces the file) and, on a Mac, the refusal of the cf-runners manager's own name and of external volumes; `manager.rs` is `ServiceManager`, the seam to the service manager; `launchd.rs` the agent `cc.codefusion.vm-runner` in the person's session (`plist`, `Agent`; `stop` waits past `ExitTimeOut` and decides by whether launchd still has the agent, not by what `bootout` says); `systemd.rs` (Linux) the user unit `codefusion-runner.service` (`unit`, with every path quoted for systemd or refused; `UserUnit` over an injected `systemctl --user`, the real one ended after 100 s; `stop` waits past `TimeoutStopSec` and decides by `ActiveState`); `linux.rs` the lines `check` adds on Linux (`Facts`, `lines`: KVM, lingering, delegated controllers, user namespaces, pasta, `noexec`, each with the root command that fixes it). `update.rs`: `CheckoutUpdate`, the `SelfUpdate` of a host that built the Runner, for the console's `update` and `update-to`: in the checkout the program was built from (`build.rs` writes it down, with `CARGO_HOME`, `RUSTUP_HOME` and `CARGO_BUILD_JOBS`), only when it is the config's `build_folder`, clean and at a commit (`own`), fetches `origin refs/heads/main` (a tag named `main` is not it), takes main's head or the commit `update-to` names when that resolves to one commit of the main fetched (`on_main`), refuses a target that does not contain the running commit (both looked up among the objects alone by `commit_named`, so a branch or tag named like a commit never stands for it), checks out the peeled commit, builds with `cargo xtask build` at low priority into `logs/update.log` and ends a build that takes longer than 45 minutes or outlasts the daemon, and keeps the program it checked as `bin/codefusion-runner.next`; `install` places that one, keeps the running program as `bin/codefusion-runner.prev` and writes `update.json` first; `guard`, run before the daemon reads its configuration, counts the new program's starts and puts the one before back after `MAX_STARTS`, and `ended_in_order` takes back the count of a start that ended as asked. `Installer` is what both updates share: `offer` asks a program its `version` (it must run here and name the commit) and keeps it as `.next`, `install` places it. `release_update.rs` (Linux and macOS builds; Linux hosts only by asset): `ReleaseUpdate`, the `SelfUpdate` of a Runner with no `build_folder`, installed from a release: `takes_head` is false, so it lists only `update-to`; `build` finds the commit's release (`runner--<12 hex>` by tag, else the newest page), waits `RELEASE_WAIT` for one not out yet, asked every `ASK_EVERY` by its tag and every `PAGE_EVERY`th look among the newest too, ended by `end`; refuses a draft, another publisher than `github-actions[bot]`, one not made at the whole commit, not on main or not newer than the running commit (`cf_ghapp::Releases::compare`); fetches `SHA256SUMS` and this host's asset (`asset_for_this_host`) into `.download` (0600), checks its SHA-256 against `SHA256SUMS` (`cf_image::parse_sums`), and removes the download whatever happens; the console shows `fetching its release` (`first_step`). `service.rs` `self_update` picks `CheckoutUpdate` with a `build_folder`, else `ReleaseUpdate`. `app_key.rs`: `take_out`: a file that still names `app_dir` has `/app.pem` deleted (a symlink: the link alone, never what it points to), `app_dir` taken out (the file without it checked as a configuration before anything is deleted, then written through `.json.next`) and the deleted file remembered first in `/app-key-file` (0600; a link's target on a second line; `remembered`), which the report's `appKeyFile` names while something is still there; refused, changing nothing, while the file names no console or the host holds no token for it (`KEPT_NO_CONSOLE`, `KEPT_NOT_ENROLLED`). A key deleted whose file could not be written is no failure: `TakenOut::app_dir_kept` says why the file still names the folder. `one_change` is held from the read to the write, by the take-out and by `ConfigFile::change` (the console's settings). `When::Answered` is the service's, once a scale set call of the console this host made was answered (`HeldKey::take_out`); `When::KeyGone` is what `daemon` runs (`start_on`) after `guard` and the state folder's lock (`only_runner`) and before it checks its configuration: it deletes nothing, and takes out only a folder whose key file this Runner's own take-out remembered; a key file missing otherwise keeps the folder (`NOT_TAKEN_OUT_HERE`), and the report's `appKeyKept` says so (`app_key_said`). `runner.entitlements` is signed into it. `config.rs`: `class` (`shared` or `trusted`) and `trusted` (the rules); `check_class` refuses a trusted host that would serve more than one job of one repository, or that holds the GitHub App's key (`app_dir`) or has no `console` to register through. | | `crates/budget` (`cf-budget`) | Admission, with no I/O: `Budget` (`observe`, `claim`, `confirm`, `assign` and `withdraw` for a job GitHub assigned without asking (`assign_in_turn` and `confirm_in_turn` with its `Turn`: `line` puts main-branch and deploy jobs first, repositories in turn, none passed over past `DEFER_LIMIT`; `set_turn` when what a waiting job stands for changed; a `Turn::refusable` job, a test run, gets room only as a claim would, under the limits in force, and waiting is passed over without holding up those behind it; `make_room` holds back every other size, `making-room`, and puts the size first in line), `adopt`, `usage`, `release`, `hold`, and `status`, which names every slot with how it started and every waiting job with what keeps it), the pure `admit`, `Settings` (every number, `for_host` has the defaults; `limited` lays the owner's `Limits` over them: `max_machines`, refused past it with `Refusal::Machines`, `idle_after`, the floor as a share of memory, which lowers the cap, `ci_bytes`, the memory CI's machines hold at once with each one's overhead (`admit` counts them; `tests/ci_memory.rs`), and `ci_cpus`, the CPUs they take together; `for_headless` those of a Linux host nobody sits at, which reserves what the host needs, `HEADLESS_HOST_USE`, and caps the sizes so that each machine fits with its overhead; `on_fast_cores` gives CI only the fast kind of core of a host whose cores differ; `one_at_a_time` those of a trusted host, which runs one machine on every core it has), `Size` and its labels, the `Host` trait with `HostReading`, `Battery`, what a host's page shows of its battery, and `FakeHost`, the host tests script (`read_takes` lets a read take time before and after the reading gets its time; `tests/fake_host.rs`). Tests: `admission`, `assigned`, `headless`, `limits` (the rule at 4, 7.75, 8, 16 and 32 GiB and its boundaries), `levels`, `making_room`, `races`, `properties`, `turns`. | | `crates/clock` (`cf-clock`) | `now`: the two clocks the budget counts time by, read from the host (Linux: `CLOCK_MONOTONIC` and `CLOCK_BOOTTIME`; macOS: `CLOCK_UPTIME_RAW` and `CLOCK_MONOTONIC_RAW`). Both host crates' `Host::now`. | | `crates/host-macos` (`cf-host-macos`) | `MacHost` reads macOS behind `cf_budget::Host` (page counts, pressure, swap, idle time, thermal, power, disk; what the volume frees on demand, Time Machine's local snapshots and purgeable caches, is its capacity for important use beyond the free space, asked at most every `IMPORTANT_EVERY`, 30 s, for the budget's disk hold and never for an eviction); `Raw` and `reading` are the arithmetic: available memory is what is ready now (free, file cache, purgeable) and, while the Mac is calm (pressure normal, `kern.memorystatus_level` at least `CALM_LEVEL`, the compressor under `HEAVY_COMPRESSOR_SHARE` of memory), what compressing its inactive anonymous pages would free, never past the memorystatus level's share (`tests/reclaim.rs`). For the console's page: `boot_time`, `model` (chip and model, `model_name`), `battery` (what `pmset -g batt` prints, read by `battery_in`, as `cf_budget::Battery`), `host_name`, `load`, `cpu_percent`. `performance_cores` (`hw.perflevel0.logicalcpu` of a chip with two kinds of core, for the budget's CPUs). `program_of` says which program a process runs (the daemon's `is_ours`, `Holder`). `backup`: `exclude_from_backup` marks a folder the Runner made as skipped by Time Machine (the sticky exclusion `tmutil addexclusion` sets, an extended attribute), `exclude_with` is the decision behind the `Attributes` edge; the daemon, `enroll`, `install` and `image install` mark the state folder (the daemon's `Mac::keep_state`, a failure is a logged line, never fatal), and a cache folder to come goes through the same call when it is made. `examples/host-probe.rs` prints a Mac's readings. | | `crates/host-linux` (`cf-host-linux`) | `LinuxHost` reads Linux behind `cf_budget::Host`: MemAvailable, swap, swap-outs (`swapouts_of`: `pswpout`, but with zram in `/proc/swaps` only what is in use on swap that is not zram, since a page in zram stays in memory and PSI says when memory is short; `pswpout` again when `/proc/swaps` cannot be read), PSI (`/proc/pressure/memory`, Normal when the kernel has none), the thermal zones by their own trip points, `/sys/class/power_supply` (a host with no battery runs on the mains; a device's own battery, such as a mouse's, is left out unread), the state folder's disk by `statvfs`; idle time is none on a headless host. Every reader takes a root folder, so tests run against trees of files on any OS. `parse` reads each file, never more than `MAX_FILE_BYTES` (`counter` is one value of a `name value` file, also a cgroup's); `Raw` and `reading` are the arithmetic, with what each field means in the doc comment of `reading`. `fast_cpus` (the kind of core, by `midr_el1`, of the highest `cpu_capacity`, for `Settings::for_headless`). For the console's page: `boot_session` (`boot_id`), `boot_time`, `model` (device tree, else DMI), `battery`, `host_name`, `load`, `cpu_percent`. `process` and `program_of` say which program a process runs and when it started, read twice around the program so a number that went to another process in between names none; `started_at` moves with a step of the wall clock, `started_ticks` with `boot_session` does not. Tests: `fixtures.rs` (an RK3588 board's files, broken in every way), `parse.rs` (with properties), `real.rs` (Linux only: this machine against `free`, `df`, `getconf`). `examples/linux-probe.rs` prints a Linux host's readings and the headless budget. | | `crates/lifecycle` (`cf-lifecycle`) | `Slot`: the state machine of one slot and its job (events in, effects out, watchdogs, `saved` and `restore` for a daemon restart; `on` answers every phase and event with no catch-all arm, keeps a word of a job in every phase, and ends a slot as `no-job` only on `Event::Unregistered`, GitHub's 204; `Asking` is the phase of a slot that asks for its registration's removal, entered at the idle limit or on `Retire`; `Slot::hear` replays GitHub's stored word on a saved slot), and `StartBreaker` (`admit` before a claim, `holds` for the budget's hold). `tests/model.rs` searches every state it can reach. | | `crates/scaleset` (`cf-scaleset`) | The scale set protocol: `ScaleSetClient` (scale sets, sessions, one-job JIT configs, `generate_plain_jit_config` for one outside any scale set, removal of an Actions runner by id and by name (`remove_runner_named`); every administrator's call is the console's `github/scaleset/*`, which holds the Runners App's key and the service's admin token, except on a host that still holds the key (`with_key`, `holds_key`; `key.rs` `KeyAdmin`: its own registration token and admin token, the admin token kept until 120 s before it ends and replaced once on 401, and only the sets named `--…` listed), and on a host whose console is older than those calls (a 404 without a code: `KeyAdmin` with the console's `github/registration` token, `Registrar::Console`, until the Runner starts again; a Runner never needs a console newer than the one that runs; `tests/older_console.rs`); the session keeps the `serviceUrl` the console gave), `runner_busy`, the REST view of whether an Actions runner holds a job, `MessageSession` (long poll, ack, acquire), `listen` with the `Scaler` trait (handle, then ack; duplicates acked only; shutdown interrupts only the wait), `Backoff` and `supervise`. A token goes only to the API's origin and to hosts of `ACTIONS_HOST`: an address the service hands back that is elsewhere is refused. A registration whose answer was lost, or that names no runner, is removed again. Every status and malformed body is an `Error`. `examples/github_probe.rs` is the real check against a throwaway repository (prints no token). Tests in `tests/`, run against `cf-fake-github`; `github_rules.rs` holds the fake to what GitHub was seen to do (`docs/github-protocol.md`). | -| `crates/daemon` (`cf-daemon`) | The service. `holds.rs`: holds set on the machine itself, one file each in `/holds` (`set`, `lift`, `read`, which holds on a damaged file or an unreadable folder), and `/status.json` (`LocalStatus`: the holds the Runner saw, its machines and the ones coming, written on a change and every `STATUS_EVERY`); `Service::look_at_holds`, each tick, takes no new job while any hold or the console's pause is set and reports them as `status.holds`. `machines.rs`: `Machines` joins the budget, the lifecycle of each slot and the vm-hosts: its scale sets share the room their polls offer GitHub (`offer_room`: a free slot one poll offers no other does until that poll ends, `OFFER_STANDS` at most, and sets with jobs queued, `set_demand`, go before those without), so GitHub never assigns more jobs than it can start; a job GitHub assigned is counted (`assign`), started when memory allows (a machine counts with what its vm-host read it holds of the Mac's memory, `count_held`, its job's peak only when that is not known; the log says once when it is not, or is more than the machine is counted with, `Budget::host_bytes`), registered just in time, watched every tick, and ended once with an `EndKind` (`ends.rs`); `restore` adopts what a Runner before left from the records (`records.rs`, `DiskStore`; a record stores facts, `up`, `ran_job`, `completed`, `unregistered`, and carries the `RULES` it was written under; `Record::agrees` says its facts agree, and `Record::upgrade` is the one place that reads an older Runner's record: its beliefs cleared, GitHub's stored word replayed; a current record that disagrees is an error) and the vm-hosts that still answer, in the same start of the Mac (`kern.bootsessionuuid`, not the boot time, which moves with the clock); one that is silent or wrote nothing yet is killed, and so is what has no record. A tick looks at the vm-hosts without the machines' lock, so a vm-host that hangs keeps no listener waiting. What the polls in flight may still offer GitHub is kept (`Offer`): every answer a listener got since all bells last rang, since many scale sets of one size each ask at their own moment. The last slot taken rings their bells at once, room that went or came back with the Mac's memory at a look, at most every 5 s, so no poll in flight goes on offering a slot that is gone, or none while there is one. An open Mac whose other programs leave no memory for a size it serves says so in the log and is closed for `memory` in the console's report (`Snapshot::short_of`); one that would carry the size without its own machines is full, not short. `Snapshot::rooms` is what one more machine of each size meets, for the report's `sizes` and each scale set's capacity: how many fit, the budget's word when none does, `never` for a size the settings never run, and for memory what it needs and lacks. `making.rs`: `Making`, room made for a larger size. A job that waits for the room the Mac's own machines hold, all of them smaller, while the empty Mac would run it, has every other size held back until it starts, for `MAKE_ROOM_LIMIT` (30 min) at most and then none for as long; its scale set offers a place in line while GitHub has the job back, for `TAKEN_BACK_WAIT` (2 min); an empty Mac whose other programs hold the room gives up after `NO_ROOM_WAIT`. A size whose polls found no room for `STARVED_AFTER` (10 min) is offered a place before any of its jobs is assigned; a size the empty Mac cannot run never is. A headless host, one of a pool, makes room and offers a place only for the size the console's answer names it (`makeRoomFor`, `Making::name`, `Machines::make_room_for`), at once. `vms.rs`: `Vms`, the seam to `codefusion-runner vm`, and `VmHosts`, the real one. It writes each vm-host's process and start time to `run/.pid` (0600) before it hands it the job, so the Runner that starts next can end one that hangs before its first state. A signal goes only to a child whose exit was not collected, to a process that answers on the job's socket, or to one that runs the Runner's binary, is not the Runner and started no later than the time written down for it (`is_ours`, through `Platform::process`): a number that went to a later process is sent nothing. `service.rs`: `Service` (`discover` repositories and labels and, on a host that still holds the App's key (`HeldKey`), asks the console's scale set calls about each served repository in turn (the next after a refusal of one, none after a console that cannot be reached or has no such calls): once one answered, `HeldKey::take_out` and a start again, else the key serves and the report's `appKeyKept` says why, one supervised listener per scale set, `sweep` of registrations nothing stands behind, `report` to the console with the job events of the outbox and how long jobs handed here waited while nothing ran, with a WARN after `STALL_WARN_AFTER`; the sweep is the one place that removes a registration: those of slots in `Asking`, and of machines that ended while theirs stands, after `RESULT_WAIT` for one whose guest came up with no word of a job; each answer goes to `Machines::answered(runner, Answer)`: `Removed` (204) is no job, `Gone` (404) under a guest that came up is a job over, `Refused` (422) a job that runs, a job implied so is stored as inferred, `discover` and `look_at_queue` run in a task beside `run`'s loop, so a slow GitHub holds up no report and no stop; `look_at_queue`, about once a minute, serves the label sets of queued and running jobs that the default branches' workflows do not name, under `queued.rs`'s `Queued::select`: each for `queued_keep` (30 min) after a job of it was last seen, `queued_bounds` (8 a repository, 32 a host) the newest first, none of a repository it does not serve, none of a fork's pull request on a public repository, and none dropped while that repository's queue cannot be read (for up to `LONGEST_WAIT`, 24 h); a change lists the scale sets of only the repositories whose sets changed; `survey` for `check`, `retire` for `uninstall`, `run`). Each look at the queue also finds the jobs GitHub holds back from a scale set with room (`held.rs`, pure: queued two minutes while every poll of the set offered room beyond GitHub's assigned count for as long, `SetState::room_since`), names them in the report's `heldJobs`, and gives the set one Actions runner of its own at a time, one in five minutes at most and none while a queued job of the repository asks for only some of its labels (`held::narrower_waits`), in a slot the budget gives: registered outside the scale set (`SetRegistry::own`, scale set number `held::OWN`, 0) with exactly its labels and started by `scaler::start_runner`, as a scale set's are; it is retired once its set's queue holds no job it could take, or after five minutes. What the Runner believes about each machine and its job, which evidence moves it and what is written before what is done, is `docs/job-state.md`. `scaler.rs`: `Scaler` (reserve before acquire, release for jobs taken elsewhere, converge on assigned jobs, each place in line with the turn of the assigned job it stands for (`Slots::set_turn`), offers taken overdue first, then main and deploy jobs; `capacity` offers a paused or held Mac's polls only what runs and what was promised (`Slots::held`), and no room to a job in line `PLACE_LIMIT` (10 min) after it was first assigned here, taken back and handed here again or not (one away as long is a re-run, and new), so GitHub's take-back sends it to another machine, retire surplus after `IDLE_GRACE` (`Hosts::retire`: idle machines ask for their registration's removal, which the sweep does); before a session opens, `reconcile` asks GitHub's REST view of each Actions runner it believes waits, since messages go with a closed session, and a busy one took a job (`Answer::Busy`); a session opens only at start, after GitHub forgot one, after a listener failed, or after a restart), `Roster` (an Actions runner belongs to a repository's scale set: GitHub numbers scale sets within a repository, so the number alone names none), `turns.rs`: `turn_of`, whether a job is the main branch's or a deploy, from its workflow's ref, name and event. `names` (runner and scale set names, `ServePolicy`; a scale set's name ends in a hash of its repository and labels, so two repositories that ask for the same labels each get their own). `update.rs`: `SettingsFile` (what the Runner's configuration file says now against what it started with, `SettingsLook`; `change` writes the console's `settings` command, read by `SettingsChange::from_args`, which names each of the cf-runners manager's settings the Runner does not take; `Service::look_at_settings` takes a new one up as an update does, waiting in `drained`, the same wait, for no machine to stand, then starting again), `SelfUpdate` (`build` takes the commit `update-to` names, or none for main's head; `takes_head` false lists no plain `update` and refuses one with `NO_HEAD`; `end` stops a build or a wait when the daemon ends), `commit_from_args`, the console's `update-to` arguments (`{"commit"}` alone, `is_commit`: 7 to 40 lowercase hex characters), `Pending` in `update.json`, `at_start` (`Trying`, `GoBack` after `MAX_STARTS` starts that did not stay up for `STAYS_UP`, `Failed`, also when a program before could not be put back: `Pending::failed`); the service builds while jobs run, then holds new jobs until no machine stands and no slot promised to GitHub is left (`Machines::coming`), installs and ends (`update`, `restart`; one at a time, the result said once by the Runner that starts next). `platform/`: `host_service`, the service on a real host, which the binary and `cargo xtask it` both build, over a `Platform` (`This` is this build's): what differs between systems and nothing else: the host's reader and the budget's settings (`Settings::for_host` on the performance cores, `on_fast_cores` with `cf_host_macos::performance_cores`; on Linux `for_headless` with the fast cores), the boot session, host name, model and what the console's page shows (`extras`), `keep_state` (Time Machine; nothing on Linux), `fit` (the virtualization entitlement; on Linux `codefusion-runner vmm` says the build carries cloud-hypervisor), `process` (program and start, for `is_ours`; on Linux `cf_host_linux::process`), `program` (the program alone, for a kill's wait), `vm_hosts` and `SERVICE_LOG`. `mac.rs` is macOS, `linux.rs` Linux, where each vm-host starts in a systemd scope of its own (`VmHosts::in_scopes`, `systemd-run --user --scope -p Delegate=yes`, `codefusion-runner-.scope`), whose delegated cgroup it claims for its machines, and `user_bus_env` finds the user's systemd under `sudo -iu`, and not another user's that `su` left in the environment (`this_user` names the user from the user database). `tests/scopes.rs` (Linux, no harness) starts a stand-in vm-host so and checks it claims the scope's cgroup. `test_run/`: the console's `test-run` command (`TestRun::from_args`, its arguments checked, the run's `url` on the Runner's own console; `admit`, an urgent claim of the budget's as `test-run` that goes before the CI jobs in line, busy while the host takes no new job, a test run is on it or a Docker run finds the Docker jobs full; `wait_turn` for `waitS`, withdrawn as soon as the console's answer no longer carries the command), started by `Machines::start_test_run` as a machine with no Actions runner: its record is `unregistered` from the start with `test_run` set, its job starts when its guest is up and ends with the machine, and `Service::test_run` answers `started tr-…`, `busy: …` or `never: …`, later for a run that waits. `agent.mjs` is the workload (`workload()`: `node --input-type=module -e `, env `CF_RUN_URL`, `CF_RUN_TOKEN`, `CF_RUN_TIMEOUT_S`), the agent that fetches the bundles, runs the suite in its own process group, spools its output decoded as UTF-8 (U+FFFD for what is not) and sends it to the console; `agent.test.mjs` its `node --test` tests against a fake console (CI's Linux job). Tests: `tests/test_run.rs`, and in `tests/service.rs` the command end to end. `docker_check.rs`: whether Docker starts in this host's job machines (`DockerCheck`: `of_end` reads the check machine's end and its console log's "Docker sandbox ended" line, `due` at the start and every `RECHECK_AFTER` (6 h) while it fails, `None` from `of_end` for a machine that ended for something else (slept, evicted, stopped), checked again after `AGAIN_AFTER_NO_ANSWER` (10 min, `Machines::docker_check_due`), `refusal` while it failed; a failure is kept across starts by the `Store`, `docker_failure` and `keep_docker_failure`, in `DiskStore`'s `docker-check.failed`, a pass is not); `Machines::start_docker_probe` boots the check's machine in a claimed slot (no Actions runner, `Record::probe`, nothing reported), `Service::look_at_docker` on every tick, and a failed check drops the `cf-docker` sets (`size_here`) and answers a test run with Docker `never:`; tests `tests/docker_check.rs` and in `tests/service.rs`. `log.rs`: `Lines`, the log file that is rolled over. `testing` has the fakes, and `CrashingStore`, records at which a test makes the daemon die between any two writes. Tests: `machines.rs` (every end kind, a crash at every step), `records.rs` (every record main and earlier builds could have written, read by `Record::upgrade`), `interleavings.rs` (the property test), `schedules.rs` (the property test of the job facts against a GitHub that keeps the truth: crashes at any write, every answer to a removal, lost and late messages, records left in older formats; with pinned schedules), `offers.rs` (the property test of what several listeners of one size offer), `making_room.rs` (the mini's cf-6g job against the cf-2g ones that keep coming), `calm.rs` (the calm minute against time that passes within a look, and against everything that is really wrong), `service.rs` (GitHub's and the console's failures; the label sets of queued jobs: served, kept 30 min, bounded, refused, unreadable, 304, a fork's on a public repository; the property test that no job gets two machines, a session opens only for a cause, a job that waits while a machine fits moves at GitHub's next look, and every job that ran is heard, start and end once each, however many messages went with a session, with a stale or failing REST view, a crash between any two writes, five minutes that take an assignment back, removals that fail until GitHub recovers, and an uninstall in the middle; machines that boot slowly, so a start or an end comes before the Runner saw one up; no VM whose job runs is stopped, and every single write's facts agree (`CrashingStore::disagreements`); a second property aims the same checks at surplus removals), `scaler.rs`, `listening.rs`, `docker.rs` (a Docker job's daemon flag, the cap of `docker_max_jobs` in `Machines`, and `ServePolicy::serves_docker`: listed and private only). A trusted host (`ServePolicy::trusted`) makes no scale set: `Service::ask_trusted` asks for one Actions runner with the label `trusted` at a time (`trusted_every`, 15 s), in the largest machine it runs, whose `JobRequest::trusted` carries the rules; a 409 with `Retry-After` (the console's `no_trusted_job`) is a wait. It takes no test run, boots no Docker check and reports `class: trusted`. | -| `crates/ghapp` (`cf-ghapp`) | `release.rs`: `Releases`, a public repository's releases read with no token (`by_tag`, `page`, `branch_head`, `compare`), and `download`, a release's file streamed into a `Sink` and hashed as it is written, followed across GitHub's redirect to its storage with no token, begun again from the first byte after a break, a 429 or a 5xx, ended at once by a full disk (`Error::Local`), a size over its limit or the stop flag (`tests/release.rs`). The GitHub App client. Its tokens come from a `TokenSource` (`source.rs`), the console's (`cf_console_client::ConsoleTokens`), through `GitHubApp::with_source`: no machine holds the App's key, a registration token or the Actions service's admin token, except one that still holds the key until the console's scale set calls answered it (`KeyTokens`, `key.rs`: the App JWT of `jwt.rs`, installation tokens and `registration_token`; `AppCredentials::load` in `credentials.rs`, `app.json` and `app.pem`, which must be 0600; tests in `tests/key.rs`); a call about a repository passes it, so a source whose installation IDs differ finds the one that covers it; `Monotonic` is the clock a token's lifetime is counted on. `rest.rs` is the REST plumbing both share (a 401 tells the source and tries once more). The clock skew is learned from the `Date` header (`clock.rs`). Repositories, workflow files (the caller names the largest file to read), repository variables, `waiting_jobs` (the jobs of a repository's queued and running runs of the label sets served, `cf_workflows::LabelSet::served`, from the REST view, each saying whether its run is a fork's, under bounds: `WAITING_RUNS` of the newest `RUNS_LISTED` runs, queued ones before those under way and each the oldest first, `JOBS_PAGE` jobs of one, `MAX_RUN_SETS` distinct sets of one run; the listings asked for again with the ETag of their last answer, so a 304 reuses it, and each run's jobs asked for again every look with theirs (a run's `updated_at` stays put while its jobs start and end), and `forget_queues` drops what repositories no longer served last read as; each set names its queued jobs, the oldest first), `generate_jit_config` (an Actions runner for one job outside any scale set, with exactly the labels given, sent once); `Http::new(timeout, retry)` with retries (429 and 5xx with jitter, `Retry-After`), HTTPS only (or this machine itself), redirects only to the same origin, at most `MAX_BODY_BYTES` of an answer and 100 pages of a listing. No error and no `Debug` carries an answer's text. Tests in `tests/app.rs`, run against `cf-fake-github`. | +| `crates/daemon` (`cf-daemon`) | The service. `holds.rs`: holds set on the machine itself, one file each in `/holds` (`set`, `lift`, `read`, which holds on a damaged file or an unreadable folder), and `/status.json` (`LocalStatus`: the holds the Runner saw, its machines and the ones coming, written on a change and every `STATUS_EVERY`); `Service::look_at_holds`, each tick, takes no new job while any hold or the console's pause is set and reports them as `status.holds`. `machines.rs`: `Machines` joins the budget, the lifecycle of each slot and the vm-hosts: its scale sets share the room their polls offer GitHub (`offer_room`: a free slot one poll offers no other does until that poll ends, `OFFER_STANDS` at most, and sets with jobs queued, `set_demand`, go before those without), so GitHub never assigns more jobs than it can start; a job GitHub assigned is counted (`assign`), started when memory allows (a machine counts with what its vm-host read it holds of the Mac's memory, `count_held`, its job's peak only when that is not known; the log says once when it is not, or is more than the machine is counted with, `Budget::host_bytes`), registered just in time, watched every tick, and ended once with an `EndKind` (`ends.rs`); `restore` adopts what a Runner before left from the records (`records.rs`, `DiskStore`; a record stores facts, `up`, `ran_job`, `completed`, `unregistered`, and carries the `RULES` it was written under; `Record::agrees` says its facts agree, and `Record::upgrade` is the one place that reads an older Runner's record: its beliefs cleared, GitHub's stored word replayed; a current record that disagrees is an error) and the vm-hosts that still answer, in the same start of the Mac (`kern.bootsessionuuid`, not the boot time, which moves with the clock); one that is silent or wrote nothing yet is killed, and so is what has no record. A tick looks at the vm-hosts without the machines' lock, so a vm-host that hangs keeps no listener waiting. What the polls in flight may still offer GitHub is kept (`Offer`): every answer a listener got since all bells last rang, since many scale sets of one size each ask at their own moment. The last slot taken rings their bells at once, room that went or came back with the Mac's memory at a look, at most every 5 s, so no poll in flight goes on offering a slot that is gone, or none while there is one. An open Mac whose other programs leave no memory for a size it serves says so in the log and is closed for `memory` in the console's report (`Snapshot::short_of`); one that would carry the size without its own machines is full, not short. `Snapshot::rooms` is what one more machine of each size meets, for the report's `sizes` and each scale set's capacity: how many fit, the budget's word when none does, `never` for a size the settings never run, and for memory what it needs and lacks. `making.rs`: `Making`, room made for a larger size. A job that waits for the room the Mac's own machines hold, all of them smaller, while the empty Mac would run it, has every other size held back until it starts, for `MAKE_ROOM_LIMIT` (30 min) at most and then none for as long; its scale set offers a place in line while GitHub has the job back, for `TAKEN_BACK_WAIT` (2 min); an empty Mac whose other programs hold the room gives up after `NO_ROOM_WAIT`. A size whose polls found no room for `STARVED_AFTER` (10 min) is offered a place before any of its jobs is assigned; a size the empty Mac cannot run never is. A headless host, one of a pool, makes room and offers a place only for the size the console's answer names it (`makeRoomFor`, `Making::name`, `Machines::make_room_for`), at once. `vms.rs`: `Vms`, the seam to `codefusion-runner vm`, and `VmHosts`, the real one. It writes each vm-host's process and start time to `run/.pid` (0600) before it hands it the job, so the Runner that starts next can end one that hangs before its first state. A signal goes only to a child whose exit was not collected, to a process that answers on the job's socket, or to one that runs the Runner's binary, is not the Runner and started no later than the time written down for it (`is_ours`, through `Platform::process`): a number that went to a later process is sent nothing. `service.rs`: `Service` (`discover` repositories and labels and, on a host that still holds the App's key (`HeldKey`), asks the console's scale set calls about each served repository in turn (the next after a refusal of one, none after a console that cannot be reached or has no such calls): once one answered, `HeldKey::take_out` and a start again, else the key serves and the report's `appKeyKept` says why, one supervised listener per scale set, `sweep` of registrations nothing stands behind, `report` to the console with the job events of the outbox and how long jobs handed here waited while nothing ran, with a WARN after `STALL_WARN_AFTER`; the sweep is the one place that removes a registration: those of slots in `Asking`, and of machines that ended while theirs stands, after `RESULT_WAIT` for one whose guest came up with no word of a job; each answer goes to `Machines::answered(runner, Answer)`: `Removed` (204) is no job, `Gone` (404) under a guest that came up is a job over, `Refused` (422) a job that runs, a job implied so is stored as inferred, `discover` and `look_at_queue` run in a task beside `run`'s loop, so a slow GitHub holds up no report and no stop; `look_at_queue`, about once a minute, serves the label sets of queued and running jobs that the default branches' workflows do not name, under `queued.rs`'s `Queued::select`: each for `queued_keep` (30 min) after a job of it was last seen, `queued_bounds` (8 a repository, 32 a host) the newest first, none of a repository it does not serve, none of a fork's pull request on a public repository, and none dropped while that repository's queue cannot be read (for up to `LONGEST_WAIT`, 24 h); a change lists the scale sets of only the repositories whose sets changed; `survey` for `check`, `retire` for `uninstall`, `run`). Each look at the queue also finds the jobs GitHub holds back from a scale set with room (`held.rs`, pure: queued two minutes while every poll of the set offered room beyond GitHub's assigned count for as long, `SetState::room_since`), names them in the report's `heldJobs`, and gives the set one Actions runner of its own at a time, one in five minutes at most and none while a queued job of the repository asks for only some of its labels (`held::narrower_waits`), in a slot the budget gives: registered outside the scale set (`SetRegistry::own`, scale set number `held::OWN`, 0) with exactly its labels and started by `scaler::start_runner`, as a scale set's are; it is retired once its set's queue holds no job it could take, or after five minutes. What the Runner believes about each machine and its job, which evidence moves it and what is written before what is done, is `docs/job-state.md`. `scaler.rs`: `Scaler` (reserve before acquire, release for jobs taken elsewhere, converge on assigned jobs, each place in line with the turn of the assigned job it stands for (`Slots::set_turn`), offers taken overdue first, then main and deploy jobs; `capacity` offers a paused or held Mac's polls only what runs and what was promised (`Slots::held`), and no room to a job in line `PLACE_LIMIT` (10 min) after it was first assigned here, taken back and handed here again or not (one away as long is a re-run, and new), so GitHub's take-back sends it to another machine, retire surplus after `IDLE_GRACE` (`Hosts::retire`: idle machines ask for their registration's removal, which the sweep does); before a session opens, `reconcile` asks GitHub's REST view of each Actions runner it believes waits, since messages go with a closed session, and a busy one took a job (`Answer::Busy`); a session opens only at start, after GitHub forgot one, after a listener failed, or after a restart), `Roster` (an Actions runner belongs to a repository's scale set: GitHub numbers scale sets within a repository, so the number alone names none), `turns.rs`: `turn_of`, whether a job is the main branch's or a deploy, from its workflow's ref, name and event. `names` (runner and scale set names, `ServePolicy`; a scale set's name ends in a hash of its repository and labels, so two repositories that ask for the same labels each get their own). `update.rs`: `SettingsFile` (what the Runner's configuration file says now against what it started with, `SettingsLook`; `change` writes the console's `settings` command, read by `SettingsChange::from_args`, which names each of the cf-runners manager's settings the Runner does not take; `Service::look_at_settings` takes a new one up as an update does, waiting in `drained`, the same wait, for no machine to stand, then starting again), `SelfUpdate` (`build` takes the commit `update-to` names, or none for main's head; `takes_head` false lists no plain `update` and refuses one with `NO_HEAD`; `end` stops a build or a wait when the daemon ends; `first_step` is what the console shows meanwhile), `commit_from_args`, the console's `update-to` arguments (`{"commit"}` alone, `is_commit`: 7 to 40 lowercase hex characters), `Pending` in `update.json`, `at_start` (`Trying`, `GoBack` after `MAX_STARTS` starts that did not stay up for `STAYS_UP`, `Failed`, also when a program before could not be put back: `Pending::failed`); the service builds while jobs run, then holds new jobs until no machine stands and no slot promised to GitHub is left (`Machines::coming`), installs and ends (`update`, `restart`; one at a time, the result said once by the Runner that starts next). `platform/`: `host_service`, the service on a real host, which the binary and `cargo xtask it` both build, over a `Platform` (`This` is this build's): what differs between systems and nothing else: the host's reader and the budget's settings (`Settings::for_host` on the performance cores, `on_fast_cores` with `cf_host_macos::performance_cores`; on Linux `for_headless` with the fast cores), the boot session, host name, model and what the console's page shows (`extras`), `keep_state` (Time Machine; nothing on Linux), `fit` (the virtualization entitlement; on Linux `codefusion-runner vmm` says the build carries cloud-hypervisor), `process` (program and start, for `is_ours`; on Linux `cf_host_linux::process`), `program` (the program alone, for a kill's wait), `vm_hosts` and `SERVICE_LOG`. `mac.rs` is macOS, `linux.rs` Linux, where each vm-host starts in a systemd scope of its own (`VmHosts::in_scopes`, `systemd-run --user --scope -p Delegate=yes`, `codefusion-runner-.scope`), whose delegated cgroup it claims for its machines, and `user_bus_env` finds the user's systemd under `sudo -iu`, and not another user's that `su` left in the environment (`this_user` names the user from the user database). `tests/scopes.rs` (Linux, no harness) starts a stand-in vm-host so and checks it claims the scope's cgroup. `test_run/`: the console's `test-run` command (`TestRun::from_args`, its arguments checked, the run's `url` on the Runner's own console; `admit`, an urgent claim of the budget's as `test-run` that goes before the CI jobs in line, busy while the host takes no new job, a test run is on it or a Docker run finds the Docker jobs full; `wait_turn` for `waitS`, withdrawn as soon as the console's answer no longer carries the command), started by `Machines::start_test_run` as a machine with no Actions runner: its record is `unregistered` from the start with `test_run` set, its job starts when its guest is up and ends with the machine, and `Service::test_run` answers `started tr-…`, `busy: …` or `never: …`, later for a run that waits. `agent.mjs` is the workload (`workload()`: `node --input-type=module -e `, env `CF_RUN_URL`, `CF_RUN_TOKEN`, `CF_RUN_TIMEOUT_S`), the agent that fetches the bundles, runs the suite in its own process group, spools its output decoded as UTF-8 (U+FFFD for what is not) and sends it to the console; `agent.test.mjs` its `node --test` tests against a fake console (CI's Linux job). Tests: `tests/test_run.rs`, and in `tests/service.rs` the command end to end. `docker_check.rs`: whether Docker starts in this host's job machines (`DockerCheck`: `of_end` reads the check machine's end and its console log's "Docker sandbox ended" line, `due` at the start and every `RECHECK_AFTER` (6 h) while it fails, `None` from `of_end` for a machine that ended for something else (slept, evicted, stopped), checked again after `AGAIN_AFTER_NO_ANSWER` (10 min, `Machines::docker_check_due`), `refusal` while it failed; a failure is kept across starts by the `Store`, `docker_failure` and `keep_docker_failure`, in `DiskStore`'s `docker-check.failed`, a pass is not); `Machines::start_docker_probe` boots the check's machine in a claimed slot (no Actions runner, `Record::probe`, nothing reported), `Service::look_at_docker` on every tick, and a failed check drops the `cf-docker` sets (`size_here`) and answers a test run with Docker `never:`; tests `tests/docker_check.rs` and in `tests/service.rs`. `log.rs`: `Lines`, the log file that is rolled over. `testing` has the fakes, and `CrashingStore`, records at which a test makes the daemon die between any two writes. Tests: `machines.rs` (every end kind, a crash at every step), `records.rs` (every record main and earlier builds could have written, read by `Record::upgrade`), `interleavings.rs` (the property test), `schedules.rs` (the property test of the job facts against a GitHub that keeps the truth: crashes at any write, every answer to a removal, lost and late messages, records left in older formats; with pinned schedules), `offers.rs` (the property test of what several listeners of one size offer), `making_room.rs` (the mini's cf-6g job against the cf-2g ones that keep coming), `calm.rs` (the calm minute against time that passes within a look, and against everything that is really wrong), `service.rs` (GitHub's and the console's failures; the label sets of queued jobs: served, kept 30 min, bounded, refused, unreadable, 304, a fork's on a public repository; the property test that no job gets two machines, a session opens only for a cause, a job that waits while a machine fits moves at GitHub's next look, and every job that ran is heard, start and end once each, however many messages went with a session, with a stale or failing REST view, a crash between any two writes, five minutes that take an assignment back, removals that fail until GitHub recovers, and an uninstall in the middle; machines that boot slowly, so a start or an end comes before the Runner saw one up; no VM whose job runs is stopped, and every single write's facts agree (`CrashingStore::disagreements`); a second property aims the same checks at surplus removals), `scaler.rs`, `listening.rs`, `docker.rs` (a Docker job's daemon flag, the cap of `docker_max_jobs` in `Machines`, and `ServePolicy::serves_docker`: listed and private only). A trusted host (`ServePolicy::trusted`) makes no scale set: `Service::ask_trusted` asks for one Actions runner with the label `trusted` at a time (`trusted_every`, 15 s), in the largest machine it runs, whose `JobRequest::trusted` carries the rules; a 409 with `Retry-After` (the console's `no_trusted_job`) is a wait. It takes no test run, boots no Docker check and reports `class: trusted`. | +| `crates/ghapp` (`cf-ghapp`) | `release.rs`: `Releases`, a public repository's releases read with no token (`by_tag`, `page`, `branch_head`, `compare`), and `download`, a release's file streamed into a `Sink` and hashed as it is written, followed across GitHub's redirect to its storage with no token, begun again from the first byte after a break, a 429, a used-up rate limit or a 5xx, ended at once by a full disk (`Error::Local`), a size over its limit or the stop flag, also while a connection stalls or a retry waits (`tests/release.rs`). `http.rs` `retry_after` reads `Retry-After`, else a used-up rate limit's `x-ratelimit-reset`. The GitHub App client. Its tokens come from a `TokenSource` (`source.rs`), the console's (`cf_console_client::ConsoleTokens`), through `GitHubApp::with_source`: no machine holds the App's key, a registration token or the Actions service's admin token, except one that still holds the key until the console's scale set calls answered it (`KeyTokens`, `key.rs`: the App JWT of `jwt.rs`, installation tokens and `registration_token`; `AppCredentials::load` in `credentials.rs`, `app.json` and `app.pem`, which must be 0600; tests in `tests/key.rs`); a call about a repository passes it, so a source whose installation IDs differ finds the one that covers it; `Monotonic` is the clock a token's lifetime is counted on. `rest.rs` is the REST plumbing both share (a 401 tells the source and tries once more). The clock skew is learned from the `Date` header (`clock.rs`). Repositories, workflow files (the caller names the largest file to read), repository variables, `waiting_jobs` (the jobs of a repository's queued and running runs of the label sets served, `cf_workflows::LabelSet::served`, from the REST view, each saying whether its run is a fork's, under bounds: `WAITING_RUNS` of the newest `RUNS_LISTED` runs, queued ones before those under way and each the oldest first, `JOBS_PAGE` jobs of one, `MAX_RUN_SETS` distinct sets of one run; the listings asked for again with the ETag of their last answer, so a 304 reuses it, and each run's jobs asked for again every look with theirs (a run's `updated_at` stays put while its jobs start and end), and `forget_queues` drops what repositories no longer served last read as; each set names its queued jobs, the oldest first), `generate_jit_config` (an Actions runner for one job outside any scale set, with exactly the labels given, sent once); `Http::new(timeout, retry)` with retries (429 and 5xx with jitter, `Retry-After`), HTTPS only (or this machine itself), redirects only to the same origin, at most `MAX_BODY_BYTES` of an answer and 100 pages of a listing. No error and no `Debug` carries an answer's text. Tests in `tests/app.rs`, run against `cf-fake-github`. | | `crates/workflows` (`cf-workflows`) | `self_hosted` / `self_hosted_with`: the label sets a repository's workflows ask for (port of the Go `workflows` package, tests in `tests/labels.rs`), `size_gb` for the `cf-2g/4g/6g/8g` labels, `self-hosted` = 4 GB, `DOCKER_LABEL` (`cf-docker`, served alone or with a size), `is_host_label` and `LabelSet::host_labels` (`cf-macos` and the like, served only by hosts whose config `labels` has them, and a host whose `only_labels` names some of them serves only the sets that ask for one: `ServePolicy::serves_labels` and `labels_refusal` in `crates/daemon/src/names.rs`), the `vars.*` expressions evaluated with the repository's variables by `expression.rs`, a reader of GitHub's expressions over variables and literals: `||`, parentheses, `format` (`{0}`, `{{`/`}}` escapes), `fromJSON` and `toJSON`, nested at most `MAX_DEPTH` deep, no value over `MAX_VALUE_BYTES` (a JSON list under `fromJSON`, with no `toJSON` anywhere in the expression, is several labels; what it cannot read stands for the pool label), `LabelSet::served`, the rules every set to serve keeps, a workflow's or a queued job's (one label of ours, no other platform, at most `MAX_SET_LABELS` labels of at most `MAX_LABEL_BYTES`, letters, digits, `-`, `_` and `.` only), and `MAX_FILE_BYTES`, the largest workflow file read. `LabelSet::served` refuses every set with `TRUSTED_LABEL` in any spelling GitHub's case-blind match takes for it (`is_trusted_label`: `TRUSTED`, `truſted`), from a list or a variable: no shared host takes a trusted job. | | `crates/secret` (`cf-secret`) | `Secret`: a value whose `Debug` prints `Secret(..)`. Every token and key is wrapped in it. `replace_private`: a file for this user alone (0600), put in place in one step; `replace_private_as` with a mode of the user's own bits (0500 for a program). `private_dir`: a folder for this user alone (0700). | | `crates/fake-github` (`cf-fake-github`) | A stateful fake GitHub over real sockets, for tests only: App JWT checks (`test_key`), installation tokens, repositories, scale sets, sessions, long poll, acquire, JIT configs, with `Options::auto_assign` (no kinder than the real service, each rule beside its evidence in `docs/github-protocol.md`: jobs go out only when a job is queued, ends, is cancelled or taken, or a test says GitHub looks (`github_looks`) or an Actions runner registered outside any scale set comes online (`runner_online`; `plain_runners` lists them, and `start_job` gives one the oldest queued job of its repository whose labels it has), against the smaller room of a scale set's last two polls; an opening hands out nothing; the counts never name a held job; a session that ends takes its pending messages; `offers_room` says when a job of a repository queued now goes out at once, `offers_room_for` when a job that waits goes out at GitHub's next look; only a session's newest poll is served), an assignment no Actions runner took is taken back after `TAKE_BACK_AFTER` on the queue's clock (`pass`), with faults (`fail`, `before`, `lose_answer`, `lose_session`, `expire_session_token`, `revoke_admin_tokens`, `send_the_service_to`, `send_the_queue_to`, clock offset, `set_rest_stale`: the REST view of an Actions runner one read behind, `set_down`: every request dropped, as for a client that died), and `assignments`, the hand-outs so far. What the console does with its own App's key, for `cf-fake-console`: `mint_read_token` (a token GitHub refuses every write, 403), `mint_registration_token`, `mint_plain_jit`, `revoke_token`, `token_works`, `revoke_installation_tokens`. `TestClock` is the wall clock and the monotonic one. `queue_job_at` queues a job at a ref, which its message's `jobWorkflowRef` and `eventName` show; `queue_run` the jobs of one run; `end_job` ends one as its run shows it; `from_fork` makes a job's run a pull request's from a fork. The REST view of runs and their jobs (`/actions/runs?status=`, `/actions/runs/{id}/jobs`) answers with an ETag and 304 to a request that names it (`not_modified` counts them). | diff --git a/scripts/host-setup.sh b/scripts/host-setup.sh index 8fcb879..171efc3 100755 --- a/scripts/host-setup.sh +++ b/scripts/host-setup.sh @@ -608,7 +608,7 @@ place_binary() { trap - ERR as_user "'$installed' version" 2>/dev/null ) || runs='' - runs=${runs#"$USER_NAME "} + runs=${runs#codefusion-runner } case $(runs_newer "$REPO_API" "${runs##*+}" "${version##*-}") in keep) say "it runs $runs, newer on main than $named: it stays, and $named waits for none" From d28e92365bc9a24194c1d6e19b278f5fd392e878 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krystian=20Paw=C5=82ow?= Date: Thu, 12 Nov 2026 21:15:11 +0100 Subject: [PATCH 3/6] Share the retry rule, the stop-aware sleep and the GitHub reads - Http::send and a release download take their waits from one RetryPolicy::wait_after; the file client sits beside the API client. - One sleep_unless for every wait the Runner's end cuts short. - Downloaded carries its SHA-256 in hex; one check of plain names. - SelfUpdate says where its updates come from (UpdateFrom), which decides the plain update and what the console shows, in place of two methods. - host-setup.sh's compare_status and main_head serve runs_newer and the release gate, which reads the push's runs in one call. - The release takes both build artifacts in one download. --- .github/workflows/release.yml | 19 ++-- crates/daemon/src/service.rs | 8 +- crates/daemon/src/testing.rs | 10 +- crates/daemon/src/update.rs | 30 ++++-- crates/ghapp/src/http.rs | 57 ++++++++--- crates/ghapp/src/lib.rs | 2 +- crates/ghapp/src/release.rs | 91 +++++++---------- crates/ghapp/tests/release.rs | 4 +- crates/runner/src/release_update.rs | 79 +++++++-------- docs/MAP.md | 8 +- scripts/host-setup.sh | 43 +++++---- scripts/release-gate.sh | 22 +++-- xtask/tests/host_setup.rs | 145 ++++++++++++++-------------- 13 files changed, 278 insertions(+), 240 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1d5530f..21af9f1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -97,18 +97,17 @@ jobs: - name: The build's programs run: | set -euo pipefail - for platform in amd64 arm64; do - for try in 1 2 3 4 5; do - rm -rf "in/$platform" - if gh run download "$BUILD" --repo "$GITHUB_REPOSITORY" --name "runner-$platform" --dir "in/$platform"; then - break - fi - ((try < 5)) || { echo "::error::run $BUILD's runner-$platform could not be fetched"; exit 1; } - sleep $((try * 15)) - done + # Both artifacts in one call, each in a folder of its name. + for try in 1 2 3 4 5; do + rm -rf in + if gh run download "$BUILD" --repo "$GITHUB_REPOSITORY" --pattern 'runner-*' --dir in; then + break + fi + ((try < 5)) || { echo "::error::the programs of run $BUILD could not be fetched"; exit 1; } + sleep $((try * 15)) done mkdir out - cp in/amd64/* in/arm64/* out/ + cp in/runner-amd64/* in/runner-arm64/* out/ - name: Publish run: | set -euo pipefail diff --git a/crates/daemon/src/service.rs b/crates/daemon/src/service.rs index 72b5d44..b1a1678 100644 --- a/crates/daemon/src/service.rs +++ b/crates/daemon/src/service.rs @@ -2470,7 +2470,7 @@ impl Service { let shared = &self.shared; let mut kinds = vec!["pause", "resume", "restart", "stop-runner"]; if let Some(updates) = &shared.updates { - if updates.takes_head() { + if updates.updates_from().takes_head() { kinds.push("update"); } kinds.push("update-to"); @@ -2694,7 +2694,9 @@ impl Service { return Some((false, "this Runner has nowhere to update from".into())); }; let commit = match commit { - Ok(None) if !updates.takes_head() => return Some((false, crate::update::NO_HEAD.into())), + Ok(None) if !updates.updates_from().takes_head() => { + return Some((false, crate::update::NO_HEAD.into())); + } Ok(commit) => commit, Err(why) => return Some((false, why)), }; @@ -2711,7 +2713,7 @@ impl Service { return Some((false, "an update is under way already".into())); } update.command = Some(id.to_string()); - let step = updates.first_step(); + let step = updates.updates_from().first_step(); update.doing = Some(match &commit { Some(commit) => format!("updating to {commit}: {step}"), None => format!("updating: {step}"), diff --git a/crates/daemon/src/testing.rs b/crates/daemon/src/testing.rs index db5d2bc..3be3eb5 100644 --- a/crates/daemon/src/testing.rs +++ b/crates/daemon/src/testing.rs @@ -951,15 +951,11 @@ impl crate::update::SelfUpdate for FakeUpdate { } answer } - fn takes_head(&self) -> bool { - !self.0.lock().expect("update").commits_only - } - - fn first_step(&self) -> &'static str { + fn updates_from(&self) -> crate::update::UpdateFrom { if self.0.lock().expect("update").commits_only { - "fetching its release" + crate::update::UpdateFrom::Releases } else { - "building" + crate::update::UpdateFrom::Checkout } } } diff --git a/crates/daemon/src/update.rs b/crates/daemon/src/update.rs index 154eef6..7acd094 100644 --- a/crates/daemon/src/update.rs +++ b/crates/daemon/src/update.rs @@ -39,19 +39,37 @@ pub trait SelfUpdate: Send + Sync { /// `pending` written down first. Refused while a machine has a record. fn install(&self, pending: &Pending) -> Result<(), String>; - /// Whether [`SelfUpdate::build`] takes the head of the branch (`update`), as a host that builds does. A host - /// that takes the Runner's releases updates only to a commit named (`update-to`), whose release it checks. - fn takes_head(&self) -> bool { - true + /// Where the updates come from: a checkout by default. + fn updates_from(&self) -> UpdateFrom { + UpdateFrom::Checkout } /// Ends what a build or a download under way started, as the Runner ends: the console sends its command again /// to the Runner that starts next. Nothing by default. fn end(&self) {} +} + +/// Where a Runner's updates come from, which decides the commands it takes and what the console sees meanwhile. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum UpdateFrom { + /// A checkout it builds: the head of main (`update`) or a commit of it (`update-to`). + Checkout, + /// The Runner's releases: only a commit the console names (`update-to`), whose release it checks. + Releases, +} + +impl UpdateFrom { + /// Whether [`SelfUpdate::build`] takes the head of the branch (`update`). + pub fn takes_head(self) -> bool { + self == UpdateFrom::Checkout + } /// What the console shows while [`SelfUpdate::build`] runs. - fn first_step(&self) -> &'static str { - "building" + pub fn first_step(self) -> &'static str { + match self { + UpdateFrom::Checkout => "building", + UpdateFrom::Releases => "fetching its release", + } } } diff --git a/crates/ghapp/src/http.rs b/crates/ghapp/src/http.rs index 9741752..f4e1105 100644 --- a/crates/ghapp/src/http.rs +++ b/crates/ghapp/src/http.rs @@ -41,8 +41,15 @@ pub fn same_origin(a: &Url, b: &Url) -> bool { && a.port_or_known_default() == b.port_or_known_default() } -/// The client every call goes through: a user agent, a connect timeout, a whole-request timeout, and redirects -/// followed only to the same scheme, host and port, so no answer can send a request and its token elsewhere. +/// What every client of GitHub has: a user agent and a connect timeout. +fn builder() -> reqwest::ClientBuilder { + reqwest::Client::builder() + .user_agent(concat!("codefusion-runner/", env!("CARGO_PKG_VERSION"))) + .connect_timeout(Duration::from_secs(10)) +} + +/// The client every call goes through: a whole-request timeout, and redirects followed only to the same scheme, host +/// and port, so no answer can send a request and its token elsewhere. fn client(timeout: Duration) -> reqwest::Client { let redirects = Policy::custom(|attempt| { let stays = attempt @@ -55,15 +62,31 @@ fn client(timeout: Duration) -> reqwest::Client { attempt.stop() } }); - reqwest::Client::builder() - .user_agent(concat!("codefusion-runner/", env!("CARGO_PKG_VERSION"))) - .connect_timeout(Duration::from_secs(10)) + builder() .timeout(timeout) .redirect(redirects) .build() .expect("a client with these settings builds") } +/// The client of a public release's files, which never carries a token: GitHub sends the address on to its storage, +/// on another host, so redirects are followed to any HTTPS address. A file read too slowly (`read_timeout` without +/// a byte) is dropped, however long it takes whole. +pub(crate) fn file_client(read_timeout: Duration) -> reqwest::Client { + let redirects = Policy::custom(|attempt| { + if attempt.previous().len() <= MAX_REDIRECTS && is_safe_address(attempt.url()) { + attempt.follow() + } else { + attempt.stop() + } + }); + builder() + .read_timeout(read_timeout) + .redirect(redirects) + .build() + .expect("a client with these settings builds") +} + /// How calls are tried again. #[derive(Debug, Clone, Copy)] pub struct RetryPolicy { @@ -159,13 +182,8 @@ impl Http { Ok(response) => return Ok(response), Err(error) => error, }; - if attempt >= self.retry.retries || !error.is_transient() { + let Some(wait) = self.retry.wait_after(&error, attempt) else { return Err(error); - } - let wait = match error.retry_after() { - Some(asked) if asked > self.retry.max_retry_after => return Err(error), - Some(asked) => asked, - None => backoff(self.retry.base, self.retry.cap, attempt), }; tracing::debug!(attempt, ?wait, "trying a GitHub call again"); tokio::time::sleep(wait).await; @@ -259,8 +277,23 @@ pub(crate) fn retry_after(headers: &HeaderMap) -> Option { )) } +impl RetryPolicy { + /// The wait before trying again after try `attempt` (from 0) failed with `error`, or `None` when the call ends + /// with it: no tries left, an error that another try does not mend, or a `Retry-After` over the longest wait. + pub(crate) fn wait_after(&self, error: &Error, attempt: u32) -> Option { + if attempt >= self.retries || !error.is_transient() { + return None; + } + match error.retry_after() { + Some(asked) if asked > self.max_retry_after => None, + Some(asked) => Some(asked), + None => Some(backoff(self.base, self.cap, attempt)), + } + } +} + /// An exponential wait with jitter: between half and the whole of `base * 2^attempt`, at most `cap`. -pub(crate) fn backoff(base: Duration, cap: Duration, attempt: u32) -> Duration { +fn backoff(base: Duration, cap: Duration, attempt: u32) -> Duration { let full = base .saturating_mul(1u32.checked_shl(attempt.min(20)).unwrap_or(u32::MAX)) .min(cap); diff --git a/crates/ghapp/src/lib.rs b/crates/ghapp/src/lib.rs index e09f5ac..ea9a0fd 100644 --- a/crates/ghapp/src/lib.rs +++ b/crates/ghapp/src/lib.rs @@ -30,6 +30,6 @@ pub use http::{ }; pub use key::KeyTokens; pub use release::{ - Asset, Author, Compared, Downloaded, RELEASES_PAGE, Release, Releases, Sink, is_whole_commit, + Asset, Author, Compared, Downloaded, Release, Releases, Sink, is_whole_commit, sleep_unless, }; pub use source::{Monotonic, Pending, SystemMonotonic, TokenSource}; diff --git a/crates/ghapp/src/release.rs b/crates/ghapp/src/release.rs index 34ca50d..3cf2e0f 100644 --- a/crates/ghapp/src/release.rs +++ b/crates/ghapp/src/release.rs @@ -6,16 +6,12 @@ use std::io::Write; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; -use reqwest::redirect::Policy; use reqwest::{StatusCode, Url}; use serde::Deserialize; use sha2::{Digest as _, Sha256}; use crate::error::{Error, StatusError}; -use crate::http::{Http, RetryPolicy, is_safe_address, retry_after, server_time}; - -/// How many redirects a download follows: GitHub sends a release's file on once, to its storage. -const MAX_REDIRECTS: usize = 5; +use crate::http::{Http, RetryPolicy, file_client, is_safe_address, retry_after, server_time}; /// How long a download may go without a byte before it counts as dropped. const READ_TIMEOUT: Duration = Duration::from_secs(60); @@ -24,7 +20,7 @@ const READ_TIMEOUT: Duration = Duration::from_secs(60); const STOP_LOOK: Duration = Duration::from_millis(200); /// How many releases one page lists: a new release is among the first, and an image release lists many files. -pub const RELEASES_PAGE: u32 = 30; +const RELEASES_PAGE: u32 = 30; /// A release as GitHub's API lists it, with the fields an update reads. #[derive(Debug, Clone, PartialEq, Eq, Deserialize)] @@ -83,12 +79,12 @@ pub enum Compared { } /// What a download wrote. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct Downloaded { /// How many bytes. pub bytes: u64, - /// Their SHA-256. - pub sha256: [u8; 32], + /// Their SHA-256, in lowercase hex. + pub sha256: String, } /// Where a download writes: a sink that can be emptied to begin again when a try breaks off. @@ -131,12 +127,6 @@ impl Releases { if !is_safe_address(&api) { return Err(Error::Refused(format!("{api} is not an HTTPS address"))); } - let plain = |part: &str| { - !part.is_empty() - && part - .bytes() - .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'.')) - }; if !repo .split_once('/') .is_some_and(|(owner, name)| plain(owner) && plain(name)) @@ -145,7 +135,7 @@ impl Releases { } Ok(Releases { http: Http::new(Duration::from_secs(30), retry), - files: file_client(), + files: file_client(READ_TIMEOUT), retry, api, repo: repo.to_string(), @@ -176,11 +166,7 @@ impl Releases { /// The release of `tag`, or `None` when there is none (GitHub answers 404). pub async fn by_tag(&self, tag: &str) -> Result, Error> { - if tag.is_empty() - || !tag - .bytes() - .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'.' | b'_')) - { + if !plain(tag) { return Err(Error::Refused(format!("{tag} is no plain tag"))); } match self.get("the release", &format!("releases/tags/{tag}")).await { @@ -271,22 +257,16 @@ impl Releases { Ok(downloaded) => return Ok(downloaded), Err(error) => error, }; - if attempt >= self.retry.retries || !error.is_transient() || stop.load(Ordering::SeqCst) { + if stop.load(Ordering::SeqCst) { return Err(error); } - let wait = match error.retry_after() { - Some(asked) if asked > self.retry.max_retry_after => return Err(error), - Some(asked) => asked, - None => crate::http::backoff(self.retry.base, self.retry.cap, attempt), + let Some(wait) = self.retry.wait_after(&error, attempt) else { + return Err(error); }; tracing::debug!(attempt, ?wait, "fetching a release's file again"); - // A wait that `stop` ends early: a Retry-After may ask for minutes. - let until = tokio::time::Instant::now() + wait; - while tokio::time::Instant::now() < until { - if stop.load(Ordering::SeqCst) { - return Err(stopped()); - } - tokio::time::sleep_until(until.min(tokio::time::Instant::now() + STOP_LOOK)).await; + // A Retry-After may ask for minutes. + if !sleep_unless(stop, wait).await { + return Err(stopped()); } attempt += 1; } @@ -347,11 +327,26 @@ impl Releases { sink.flush().map_err(|error| local(&error))?; Ok(Downloaded { bytes, - sha256: hash.finalize().into(), + sha256: format!("{:x}", hash.finalize()), }) } } +/// Sleeps for `time`, looking at `stop` meanwhile: `false` when it was set before the time was up. +pub async fn sleep_unless(stop: &AtomicBool, time: Duration) -> bool { + let until = tokio::time::Instant::now() + time; + loop { + if stop.load(Ordering::SeqCst) { + return false; + } + let now = tokio::time::Instant::now(); + if now >= until { + return true; + } + tokio::time::sleep_until(until.min(now + STOP_LOOK)).await; + } +} + fn stopped() -> Error { Error::Local("the download was stopped".into()) } @@ -365,27 +360,15 @@ fn local(error: &std::io::Error) -> Error { } } +/// Whether `name` is a plain name in a GitHub address: letters, digits, `-`, `_` and `.`, and not empty. +fn plain(name: &str) -> bool { + !name.is_empty() + && name + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'.')) +} + /// Whether `name` is a whole commit's name: 40 lowercase hex characters. pub fn is_whole_commit(name: &str) -> bool { name.len() == 40 && name.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) } - -/// The client of a release's files: GitHub sends the address on to its storage, on another host, so redirects are -/// followed to any HTTPS address. No token is ever sent; a file read too slowly is dropped, however long it takes -/// whole. -fn file_client() -> reqwest::Client { - let redirects = Policy::custom(|attempt| { - if attempt.previous().len() <= MAX_REDIRECTS && is_safe_address(attempt.url()) { - attempt.follow() - } else { - attempt.stop() - } - }); - reqwest::Client::builder() - .user_agent(concat!("codefusion-runner/", env!("CARGO_PKG_VERSION"))) - .connect_timeout(Duration::from_secs(10)) - .read_timeout(READ_TIMEOUT) - .redirect(redirects) - .build() - .expect("a client with these settings builds") -} diff --git a/crates/ghapp/tests/release.rs b/crates/ghapp/tests/release.rs index 8f18dc8..1aecdc2 100644 --- a/crates/ghapp/tests/release.rs +++ b/crates/ghapp/tests/release.rs @@ -73,8 +73,8 @@ fn program() -> Vec { (0..1_048_576u32).map(|n| (n % 251) as u8).collect() } -fn sha256(bytes: &[u8]) -> [u8; 32] { - Sha256::digest(bytes).into() +fn sha256(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) } /// A file in a folder of the test's own. diff --git a/crates/runner/src/release_update.rs b/crates/runner/src/release_update.rs index 2700aad..8070581 100644 --- a/crates/runner/src/release_update.rs +++ b/crates/runner/src/release_update.rs @@ -12,7 +12,7 @@ use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{Duration, Instant}; -use cf_daemon::update::{NO_HEAD, Pending, SelfUpdate, is_commit}; +use cf_daemon::update::{NO_HEAD, Pending, SelfUpdate, UpdateFrom, is_commit}; use cf_ghapp::{Compared, Release, Releases, RetryPolicy, is_whole_commit}; use crate::commands::VERSION; @@ -53,6 +53,14 @@ pub fn asset_for_this_host() -> Option<&'static str> { } } +/// Whether `version` is a package version, three numbers: `0.1.0`. +fn is_package_version(version: &str) -> bool { + version.split('.').count() == 3 + && version + .split('.') + .all(|part| !part.is_empty() && part.bytes().all(|b| b.is_ascii_digit())) +} + /// The tag of a release of version `version` (`0.1.0`) at the commit `short` (12 characters). pub fn tag_of(version: &str, short: &str) -> String { format!("runner-{version}-{short}") @@ -131,9 +139,13 @@ impl ReleaseUpdate { async fn release_of(&self, short: &str) -> Result { let (wait, every) = self.wait; let began = Instant::now(); - for look in 0u32.. { + let mut look = 0u32; + loop { let last = began.elapsed() + every > wait || self.stop.load(Ordering::SeqCst); - if let Some(release) = self.look_for(short, last || look % PAGE_EVERY == 0).await? { + if let Some(release) = self + .look_for(short, last || look.is_multiple_of(PAGE_EVERY)) + .await? + { return Ok(release); } if last { @@ -143,9 +155,9 @@ impl ReleaseUpdate { )); } tracing::info!(commit = short, "its release is not out yet; asking again"); - self.sleep(every).await; + cf_ghapp::sleep_unless(&self.stop, every).await; + look += 1; } - unreachable!("the looks end by the wait") } async fn look_for(&self, short: &str, with_page: bool) -> Result, String> { @@ -170,27 +182,13 @@ impl ReleaseUpdate { .tag_name .strip_prefix("runner-") .and_then(|rest| rest.strip_suffix(&ends)) - .is_some_and(|version| { - let parts: Vec<&str> = version.split('.').collect(); - parts.len() == 3 - && parts - .iter() - .all(|part| !part.is_empty() && part.bytes().all(|b| b.is_ascii_digit())) - }) + .is_some_and(is_package_version) })) } - /// Sleeps for `time`, or less once the Runner ends. - async fn sleep(&self, time: Duration) { - let until = Instant::now() + time; - while Instant::now() < until && !self.stop.load(Ordering::SeqCst) { - tokio::time::sleep(Duration::from_millis(200).min(until - Instant::now())).await; - } - } - /// Finds the release of `name`, checks it is newer on main than `running`, and fetches and checks its program - /// into [`ReleaseUpdate::download`]. Answers `None` when `running` is that commit already. - async fn fetch(&self, name: &str, short: &str, running: &str) -> Result, String> { + /// into [`ReleaseUpdate::download`]. Answers whether it fetched one: not when `running` is that commit already. + async fn fetch(&self, name: &str, short: &str, running: &str) -> Result { let release = self.release_of(short).await?; let tag = &release.tag_name; let not = "nothing was downloaded"; @@ -226,7 +224,7 @@ impl ReleaseUpdate { } match self.releases.compare(running, full).await.map_err(github)? { Compared::Ahead => {} - Compared::Identical => return Ok(None), + Compared::Identical => return Ok(false), Compared::Behind => { return Err(format!( "{short} is not newer than {running}, which runs: a Runner is not taken back; {not}" @@ -268,14 +266,14 @@ impl ReleaseUpdate { .await .map_err(|error| fetched(&self.asset, error))?; drop(file); - let got = hex(&got.sha256); + let got = got.sha256; if got != want { return Err(format!( "{} of {tag} has the SHA-256 {got}, not the {want} its {SUMS} names; {not}", self.asset )); } - Ok(Some(())) + Ok(true) } } @@ -304,13 +302,14 @@ impl SelfUpdate for ReleaseUpdate { } let path = self.download(); let fetched = block_on(self.fetch(name, short, running)); - let offered = match fetched { - Ok(None) => Ok(self.running.clone()), - Ok(Some(())) => make_runnable(&path) + let offered = fetched.and_then(|downloaded| { + if !downloaded { + return Ok(self.running.clone()); + } + make_runnable(&path) .map_err(|error| format!("{}: {error}", path.display())) - .and_then(|()| self.installer.offer(&path, short, &self.asset)), - Err(why) => Err(why), - }; + .and_then(|()| self.installer.offer(&path, short, &self.asset)) + }); let _ = std::fs::remove_file(&path); offered } @@ -319,17 +318,13 @@ impl SelfUpdate for ReleaseUpdate { self.installer.install(pending) } - fn takes_head(&self) -> bool { - false + fn updates_from(&self) -> UpdateFrom { + UpdateFrom::Releases } fn end(&self) { self.stop.store(true, Ordering::SeqCst); } - - fn first_step(&self) -> &'static str { - "fetching its release" - } } /// Runs `future` to its end from a thread that may wait: the service's runtime when the thread is one of its @@ -361,10 +356,6 @@ fn make_runnable(path: &Path) -> std::io::Result<()> { std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700)) } -fn hex(bytes: &[u8]) -> String { - bytes.iter().map(|byte| format!("{byte:02x}")).collect() -} - #[cfg(test)] mod tests { use std::collections::{HashMap, VecDeque}; @@ -394,7 +385,7 @@ mod tests { } fn sum(bytes: &[u8]) -> String { - hex(&Sha256::digest(bytes)) + format!("{:x}", Sha256::digest(bytes)) } /// What the fake GitHub holds and answers. @@ -855,7 +846,7 @@ mod tests { let mut bench = bench(); bench .update - .wait_for(Duration::from_millis(1500), Duration::from_millis(100)); + .wait_for(Duration::from_millis(600), Duration::from_millis(40)); let none = "dddddddddddd3333333333333333333333333333"; bench.update.build(Some(none)).expect_err("no release"); let seen = bench.seen(); @@ -1088,7 +1079,7 @@ mod tests { fn main_s_head_is_not_taken_and_names_that_cannot_name_a_release_ask_nothing() { let bench = bench(); assert_eq!(bench.update.build(None), Err(NO_HEAD.into())); - assert!(!bench.update.takes_head()); + assert_eq!(bench.update.updates_from(), UpdateFrom::Releases); assert_eq!( bench.update.build(Some("bbbbbbb")), Err("bbbbbbb is too short to name a release: send 12 characters of it or more; nothing was downloaded".into()) diff --git a/docs/MAP.md b/docs/MAP.md index bce5e73..92d8ba6 100644 --- a/docs/MAP.md +++ b/docs/MAP.md @@ -13,22 +13,22 @@ | `crates/nft` (`cf-nft`) | nf_tables rules without the `nft` program. `lib.rs` is the model (`Match`, `Rule` and `Rule::new`, `Chain` on a `Hook`, `Table` in a `Family`: inet, bridge, netdev) and `decide`, what a rule set does with a packet (pure). `encode.rs` turns tables into one netlink batch (`batch`, `Load::Replace` swaps the whole ruleset in one transaction) and refuses what the kernel would; `decode.rs` reads the kernel's messages back (`read`). `apply` and `list` (Linux) talk to the kernel through `cf-netlink`. `BLOCKED` is the one list of private and special ranges, used by the guest's firewall and by the host's in front of a Linux machine. Tests: `tests/nft.rs` (bytes `nft` 1.0.6 wrote, recorded in `tests/nft/`), `tests/model.rs` (a model of the kernel's expressions against `decide`, round trips), `tests/kernel.rs` (Linux: applied and listed back in a fresh user and network namespace). | | `crates/netlink` (`cf-netlink`) | Netlink itself, for any protocol: `message`, `attribute`, `attribute_str` (a NUL-ended text), `nested`, `messages` and `attributes` to read them back, and on Linux the `Socket` that sends requests and reads every answer (`request`, `dump`), with its own `Error`. Users: `cf-nft` (nf_tables), `cf-vmm-ch` (rtnetlink). Tests in `tests/wire.rs`. | | `crates/vmm-ch` (`cf-vmm-ch`) | `ChVmm`: machines on Linux, cloud-hypervisor under KVM, one process per machine (`machine.rs`, Linux). Each machine has a cgroup of its own under the part of the tree delegated to the Runner (`cgroup.rs`: memory at its size and `overhead`, no swap (a kernel without a swap account per cgroup only on a host that swaps to nothing), `pids.max`, the cpuset of its cores, `cgroup.kill`, `wait_empty` woken by `cgroup.events`; `memory.current` is `Vm::resident`; `own` and `enter_host` for a test suite that shares its delegated cgroup), its vCPUs on the host's fast cores that it may use (`cpus.rs` with `cf_host_linux::fast_cpus`, `--host-cpus` overrides). With a network it runs under the system's pasta (`programs::pasta`: found on `PATH`, refused when another user could change it or what it links to, tried when the first machine with a network starts; `apt install passt`, 2023-03 or newer) in a user and network namespace; the hidden `vm-sandbox` subcommand of the binary (`sandbox.rs`) makes the guest's tap, loads `firewall.rs` onto it (netdev ingress: ARP, DHCP and DNS to pasta on the gateway, IPv4 to the internet; drops `BLOCKED`, 127/8 and every network of the host, IPv6 and everything else), bridges it with pasta's device (`link.rs`, rtnetlink), and starts the monitor without capabilities and without user namespaces in a PID, cgroup, mount and empty network namespace of its own (the tap's descriptor its only way out), no file it writes growing past the largest it is shown, its root a read-only tmpfs with only what `Sandbox::for_machine` shows it (its program, the guest's files, its console and folder, `/dev/kvm`, `/dev/urandom`) and a `/proc` of its own, then reports how it ended (`ending.rs`). `command.rs` builds both command lines, `events.rs` reads `--event-monitor`, `vsock.rs` the CONNECT handshake. cloud-hypervisor is pinned in `pins.rs` for each architecture, the host's fetched into `target/vmm-programs` and embedded by `build.rs` (Linux on aarch64 or x86_64), unpacked into `/programs` and checked before each machine (`programs.rs`). Tests: unit tests of everything pure, `tests/isolation.rs` (Linux: a stand-in guest on a veth pair behind the real pasta, guard and firewall; each blocked target a test of its own), `tests/monitor_root.rs` (Linux, no harness: a stand-in monitor in the real sandbox tries the host's files, processes and mounts, and under pasta tries to send past its tap), `tests/kvm.rs` (Linux with KVM, no harness: how a real machine ends, a panic, a stop, a kill, its cgroup's OOM and a dead monitor; it skips without `/dev/kvm` and the guest, `CF_REQUIRE_KVM=1` makes that a failure). | -| `crates/runner` (`codefusion-runner`) | The binary. `commands.rs`: `version` (the package's version and the commit, which `build.rs` reads from git or `CF_COMMIT`), `image verify`, `image install`, `pause`, `resume` and `status` (`holds.rs`: a named hold on the machine, `--wait` until the Runner saw it and runs nothing, exit 75 when the wait runs out; `status --json`), `vm` (the vm-host, `src/vm.rs`), `boot-proof`, `vmm` (Linux: the cloud-hypervisor the build carries, which `install` and an update ask of a program), and the service's commands in `service.rs` (macOS and Linux): `daemon` (what the service manager runs; ends on SIGTERM; one per state folder, held by a lock on `/runner.lock`), `enroll` (the console's one-time code on standard input, read with the terminal's echo off by `hidden.rs`, which gives the terminal back on every way out, a signal included; the token to `/console-token`, 0600, never printed), `check [--github]` (what an install needs, and the memory at that moment against the budget's floor with the sizes that fit; with `--github` what GitHub shows, changing nothing there and nothing in the state folder), `install` (copies the program to `/bin`, writes the service manager's entry, refuses while a machine stands), `uninstall [--leave] [--purge]` (refused while a Runner started by hand holds the state folder; a Mac without a usable token goes without telling the console). `config.rs` is the configuration file (refused when others can write it; `console` required unless the file names `app_dir` (absolute, not through `..`), the folder of a GitHub App key the host held before the console held it: while its `app.pem` is there `service.rs` `app_and_key` signs with it (`held_key`: `KeyTokens`, and `cf_daemon::HeldKey` for the service), else the GitHub tokens come from the console (`ConsoleTokens`); `check` and `uninstall` sign with a held key and never take it out (`looking`); `use` required on a Mac; on Linux `dedicated` or left out; `max_jobs`, `idle_minutes` (a workstation's only), `min_free_memory_percent` and `vcpus` (a size's vCPUs by its label, which `Settings::limited` checks against the CPUs CI may use), `ci_memory_gb` (the memory CI's machines hold at once, each with its overhead, in GB of 2^30 bytes) and `ci_cpus` (the CPUs they take together) are the owner's `Limits`, which `budget_of` and the service lay over the host's settings; `ConfigFile::change` writes the console's `settings` command into the file through `.json.next`, which must start a Runner before it replaces the file) and, on a Mac, the refusal of the cf-runners manager's own name and of external volumes; `manager.rs` is `ServiceManager`, the seam to the service manager; `launchd.rs` the agent `cc.codefusion.vm-runner` in the person's session (`plist`, `Agent`; `stop` waits past `ExitTimeOut` and decides by whether launchd still has the agent, not by what `bootout` says); `systemd.rs` (Linux) the user unit `codefusion-runner.service` (`unit`, with every path quoted for systemd or refused; `UserUnit` over an injected `systemctl --user`, the real one ended after 100 s; `stop` waits past `TimeoutStopSec` and decides by `ActiveState`); `linux.rs` the lines `check` adds on Linux (`Facts`, `lines`: KVM, lingering, delegated controllers, user namespaces, pasta, `noexec`, each with the root command that fixes it). `update.rs`: `CheckoutUpdate`, the `SelfUpdate` of a host that built the Runner, for the console's `update` and `update-to`: in the checkout the program was built from (`build.rs` writes it down, with `CARGO_HOME`, `RUSTUP_HOME` and `CARGO_BUILD_JOBS`), only when it is the config's `build_folder`, clean and at a commit (`own`), fetches `origin refs/heads/main` (a tag named `main` is not it), takes main's head or the commit `update-to` names when that resolves to one commit of the main fetched (`on_main`), refuses a target that does not contain the running commit (both looked up among the objects alone by `commit_named`, so a branch or tag named like a commit never stands for it), checks out the peeled commit, builds with `cargo xtask build` at low priority into `logs/update.log` and ends a build that takes longer than 45 minutes or outlasts the daemon, and keeps the program it checked as `bin/codefusion-runner.next`; `install` places that one, keeps the running program as `bin/codefusion-runner.prev` and writes `update.json` first; `guard`, run before the daemon reads its configuration, counts the new program's starts and puts the one before back after `MAX_STARTS`, and `ended_in_order` takes back the count of a start that ended as asked. `Installer` is what both updates share: `offer` asks a program its `version` (it must run here and name the commit) and keeps it as `.next`, `install` places it. `release_update.rs` (Linux and macOS builds; Linux hosts only by asset): `ReleaseUpdate`, the `SelfUpdate` of a Runner with no `build_folder`, installed from a release: `takes_head` is false, so it lists only `update-to`; `build` finds the commit's release (`runner--<12 hex>` by tag, else the newest page), waits `RELEASE_WAIT` for one not out yet, asked every `ASK_EVERY` by its tag and every `PAGE_EVERY`th look among the newest too, ended by `end`; refuses a draft, another publisher than `github-actions[bot]`, one not made at the whole commit, not on main or not newer than the running commit (`cf_ghapp::Releases::compare`); fetches `SHA256SUMS` and this host's asset (`asset_for_this_host`) into `.download` (0600), checks its SHA-256 against `SHA256SUMS` (`cf_image::parse_sums`), and removes the download whatever happens; the console shows `fetching its release` (`first_step`). `service.rs` `self_update` picks `CheckoutUpdate` with a `build_folder`, else `ReleaseUpdate`. `app_key.rs`: `take_out`: a file that still names `app_dir` has `/app.pem` deleted (a symlink: the link alone, never what it points to), `app_dir` taken out (the file without it checked as a configuration before anything is deleted, then written through `.json.next`) and the deleted file remembered first in `/app-key-file` (0600; a link's target on a second line; `remembered`), which the report's `appKeyFile` names while something is still there; refused, changing nothing, while the file names no console or the host holds no token for it (`KEPT_NO_CONSOLE`, `KEPT_NOT_ENROLLED`). A key deleted whose file could not be written is no failure: `TakenOut::app_dir_kept` says why the file still names the folder. `one_change` is held from the read to the write, by the take-out and by `ConfigFile::change` (the console's settings). `When::Answered` is the service's, once a scale set call of the console this host made was answered (`HeldKey::take_out`); `When::KeyGone` is what `daemon` runs (`start_on`) after `guard` and the state folder's lock (`only_runner`) and before it checks its configuration: it deletes nothing, and takes out only a folder whose key file this Runner's own take-out remembered; a key file missing otherwise keeps the folder (`NOT_TAKEN_OUT_HERE`), and the report's `appKeyKept` says so (`app_key_said`). `runner.entitlements` is signed into it. `config.rs`: `class` (`shared` or `trusted`) and `trusted` (the rules); `check_class` refuses a trusted host that would serve more than one job of one repository, or that holds the GitHub App's key (`app_dir`) or has no `console` to register through. | +| `crates/runner` (`codefusion-runner`) | The binary. `commands.rs`: `version` (the package's version and the commit, which `build.rs` reads from git or `CF_COMMIT`), `image verify`, `image install`, `pause`, `resume` and `status` (`holds.rs`: a named hold on the machine, `--wait` until the Runner saw it and runs nothing, exit 75 when the wait runs out; `status --json`), `vm` (the vm-host, `src/vm.rs`), `boot-proof`, `vmm` (Linux: the cloud-hypervisor the build carries, which `install` and an update ask of a program), and the service's commands in `service.rs` (macOS and Linux): `daemon` (what the service manager runs; ends on SIGTERM; one per state folder, held by a lock on `/runner.lock`), `enroll` (the console's one-time code on standard input, read with the terminal's echo off by `hidden.rs`, which gives the terminal back on every way out, a signal included; the token to `/console-token`, 0600, never printed), `check [--github]` (what an install needs, and the memory at that moment against the budget's floor with the sizes that fit; with `--github` what GitHub shows, changing nothing there and nothing in the state folder), `install` (copies the program to `/bin`, writes the service manager's entry, refuses while a machine stands), `uninstall [--leave] [--purge]` (refused while a Runner started by hand holds the state folder; a Mac without a usable token goes without telling the console). `config.rs` is the configuration file (refused when others can write it; `console` required unless the file names `app_dir` (absolute, not through `..`), the folder of a GitHub App key the host held before the console held it: while its `app.pem` is there `service.rs` `app_and_key` signs with it (`held_key`: `KeyTokens`, and `cf_daemon::HeldKey` for the service), else the GitHub tokens come from the console (`ConsoleTokens`); `check` and `uninstall` sign with a held key and never take it out (`looking`); `use` required on a Mac; on Linux `dedicated` or left out; `max_jobs`, `idle_minutes` (a workstation's only), `min_free_memory_percent` and `vcpus` (a size's vCPUs by its label, which `Settings::limited` checks against the CPUs CI may use), `ci_memory_gb` (the memory CI's machines hold at once, each with its overhead, in GB of 2^30 bytes) and `ci_cpus` (the CPUs they take together) are the owner's `Limits`, which `budget_of` and the service lay over the host's settings; `ConfigFile::change` writes the console's `settings` command into the file through `.json.next`, which must start a Runner before it replaces the file) and, on a Mac, the refusal of the cf-runners manager's own name and of external volumes; `manager.rs` is `ServiceManager`, the seam to the service manager; `launchd.rs` the agent `cc.codefusion.vm-runner` in the person's session (`plist`, `Agent`; `stop` waits past `ExitTimeOut` and decides by whether launchd still has the agent, not by what `bootout` says); `systemd.rs` (Linux) the user unit `codefusion-runner.service` (`unit`, with every path quoted for systemd or refused; `UserUnit` over an injected `systemctl --user`, the real one ended after 100 s; `stop` waits past `TimeoutStopSec` and decides by `ActiveState`); `linux.rs` the lines `check` adds on Linux (`Facts`, `lines`: KVM, lingering, delegated controllers, user namespaces, pasta, `noexec`, each with the root command that fixes it). `update.rs`: `CheckoutUpdate`, the `SelfUpdate` of a host that built the Runner, for the console's `update` and `update-to`: in the checkout the program was built from (`build.rs` writes it down, with `CARGO_HOME`, `RUSTUP_HOME` and `CARGO_BUILD_JOBS`), only when it is the config's `build_folder`, clean and at a commit (`own`), fetches `origin refs/heads/main` (a tag named `main` is not it), takes main's head or the commit `update-to` names when that resolves to one commit of the main fetched (`on_main`), refuses a target that does not contain the running commit (both looked up among the objects alone by `commit_named`, so a branch or tag named like a commit never stands for it), checks out the peeled commit, builds with `cargo xtask build` at low priority into `logs/update.log` and ends a build that takes longer than 45 minutes or outlasts the daemon, and keeps the program it checked as `bin/codefusion-runner.next`; `install` places that one, keeps the running program as `bin/codefusion-runner.prev` and writes `update.json` first; `guard`, run before the daemon reads its configuration, counts the new program's starts and puts the one before back after `MAX_STARTS`, and `ended_in_order` takes back the count of a start that ended as asked. `Installer` is what both updates share: `offer` asks a program its `version` (it must run here and name the commit) and keeps it as `.next`, `install` places it. `release_update.rs` (Linux and macOS builds; Linux hosts only by asset): `ReleaseUpdate`, the `SelfUpdate` of a Runner with no `build_folder`, installed from a release: `updates_from` is `Releases`, so it lists only `update-to`; `build` finds the commit's release (`runner--<12 hex>` by tag, else the newest page), waits `RELEASE_WAIT` for one not out yet, asked every `ASK_EVERY` by its tag and every `PAGE_EVERY`th look among the newest too, ended by `end`; refuses a draft, another publisher than `github-actions[bot]`, one not made at the whole commit, not on main or not newer than the running commit (`cf_ghapp::Releases::compare`); fetches `SHA256SUMS` and this host's asset (`asset_for_this_host`) into `.download` (0600), checks its SHA-256 against `SHA256SUMS` (`cf_image::parse_sums`), and removes the download whatever happens. `service.rs` `self_update` picks `CheckoutUpdate` with a `build_folder`, else `ReleaseUpdate`. `app_key.rs`: `take_out`: a file that still names `app_dir` has `/app.pem` deleted (a symlink: the link alone, never what it points to), `app_dir` taken out (the file without it checked as a configuration before anything is deleted, then written through `.json.next`) and the deleted file remembered first in `/app-key-file` (0600; a link's target on a second line; `remembered`), which the report's `appKeyFile` names while something is still there; refused, changing nothing, while the file names no console or the host holds no token for it (`KEPT_NO_CONSOLE`, `KEPT_NOT_ENROLLED`). A key deleted whose file could not be written is no failure: `TakenOut::app_dir_kept` says why the file still names the folder. `one_change` is held from the read to the write, by the take-out and by `ConfigFile::change` (the console's settings). `When::Answered` is the service's, once a scale set call of the console this host made was answered (`HeldKey::take_out`); `When::KeyGone` is what `daemon` runs (`start_on`) after `guard` and the state folder's lock (`only_runner`) and before it checks its configuration: it deletes nothing, and takes out only a folder whose key file this Runner's own take-out remembered; a key file missing otherwise keeps the folder (`NOT_TAKEN_OUT_HERE`), and the report's `appKeyKept` says so (`app_key_said`). `runner.entitlements` is signed into it. `config.rs`: `class` (`shared` or `trusted`) and `trusted` (the rules); `check_class` refuses a trusted host that would serve more than one job of one repository, or that holds the GitHub App's key (`app_dir`) or has no `console` to register through. | | `crates/budget` (`cf-budget`) | Admission, with no I/O: `Budget` (`observe`, `claim`, `confirm`, `assign` and `withdraw` for a job GitHub assigned without asking (`assign_in_turn` and `confirm_in_turn` with its `Turn`: `line` puts main-branch and deploy jobs first, repositories in turn, none passed over past `DEFER_LIMIT`; `set_turn` when what a waiting job stands for changed; a `Turn::refusable` job, a test run, gets room only as a claim would, under the limits in force, and waiting is passed over without holding up those behind it; `make_room` holds back every other size, `making-room`, and puts the size first in line), `adopt`, `usage`, `release`, `hold`, and `status`, which names every slot with how it started and every waiting job with what keeps it), the pure `admit`, `Settings` (every number, `for_host` has the defaults; `limited` lays the owner's `Limits` over them: `max_machines`, refused past it with `Refusal::Machines`, `idle_after`, the floor as a share of memory, which lowers the cap, `ci_bytes`, the memory CI's machines hold at once with each one's overhead (`admit` counts them; `tests/ci_memory.rs`), and `ci_cpus`, the CPUs they take together; `for_headless` those of a Linux host nobody sits at, which reserves what the host needs, `HEADLESS_HOST_USE`, and caps the sizes so that each machine fits with its overhead; `on_fast_cores` gives CI only the fast kind of core of a host whose cores differ; `one_at_a_time` those of a trusted host, which runs one machine on every core it has), `Size` and its labels, the `Host` trait with `HostReading`, `Battery`, what a host's page shows of its battery, and `FakeHost`, the host tests script (`read_takes` lets a read take time before and after the reading gets its time; `tests/fake_host.rs`). Tests: `admission`, `assigned`, `headless`, `limits` (the rule at 4, 7.75, 8, 16 and 32 GiB and its boundaries), `levels`, `making_room`, `races`, `properties`, `turns`. | | `crates/clock` (`cf-clock`) | `now`: the two clocks the budget counts time by, read from the host (Linux: `CLOCK_MONOTONIC` and `CLOCK_BOOTTIME`; macOS: `CLOCK_UPTIME_RAW` and `CLOCK_MONOTONIC_RAW`). Both host crates' `Host::now`. | | `crates/host-macos` (`cf-host-macos`) | `MacHost` reads macOS behind `cf_budget::Host` (page counts, pressure, swap, idle time, thermal, power, disk; what the volume frees on demand, Time Machine's local snapshots and purgeable caches, is its capacity for important use beyond the free space, asked at most every `IMPORTANT_EVERY`, 30 s, for the budget's disk hold and never for an eviction); `Raw` and `reading` are the arithmetic: available memory is what is ready now (free, file cache, purgeable) and, while the Mac is calm (pressure normal, `kern.memorystatus_level` at least `CALM_LEVEL`, the compressor under `HEAVY_COMPRESSOR_SHARE` of memory), what compressing its inactive anonymous pages would free, never past the memorystatus level's share (`tests/reclaim.rs`). For the console's page: `boot_time`, `model` (chip and model, `model_name`), `battery` (what `pmset -g batt` prints, read by `battery_in`, as `cf_budget::Battery`), `host_name`, `load`, `cpu_percent`. `performance_cores` (`hw.perflevel0.logicalcpu` of a chip with two kinds of core, for the budget's CPUs). `program_of` says which program a process runs (the daemon's `is_ours`, `Holder`). `backup`: `exclude_from_backup` marks a folder the Runner made as skipped by Time Machine (the sticky exclusion `tmutil addexclusion` sets, an extended attribute), `exclude_with` is the decision behind the `Attributes` edge; the daemon, `enroll`, `install` and `image install` mark the state folder (the daemon's `Mac::keep_state`, a failure is a logged line, never fatal), and a cache folder to come goes through the same call when it is made. `examples/host-probe.rs` prints a Mac's readings. | | `crates/host-linux` (`cf-host-linux`) | `LinuxHost` reads Linux behind `cf_budget::Host`: MemAvailable, swap, swap-outs (`swapouts_of`: `pswpout`, but with zram in `/proc/swaps` only what is in use on swap that is not zram, since a page in zram stays in memory and PSI says when memory is short; `pswpout` again when `/proc/swaps` cannot be read), PSI (`/proc/pressure/memory`, Normal when the kernel has none), the thermal zones by their own trip points, `/sys/class/power_supply` (a host with no battery runs on the mains; a device's own battery, such as a mouse's, is left out unread), the state folder's disk by `statvfs`; idle time is none on a headless host. Every reader takes a root folder, so tests run against trees of files on any OS. `parse` reads each file, never more than `MAX_FILE_BYTES` (`counter` is one value of a `name value` file, also a cgroup's); `Raw` and `reading` are the arithmetic, with what each field means in the doc comment of `reading`. `fast_cpus` (the kind of core, by `midr_el1`, of the highest `cpu_capacity`, for `Settings::for_headless`). For the console's page: `boot_session` (`boot_id`), `boot_time`, `model` (device tree, else DMI), `battery`, `host_name`, `load`, `cpu_percent`. `process` and `program_of` say which program a process runs and when it started, read twice around the program so a number that went to another process in between names none; `started_at` moves with a step of the wall clock, `started_ticks` with `boot_session` does not. Tests: `fixtures.rs` (an RK3588 board's files, broken in every way), `parse.rs` (with properties), `real.rs` (Linux only: this machine against `free`, `df`, `getconf`). `examples/linux-probe.rs` prints a Linux host's readings and the headless budget. | | `crates/lifecycle` (`cf-lifecycle`) | `Slot`: the state machine of one slot and its job (events in, effects out, watchdogs, `saved` and `restore` for a daemon restart; `on` answers every phase and event with no catch-all arm, keeps a word of a job in every phase, and ends a slot as `no-job` only on `Event::Unregistered`, GitHub's 204; `Asking` is the phase of a slot that asks for its registration's removal, entered at the idle limit or on `Retire`; `Slot::hear` replays GitHub's stored word on a saved slot), and `StartBreaker` (`admit` before a claim, `holds` for the budget's hold). `tests/model.rs` searches every state it can reach. | | `crates/scaleset` (`cf-scaleset`) | The scale set protocol: `ScaleSetClient` (scale sets, sessions, one-job JIT configs, `generate_plain_jit_config` for one outside any scale set, removal of an Actions runner by id and by name (`remove_runner_named`); every administrator's call is the console's `github/scaleset/*`, which holds the Runners App's key and the service's admin token, except on a host that still holds the key (`with_key`, `holds_key`; `key.rs` `KeyAdmin`: its own registration token and admin token, the admin token kept until 120 s before it ends and replaced once on 401, and only the sets named `--…` listed), and on a host whose console is older than those calls (a 404 without a code: `KeyAdmin` with the console's `github/registration` token, `Registrar::Console`, until the Runner starts again; a Runner never needs a console newer than the one that runs; `tests/older_console.rs`); the session keeps the `serviceUrl` the console gave), `runner_busy`, the REST view of whether an Actions runner holds a job, `MessageSession` (long poll, ack, acquire), `listen` with the `Scaler` trait (handle, then ack; duplicates acked only; shutdown interrupts only the wait), `Backoff` and `supervise`. A token goes only to the API's origin and to hosts of `ACTIONS_HOST`: an address the service hands back that is elsewhere is refused. A registration whose answer was lost, or that names no runner, is removed again. Every status and malformed body is an `Error`. `examples/github_probe.rs` is the real check against a throwaway repository (prints no token). Tests in `tests/`, run against `cf-fake-github`; `github_rules.rs` holds the fake to what GitHub was seen to do (`docs/github-protocol.md`). | -| `crates/daemon` (`cf-daemon`) | The service. `holds.rs`: holds set on the machine itself, one file each in `/holds` (`set`, `lift`, `read`, which holds on a damaged file or an unreadable folder), and `/status.json` (`LocalStatus`: the holds the Runner saw, its machines and the ones coming, written on a change and every `STATUS_EVERY`); `Service::look_at_holds`, each tick, takes no new job while any hold or the console's pause is set and reports them as `status.holds`. `machines.rs`: `Machines` joins the budget, the lifecycle of each slot and the vm-hosts: its scale sets share the room their polls offer GitHub (`offer_room`: a free slot one poll offers no other does until that poll ends, `OFFER_STANDS` at most, and sets with jobs queued, `set_demand`, go before those without), so GitHub never assigns more jobs than it can start; a job GitHub assigned is counted (`assign`), started when memory allows (a machine counts with what its vm-host read it holds of the Mac's memory, `count_held`, its job's peak only when that is not known; the log says once when it is not, or is more than the machine is counted with, `Budget::host_bytes`), registered just in time, watched every tick, and ended once with an `EndKind` (`ends.rs`); `restore` adopts what a Runner before left from the records (`records.rs`, `DiskStore`; a record stores facts, `up`, `ran_job`, `completed`, `unregistered`, and carries the `RULES` it was written under; `Record::agrees` says its facts agree, and `Record::upgrade` is the one place that reads an older Runner's record: its beliefs cleared, GitHub's stored word replayed; a current record that disagrees is an error) and the vm-hosts that still answer, in the same start of the Mac (`kern.bootsessionuuid`, not the boot time, which moves with the clock); one that is silent or wrote nothing yet is killed, and so is what has no record. A tick looks at the vm-hosts without the machines' lock, so a vm-host that hangs keeps no listener waiting. What the polls in flight may still offer GitHub is kept (`Offer`): every answer a listener got since all bells last rang, since many scale sets of one size each ask at their own moment. The last slot taken rings their bells at once, room that went or came back with the Mac's memory at a look, at most every 5 s, so no poll in flight goes on offering a slot that is gone, or none while there is one. An open Mac whose other programs leave no memory for a size it serves says so in the log and is closed for `memory` in the console's report (`Snapshot::short_of`); one that would carry the size without its own machines is full, not short. `Snapshot::rooms` is what one more machine of each size meets, for the report's `sizes` and each scale set's capacity: how many fit, the budget's word when none does, `never` for a size the settings never run, and for memory what it needs and lacks. `making.rs`: `Making`, room made for a larger size. A job that waits for the room the Mac's own machines hold, all of them smaller, while the empty Mac would run it, has every other size held back until it starts, for `MAKE_ROOM_LIMIT` (30 min) at most and then none for as long; its scale set offers a place in line while GitHub has the job back, for `TAKEN_BACK_WAIT` (2 min); an empty Mac whose other programs hold the room gives up after `NO_ROOM_WAIT`. A size whose polls found no room for `STARVED_AFTER` (10 min) is offered a place before any of its jobs is assigned; a size the empty Mac cannot run never is. A headless host, one of a pool, makes room and offers a place only for the size the console's answer names it (`makeRoomFor`, `Making::name`, `Machines::make_room_for`), at once. `vms.rs`: `Vms`, the seam to `codefusion-runner vm`, and `VmHosts`, the real one. It writes each vm-host's process and start time to `run/.pid` (0600) before it hands it the job, so the Runner that starts next can end one that hangs before its first state. A signal goes only to a child whose exit was not collected, to a process that answers on the job's socket, or to one that runs the Runner's binary, is not the Runner and started no later than the time written down for it (`is_ours`, through `Platform::process`): a number that went to a later process is sent nothing. `service.rs`: `Service` (`discover` repositories and labels and, on a host that still holds the App's key (`HeldKey`), asks the console's scale set calls about each served repository in turn (the next after a refusal of one, none after a console that cannot be reached or has no such calls): once one answered, `HeldKey::take_out` and a start again, else the key serves and the report's `appKeyKept` says why, one supervised listener per scale set, `sweep` of registrations nothing stands behind, `report` to the console with the job events of the outbox and how long jobs handed here waited while nothing ran, with a WARN after `STALL_WARN_AFTER`; the sweep is the one place that removes a registration: those of slots in `Asking`, and of machines that ended while theirs stands, after `RESULT_WAIT` for one whose guest came up with no word of a job; each answer goes to `Machines::answered(runner, Answer)`: `Removed` (204) is no job, `Gone` (404) under a guest that came up is a job over, `Refused` (422) a job that runs, a job implied so is stored as inferred, `discover` and `look_at_queue` run in a task beside `run`'s loop, so a slow GitHub holds up no report and no stop; `look_at_queue`, about once a minute, serves the label sets of queued and running jobs that the default branches' workflows do not name, under `queued.rs`'s `Queued::select`: each for `queued_keep` (30 min) after a job of it was last seen, `queued_bounds` (8 a repository, 32 a host) the newest first, none of a repository it does not serve, none of a fork's pull request on a public repository, and none dropped while that repository's queue cannot be read (for up to `LONGEST_WAIT`, 24 h); a change lists the scale sets of only the repositories whose sets changed; `survey` for `check`, `retire` for `uninstall`, `run`). Each look at the queue also finds the jobs GitHub holds back from a scale set with room (`held.rs`, pure: queued two minutes while every poll of the set offered room beyond GitHub's assigned count for as long, `SetState::room_since`), names them in the report's `heldJobs`, and gives the set one Actions runner of its own at a time, one in five minutes at most and none while a queued job of the repository asks for only some of its labels (`held::narrower_waits`), in a slot the budget gives: registered outside the scale set (`SetRegistry::own`, scale set number `held::OWN`, 0) with exactly its labels and started by `scaler::start_runner`, as a scale set's are; it is retired once its set's queue holds no job it could take, or after five minutes. What the Runner believes about each machine and its job, which evidence moves it and what is written before what is done, is `docs/job-state.md`. `scaler.rs`: `Scaler` (reserve before acquire, release for jobs taken elsewhere, converge on assigned jobs, each place in line with the turn of the assigned job it stands for (`Slots::set_turn`), offers taken overdue first, then main and deploy jobs; `capacity` offers a paused or held Mac's polls only what runs and what was promised (`Slots::held`), and no room to a job in line `PLACE_LIMIT` (10 min) after it was first assigned here, taken back and handed here again or not (one away as long is a re-run, and new), so GitHub's take-back sends it to another machine, retire surplus after `IDLE_GRACE` (`Hosts::retire`: idle machines ask for their registration's removal, which the sweep does); before a session opens, `reconcile` asks GitHub's REST view of each Actions runner it believes waits, since messages go with a closed session, and a busy one took a job (`Answer::Busy`); a session opens only at start, after GitHub forgot one, after a listener failed, or after a restart), `Roster` (an Actions runner belongs to a repository's scale set: GitHub numbers scale sets within a repository, so the number alone names none), `turns.rs`: `turn_of`, whether a job is the main branch's or a deploy, from its workflow's ref, name and event. `names` (runner and scale set names, `ServePolicy`; a scale set's name ends in a hash of its repository and labels, so two repositories that ask for the same labels each get their own). `update.rs`: `SettingsFile` (what the Runner's configuration file says now against what it started with, `SettingsLook`; `change` writes the console's `settings` command, read by `SettingsChange::from_args`, which names each of the cf-runners manager's settings the Runner does not take; `Service::look_at_settings` takes a new one up as an update does, waiting in `drained`, the same wait, for no machine to stand, then starting again), `SelfUpdate` (`build` takes the commit `update-to` names, or none for main's head; `takes_head` false lists no plain `update` and refuses one with `NO_HEAD`; `end` stops a build or a wait when the daemon ends; `first_step` is what the console shows meanwhile), `commit_from_args`, the console's `update-to` arguments (`{"commit"}` alone, `is_commit`: 7 to 40 lowercase hex characters), `Pending` in `update.json`, `at_start` (`Trying`, `GoBack` after `MAX_STARTS` starts that did not stay up for `STAYS_UP`, `Failed`, also when a program before could not be put back: `Pending::failed`); the service builds while jobs run, then holds new jobs until no machine stands and no slot promised to GitHub is left (`Machines::coming`), installs and ends (`update`, `restart`; one at a time, the result said once by the Runner that starts next). `platform/`: `host_service`, the service on a real host, which the binary and `cargo xtask it` both build, over a `Platform` (`This` is this build's): what differs between systems and nothing else: the host's reader and the budget's settings (`Settings::for_host` on the performance cores, `on_fast_cores` with `cf_host_macos::performance_cores`; on Linux `for_headless` with the fast cores), the boot session, host name, model and what the console's page shows (`extras`), `keep_state` (Time Machine; nothing on Linux), `fit` (the virtualization entitlement; on Linux `codefusion-runner vmm` says the build carries cloud-hypervisor), `process` (program and start, for `is_ours`; on Linux `cf_host_linux::process`), `program` (the program alone, for a kill's wait), `vm_hosts` and `SERVICE_LOG`. `mac.rs` is macOS, `linux.rs` Linux, where each vm-host starts in a systemd scope of its own (`VmHosts::in_scopes`, `systemd-run --user --scope -p Delegate=yes`, `codefusion-runner-.scope`), whose delegated cgroup it claims for its machines, and `user_bus_env` finds the user's systemd under `sudo -iu`, and not another user's that `su` left in the environment (`this_user` names the user from the user database). `tests/scopes.rs` (Linux, no harness) starts a stand-in vm-host so and checks it claims the scope's cgroup. `test_run/`: the console's `test-run` command (`TestRun::from_args`, its arguments checked, the run's `url` on the Runner's own console; `admit`, an urgent claim of the budget's as `test-run` that goes before the CI jobs in line, busy while the host takes no new job, a test run is on it or a Docker run finds the Docker jobs full; `wait_turn` for `waitS`, withdrawn as soon as the console's answer no longer carries the command), started by `Machines::start_test_run` as a machine with no Actions runner: its record is `unregistered` from the start with `test_run` set, its job starts when its guest is up and ends with the machine, and `Service::test_run` answers `started tr-…`, `busy: …` or `never: …`, later for a run that waits. `agent.mjs` is the workload (`workload()`: `node --input-type=module -e `, env `CF_RUN_URL`, `CF_RUN_TOKEN`, `CF_RUN_TIMEOUT_S`), the agent that fetches the bundles, runs the suite in its own process group, spools its output decoded as UTF-8 (U+FFFD for what is not) and sends it to the console; `agent.test.mjs` its `node --test` tests against a fake console (CI's Linux job). Tests: `tests/test_run.rs`, and in `tests/service.rs` the command end to end. `docker_check.rs`: whether Docker starts in this host's job machines (`DockerCheck`: `of_end` reads the check machine's end and its console log's "Docker sandbox ended" line, `due` at the start and every `RECHECK_AFTER` (6 h) while it fails, `None` from `of_end` for a machine that ended for something else (slept, evicted, stopped), checked again after `AGAIN_AFTER_NO_ANSWER` (10 min, `Machines::docker_check_due`), `refusal` while it failed; a failure is kept across starts by the `Store`, `docker_failure` and `keep_docker_failure`, in `DiskStore`'s `docker-check.failed`, a pass is not); `Machines::start_docker_probe` boots the check's machine in a claimed slot (no Actions runner, `Record::probe`, nothing reported), `Service::look_at_docker` on every tick, and a failed check drops the `cf-docker` sets (`size_here`) and answers a test run with Docker `never:`; tests `tests/docker_check.rs` and in `tests/service.rs`. `log.rs`: `Lines`, the log file that is rolled over. `testing` has the fakes, and `CrashingStore`, records at which a test makes the daemon die between any two writes. Tests: `machines.rs` (every end kind, a crash at every step), `records.rs` (every record main and earlier builds could have written, read by `Record::upgrade`), `interleavings.rs` (the property test), `schedules.rs` (the property test of the job facts against a GitHub that keeps the truth: crashes at any write, every answer to a removal, lost and late messages, records left in older formats; with pinned schedules), `offers.rs` (the property test of what several listeners of one size offer), `making_room.rs` (the mini's cf-6g job against the cf-2g ones that keep coming), `calm.rs` (the calm minute against time that passes within a look, and against everything that is really wrong), `service.rs` (GitHub's and the console's failures; the label sets of queued jobs: served, kept 30 min, bounded, refused, unreadable, 304, a fork's on a public repository; the property test that no job gets two machines, a session opens only for a cause, a job that waits while a machine fits moves at GitHub's next look, and every job that ran is heard, start and end once each, however many messages went with a session, with a stale or failing REST view, a crash between any two writes, five minutes that take an assignment back, removals that fail until GitHub recovers, and an uninstall in the middle; machines that boot slowly, so a start or an end comes before the Runner saw one up; no VM whose job runs is stopped, and every single write's facts agree (`CrashingStore::disagreements`); a second property aims the same checks at surplus removals), `scaler.rs`, `listening.rs`, `docker.rs` (a Docker job's daemon flag, the cap of `docker_max_jobs` in `Machines`, and `ServePolicy::serves_docker`: listed and private only). A trusted host (`ServePolicy::trusted`) makes no scale set: `Service::ask_trusted` asks for one Actions runner with the label `trusted` at a time (`trusted_every`, 15 s), in the largest machine it runs, whose `JobRequest::trusted` carries the rules; a 409 with `Retry-After` (the console's `no_trusted_job`) is a wait. It takes no test run, boots no Docker check and reports `class: trusted`. | -| `crates/ghapp` (`cf-ghapp`) | `release.rs`: `Releases`, a public repository's releases read with no token (`by_tag`, `page`, `branch_head`, `compare`), and `download`, a release's file streamed into a `Sink` and hashed as it is written, followed across GitHub's redirect to its storage with no token, begun again from the first byte after a break, a 429, a used-up rate limit or a 5xx, ended at once by a full disk (`Error::Local`), a size over its limit or the stop flag, also while a connection stalls or a retry waits (`tests/release.rs`). `http.rs` `retry_after` reads `Retry-After`, else a used-up rate limit's `x-ratelimit-reset`. The GitHub App client. Its tokens come from a `TokenSource` (`source.rs`), the console's (`cf_console_client::ConsoleTokens`), through `GitHubApp::with_source`: no machine holds the App's key, a registration token or the Actions service's admin token, except one that still holds the key until the console's scale set calls answered it (`KeyTokens`, `key.rs`: the App JWT of `jwt.rs`, installation tokens and `registration_token`; `AppCredentials::load` in `credentials.rs`, `app.json` and `app.pem`, which must be 0600; tests in `tests/key.rs`); a call about a repository passes it, so a source whose installation IDs differ finds the one that covers it; `Monotonic` is the clock a token's lifetime is counted on. `rest.rs` is the REST plumbing both share (a 401 tells the source and tries once more). The clock skew is learned from the `Date` header (`clock.rs`). Repositories, workflow files (the caller names the largest file to read), repository variables, `waiting_jobs` (the jobs of a repository's queued and running runs of the label sets served, `cf_workflows::LabelSet::served`, from the REST view, each saying whether its run is a fork's, under bounds: `WAITING_RUNS` of the newest `RUNS_LISTED` runs, queued ones before those under way and each the oldest first, `JOBS_PAGE` jobs of one, `MAX_RUN_SETS` distinct sets of one run; the listings asked for again with the ETag of their last answer, so a 304 reuses it, and each run's jobs asked for again every look with theirs (a run's `updated_at` stays put while its jobs start and end), and `forget_queues` drops what repositories no longer served last read as; each set names its queued jobs, the oldest first), `generate_jit_config` (an Actions runner for one job outside any scale set, with exactly the labels given, sent once); `Http::new(timeout, retry)` with retries (429 and 5xx with jitter, `Retry-After`), HTTPS only (or this machine itself), redirects only to the same origin, at most `MAX_BODY_BYTES` of an answer and 100 pages of a listing. No error and no `Debug` carries an answer's text. Tests in `tests/app.rs`, run against `cf-fake-github`. | +| `crates/daemon` (`cf-daemon`) | The service. `holds.rs`: holds set on the machine itself, one file each in `/holds` (`set`, `lift`, `read`, which holds on a damaged file or an unreadable folder), and `/status.json` (`LocalStatus`: the holds the Runner saw, its machines and the ones coming, written on a change and every `STATUS_EVERY`); `Service::look_at_holds`, each tick, takes no new job while any hold or the console's pause is set and reports them as `status.holds`. `machines.rs`: `Machines` joins the budget, the lifecycle of each slot and the vm-hosts: its scale sets share the room their polls offer GitHub (`offer_room`: a free slot one poll offers no other does until that poll ends, `OFFER_STANDS` at most, and sets with jobs queued, `set_demand`, go before those without), so GitHub never assigns more jobs than it can start; a job GitHub assigned is counted (`assign`), started when memory allows (a machine counts with what its vm-host read it holds of the Mac's memory, `count_held`, its job's peak only when that is not known; the log says once when it is not, or is more than the machine is counted with, `Budget::host_bytes`), registered just in time, watched every tick, and ended once with an `EndKind` (`ends.rs`); `restore` adopts what a Runner before left from the records (`records.rs`, `DiskStore`; a record stores facts, `up`, `ran_job`, `completed`, `unregistered`, and carries the `RULES` it was written under; `Record::agrees` says its facts agree, and `Record::upgrade` is the one place that reads an older Runner's record: its beliefs cleared, GitHub's stored word replayed; a current record that disagrees is an error) and the vm-hosts that still answer, in the same start of the Mac (`kern.bootsessionuuid`, not the boot time, which moves with the clock); one that is silent or wrote nothing yet is killed, and so is what has no record. A tick looks at the vm-hosts without the machines' lock, so a vm-host that hangs keeps no listener waiting. What the polls in flight may still offer GitHub is kept (`Offer`): every answer a listener got since all bells last rang, since many scale sets of one size each ask at their own moment. The last slot taken rings their bells at once, room that went or came back with the Mac's memory at a look, at most every 5 s, so no poll in flight goes on offering a slot that is gone, or none while there is one. An open Mac whose other programs leave no memory for a size it serves says so in the log and is closed for `memory` in the console's report (`Snapshot::short_of`); one that would carry the size without its own machines is full, not short. `Snapshot::rooms` is what one more machine of each size meets, for the report's `sizes` and each scale set's capacity: how many fit, the budget's word when none does, `never` for a size the settings never run, and for memory what it needs and lacks. `making.rs`: `Making`, room made for a larger size. A job that waits for the room the Mac's own machines hold, all of them smaller, while the empty Mac would run it, has every other size held back until it starts, for `MAKE_ROOM_LIMIT` (30 min) at most and then none for as long; its scale set offers a place in line while GitHub has the job back, for `TAKEN_BACK_WAIT` (2 min); an empty Mac whose other programs hold the room gives up after `NO_ROOM_WAIT`. A size whose polls found no room for `STARVED_AFTER` (10 min) is offered a place before any of its jobs is assigned; a size the empty Mac cannot run never is. A headless host, one of a pool, makes room and offers a place only for the size the console's answer names it (`makeRoomFor`, `Making::name`, `Machines::make_room_for`), at once. `vms.rs`: `Vms`, the seam to `codefusion-runner vm`, and `VmHosts`, the real one. It writes each vm-host's process and start time to `run/.pid` (0600) before it hands it the job, so the Runner that starts next can end one that hangs before its first state. A signal goes only to a child whose exit was not collected, to a process that answers on the job's socket, or to one that runs the Runner's binary, is not the Runner and started no later than the time written down for it (`is_ours`, through `Platform::process`): a number that went to a later process is sent nothing. `service.rs`: `Service` (`discover` repositories and labels and, on a host that still holds the App's key (`HeldKey`), asks the console's scale set calls about each served repository in turn (the next after a refusal of one, none after a console that cannot be reached or has no such calls): once one answered, `HeldKey::take_out` and a start again, else the key serves and the report's `appKeyKept` says why, one supervised listener per scale set, `sweep` of registrations nothing stands behind, `report` to the console with the job events of the outbox and how long jobs handed here waited while nothing ran, with a WARN after `STALL_WARN_AFTER`; the sweep is the one place that removes a registration: those of slots in `Asking`, and of machines that ended while theirs stands, after `RESULT_WAIT` for one whose guest came up with no word of a job; each answer goes to `Machines::answered(runner, Answer)`: `Removed` (204) is no job, `Gone` (404) under a guest that came up is a job over, `Refused` (422) a job that runs, a job implied so is stored as inferred, `discover` and `look_at_queue` run in a task beside `run`'s loop, so a slow GitHub holds up no report and no stop; `look_at_queue`, about once a minute, serves the label sets of queued and running jobs that the default branches' workflows do not name, under `queued.rs`'s `Queued::select`: each for `queued_keep` (30 min) after a job of it was last seen, `queued_bounds` (8 a repository, 32 a host) the newest first, none of a repository it does not serve, none of a fork's pull request on a public repository, and none dropped while that repository's queue cannot be read (for up to `LONGEST_WAIT`, 24 h); a change lists the scale sets of only the repositories whose sets changed; `survey` for `check`, `retire` for `uninstall`, `run`). Each look at the queue also finds the jobs GitHub holds back from a scale set with room (`held.rs`, pure: queued two minutes while every poll of the set offered room beyond GitHub's assigned count for as long, `SetState::room_since`), names them in the report's `heldJobs`, and gives the set one Actions runner of its own at a time, one in five minutes at most and none while a queued job of the repository asks for only some of its labels (`held::narrower_waits`), in a slot the budget gives: registered outside the scale set (`SetRegistry::own`, scale set number `held::OWN`, 0) with exactly its labels and started by `scaler::start_runner`, as a scale set's are; it is retired once its set's queue holds no job it could take, or after five minutes. What the Runner believes about each machine and its job, which evidence moves it and what is written before what is done, is `docs/job-state.md`. `scaler.rs`: `Scaler` (reserve before acquire, release for jobs taken elsewhere, converge on assigned jobs, each place in line with the turn of the assigned job it stands for (`Slots::set_turn`), offers taken overdue first, then main and deploy jobs; `capacity` offers a paused or held Mac's polls only what runs and what was promised (`Slots::held`), and no room to a job in line `PLACE_LIMIT` (10 min) after it was first assigned here, taken back and handed here again or not (one away as long is a re-run, and new), so GitHub's take-back sends it to another machine, retire surplus after `IDLE_GRACE` (`Hosts::retire`: idle machines ask for their registration's removal, which the sweep does); before a session opens, `reconcile` asks GitHub's REST view of each Actions runner it believes waits, since messages go with a closed session, and a busy one took a job (`Answer::Busy`); a session opens only at start, after GitHub forgot one, after a listener failed, or after a restart), `Roster` (an Actions runner belongs to a repository's scale set: GitHub numbers scale sets within a repository, so the number alone names none), `turns.rs`: `turn_of`, whether a job is the main branch's or a deploy, from its workflow's ref, name and event. `names` (runner and scale set names, `ServePolicy`; a scale set's name ends in a hash of its repository and labels, so two repositories that ask for the same labels each get their own). `update.rs`: `SettingsFile` (what the Runner's configuration file says now against what it started with, `SettingsLook`; `change` writes the console's `settings` command, read by `SettingsChange::from_args`, which names each of the cf-runners manager's settings the Runner does not take; `Service::look_at_settings` takes a new one up as an update does, waiting in `drained`, the same wait, for no machine to stand, then starting again), `SelfUpdate` (`build` takes the commit `update-to` names, or none for main's head; `updates_from`: `UpdateFrom::Checkout` by default, `Releases` lists no plain `update`, refuses one with `NO_HEAD` and shows `fetching its release` in place of `building` (`takes_head`, `first_step`); `end` stops a build or a wait when the daemon ends), `commit_from_args`, the console's `update-to` arguments (`{"commit"}` alone, `is_commit`: 7 to 40 lowercase hex characters), `Pending` in `update.json`, `at_start` (`Trying`, `GoBack` after `MAX_STARTS` starts that did not stay up for `STAYS_UP`, `Failed`, also when a program before could not be put back: `Pending::failed`); the service builds while jobs run, then holds new jobs until no machine stands and no slot promised to GitHub is left (`Machines::coming`), installs and ends (`update`, `restart`; one at a time, the result said once by the Runner that starts next). `platform/`: `host_service`, the service on a real host, which the binary and `cargo xtask it` both build, over a `Platform` (`This` is this build's): what differs between systems and nothing else: the host's reader and the budget's settings (`Settings::for_host` on the performance cores, `on_fast_cores` with `cf_host_macos::performance_cores`; on Linux `for_headless` with the fast cores), the boot session, host name, model and what the console's page shows (`extras`), `keep_state` (Time Machine; nothing on Linux), `fit` (the virtualization entitlement; on Linux `codefusion-runner vmm` says the build carries cloud-hypervisor), `process` (program and start, for `is_ours`; on Linux `cf_host_linux::process`), `program` (the program alone, for a kill's wait), `vm_hosts` and `SERVICE_LOG`. `mac.rs` is macOS, `linux.rs` Linux, where each vm-host starts in a systemd scope of its own (`VmHosts::in_scopes`, `systemd-run --user --scope -p Delegate=yes`, `codefusion-runner-.scope`), whose delegated cgroup it claims for its machines, and `user_bus_env` finds the user's systemd under `sudo -iu`, and not another user's that `su` left in the environment (`this_user` names the user from the user database). `tests/scopes.rs` (Linux, no harness) starts a stand-in vm-host so and checks it claims the scope's cgroup. `test_run/`: the console's `test-run` command (`TestRun::from_args`, its arguments checked, the run's `url` on the Runner's own console; `admit`, an urgent claim of the budget's as `test-run` that goes before the CI jobs in line, busy while the host takes no new job, a test run is on it or a Docker run finds the Docker jobs full; `wait_turn` for `waitS`, withdrawn as soon as the console's answer no longer carries the command), started by `Machines::start_test_run` as a machine with no Actions runner: its record is `unregistered` from the start with `test_run` set, its job starts when its guest is up and ends with the machine, and `Service::test_run` answers `started tr-…`, `busy: …` or `never: …`, later for a run that waits. `agent.mjs` is the workload (`workload()`: `node --input-type=module -e `, env `CF_RUN_URL`, `CF_RUN_TOKEN`, `CF_RUN_TIMEOUT_S`), the agent that fetches the bundles, runs the suite in its own process group, spools its output decoded as UTF-8 (U+FFFD for what is not) and sends it to the console; `agent.test.mjs` its `node --test` tests against a fake console (CI's Linux job). Tests: `tests/test_run.rs`, and in `tests/service.rs` the command end to end. `docker_check.rs`: whether Docker starts in this host's job machines (`DockerCheck`: `of_end` reads the check machine's end and its console log's "Docker sandbox ended" line, `due` at the start and every `RECHECK_AFTER` (6 h) while it fails, `None` from `of_end` for a machine that ended for something else (slept, evicted, stopped), checked again after `AGAIN_AFTER_NO_ANSWER` (10 min, `Machines::docker_check_due`), `refusal` while it failed; a failure is kept across starts by the `Store`, `docker_failure` and `keep_docker_failure`, in `DiskStore`'s `docker-check.failed`, a pass is not); `Machines::start_docker_probe` boots the check's machine in a claimed slot (no Actions runner, `Record::probe`, nothing reported), `Service::look_at_docker` on every tick, and a failed check drops the `cf-docker` sets (`size_here`) and answers a test run with Docker `never:`; tests `tests/docker_check.rs` and in `tests/service.rs`. `log.rs`: `Lines`, the log file that is rolled over. `testing` has the fakes, and `CrashingStore`, records at which a test makes the daemon die between any two writes. Tests: `machines.rs` (every end kind, a crash at every step), `records.rs` (every record main and earlier builds could have written, read by `Record::upgrade`), `interleavings.rs` (the property test), `schedules.rs` (the property test of the job facts against a GitHub that keeps the truth: crashes at any write, every answer to a removal, lost and late messages, records left in older formats; with pinned schedules), `offers.rs` (the property test of what several listeners of one size offer), `making_room.rs` (the mini's cf-6g job against the cf-2g ones that keep coming), `calm.rs` (the calm minute against time that passes within a look, and against everything that is really wrong), `service.rs` (GitHub's and the console's failures; the label sets of queued jobs: served, kept 30 min, bounded, refused, unreadable, 304, a fork's on a public repository; the property test that no job gets two machines, a session opens only for a cause, a job that waits while a machine fits moves at GitHub's next look, and every job that ran is heard, start and end once each, however many messages went with a session, with a stale or failing REST view, a crash between any two writes, five minutes that take an assignment back, removals that fail until GitHub recovers, and an uninstall in the middle; machines that boot slowly, so a start or an end comes before the Runner saw one up; no VM whose job runs is stopped, and every single write's facts agree (`CrashingStore::disagreements`); a second property aims the same checks at surplus removals), `scaler.rs`, `listening.rs`, `docker.rs` (a Docker job's daemon flag, the cap of `docker_max_jobs` in `Machines`, and `ServePolicy::serves_docker`: listed and private only). A trusted host (`ServePolicy::trusted`) makes no scale set: `Service::ask_trusted` asks for one Actions runner with the label `trusted` at a time (`trusted_every`, 15 s), in the largest machine it runs, whose `JobRequest::trusted` carries the rules; a 409 with `Retry-After` (the console's `no_trusted_job`) is a wait. It takes no test run, boots no Docker check and reports `class: trusted`. | +| `crates/ghapp` (`cf-ghapp`) | `release.rs`: `Releases`, a public repository's releases read with no token (`by_tag`, `page`, `branch_head`, `compare`), and `download`, a release's file streamed into a `Sink` and hashed as it is written, followed across GitHub's redirect to its storage with no token, begun again from the first byte after a break, a 429, a used-up rate limit or a 5xx, ended at once by a full disk (`Error::Local`), a size over its limit or the stop flag, also while a connection stalls or a retry waits (`sleep_unless`) (`tests/release.rs`). `http.rs`: `retry_after` reads `Retry-After`, else a used-up rate limit's `x-ratelimit-reset`; `RetryPolicy::wait_after` is the one rule of when and how long both `Http::send` and a download try again; `file_client` follows a release file's redirect to another host and never carries a token. The GitHub App client. Its tokens come from a `TokenSource` (`source.rs`), the console's (`cf_console_client::ConsoleTokens`), through `GitHubApp::with_source`: no machine holds the App's key, a registration token or the Actions service's admin token, except one that still holds the key until the console's scale set calls answered it (`KeyTokens`, `key.rs`: the App JWT of `jwt.rs`, installation tokens and `registration_token`; `AppCredentials::load` in `credentials.rs`, `app.json` and `app.pem`, which must be 0600; tests in `tests/key.rs`); a call about a repository passes it, so a source whose installation IDs differ finds the one that covers it; `Monotonic` is the clock a token's lifetime is counted on. `rest.rs` is the REST plumbing both share (a 401 tells the source and tries once more). The clock skew is learned from the `Date` header (`clock.rs`). Repositories, workflow files (the caller names the largest file to read), repository variables, `waiting_jobs` (the jobs of a repository's queued and running runs of the label sets served, `cf_workflows::LabelSet::served`, from the REST view, each saying whether its run is a fork's, under bounds: `WAITING_RUNS` of the newest `RUNS_LISTED` runs, queued ones before those under way and each the oldest first, `JOBS_PAGE` jobs of one, `MAX_RUN_SETS` distinct sets of one run; the listings asked for again with the ETag of their last answer, so a 304 reuses it, and each run's jobs asked for again every look with theirs (a run's `updated_at` stays put while its jobs start and end), and `forget_queues` drops what repositories no longer served last read as; each set names its queued jobs, the oldest first), `generate_jit_config` (an Actions runner for one job outside any scale set, with exactly the labels given, sent once); `Http::new(timeout, retry)` with retries (429 and 5xx with jitter, `Retry-After`), HTTPS only (or this machine itself), redirects only to the same origin, at most `MAX_BODY_BYTES` of an answer and 100 pages of a listing. No error and no `Debug` carries an answer's text. Tests in `tests/app.rs`, run against `cf-fake-github`. | | `crates/workflows` (`cf-workflows`) | `self_hosted` / `self_hosted_with`: the label sets a repository's workflows ask for (port of the Go `workflows` package, tests in `tests/labels.rs`), `size_gb` for the `cf-2g/4g/6g/8g` labels, `self-hosted` = 4 GB, `DOCKER_LABEL` (`cf-docker`, served alone or with a size), `is_host_label` and `LabelSet::host_labels` (`cf-macos` and the like, served only by hosts whose config `labels` has them, and a host whose `only_labels` names some of them serves only the sets that ask for one: `ServePolicy::serves_labels` and `labels_refusal` in `crates/daemon/src/names.rs`), the `vars.*` expressions evaluated with the repository's variables by `expression.rs`, a reader of GitHub's expressions over variables and literals: `||`, parentheses, `format` (`{0}`, `{{`/`}}` escapes), `fromJSON` and `toJSON`, nested at most `MAX_DEPTH` deep, no value over `MAX_VALUE_BYTES` (a JSON list under `fromJSON`, with no `toJSON` anywhere in the expression, is several labels; what it cannot read stands for the pool label), `LabelSet::served`, the rules every set to serve keeps, a workflow's or a queued job's (one label of ours, no other platform, at most `MAX_SET_LABELS` labels of at most `MAX_LABEL_BYTES`, letters, digits, `-`, `_` and `.` only), and `MAX_FILE_BYTES`, the largest workflow file read. `LabelSet::served` refuses every set with `TRUSTED_LABEL` in any spelling GitHub's case-blind match takes for it (`is_trusted_label`: `TRUSTED`, `truſted`), from a list or a variable: no shared host takes a trusted job. | | `crates/secret` (`cf-secret`) | `Secret`: a value whose `Debug` prints `Secret(..)`. Every token and key is wrapped in it. `replace_private`: a file for this user alone (0600), put in place in one step; `replace_private_as` with a mode of the user's own bits (0500 for a program). `private_dir`: a folder for this user alone (0700). | | `crates/fake-github` (`cf-fake-github`) | A stateful fake GitHub over real sockets, for tests only: App JWT checks (`test_key`), installation tokens, repositories, scale sets, sessions, long poll, acquire, JIT configs, with `Options::auto_assign` (no kinder than the real service, each rule beside its evidence in `docs/github-protocol.md`: jobs go out only when a job is queued, ends, is cancelled or taken, or a test says GitHub looks (`github_looks`) or an Actions runner registered outside any scale set comes online (`runner_online`; `plain_runners` lists them, and `start_job` gives one the oldest queued job of its repository whose labels it has), against the smaller room of a scale set's last two polls; an opening hands out nothing; the counts never name a held job; a session that ends takes its pending messages; `offers_room` says when a job of a repository queued now goes out at once, `offers_room_for` when a job that waits goes out at GitHub's next look; only a session's newest poll is served), an assignment no Actions runner took is taken back after `TAKE_BACK_AFTER` on the queue's clock (`pass`), with faults (`fail`, `before`, `lose_answer`, `lose_session`, `expire_session_token`, `revoke_admin_tokens`, `send_the_service_to`, `send_the_queue_to`, clock offset, `set_rest_stale`: the REST view of an Actions runner one read behind, `set_down`: every request dropped, as for a client that died), and `assignments`, the hand-outs so far. What the console does with its own App's key, for `cf-fake-console`: `mint_read_token` (a token GitHub refuses every write, 403), `mint_registration_token`, `mint_plain_jit`, `revoke_token`, `token_works`, `revoke_installation_tokens`. `TestClock` is the wall clock and the monotonic one. `queue_job_at` queues a job at a ref, which its message's `jobWorkflowRef` and `eventName` show; `queue_run` the jobs of one run; `end_job` ends one as its run shows it; `from_fork` makes a job's run a pull request's from a fork. The REST view of runs and their jobs (`/actions/runs?status=`, `/actions/runs/{id}/jobs`) answers with an ETag and 304 to a request that names it (`not_modified` counts them). | | `crates/harness` (`cf-harness`) | What every crate's tests start from (dev-dependency only): `cf-fake-github` behind `cf-fake-console`, with the real clients in front of them as a machine has them (`enrolled`, `app`, `app_on`: `ConsoleTokens` and `ScaleSetClient`), `TestClock`, `fast` retries, `until`, `elsewhere` (a host no token may reach), and what a test asserts no machine did: `registrations` (requests for a registration token or traded one) and `admin_requests_from_machines` (service calls not made by the console, `CONSOLE_MARK`); `key_tokens` and `key_app` for a host that still holds the test key. | | `crates/console-client` (`cf-console-client`) | The client of the CodeFusion Console's runner protocol 1 (`/agent/runners/{enroll,report,leave}`): the wire types (`Enroll` and `Report` with the system they run on (`Os`, `macos` or `linux`, left out when none), `Status` with `commands` (the console's commands this Runner does: `pause` and `resume` (the answer's `paused`) on every system, `update` and `update-to` only with somewhere to update from, `test-run` with a console), `memoryForJobsBytes` (the cap of the limits in force), `idleWithJobsWaitingSeconds` and its reason, `heldJobs` (`HeldJobs`: per repository and label set, the jobs GitHub holds back from a scale set with room and the oldest one's wait, left out when none), `appKeyFile` (a GitHub App key file the host still holds, which `app_key::take_out` could not delete or found linked, or the one it signs with, left out when none), `appKeyKept` (why it still signs with it), `docker` (`DockerStatus`: the last Docker check, `ok`, `checkedAt`, `why`, left out before the first) and `sizes`, what one more machine of each size meets: `free` (-1 for a size the host never runs), `why`, `needBytes`, `shortBytes`, `JobLimits`, the report's `limits` that the `settings` command changes, `RunnerStatus` with its machine's `size` and `countedBytes` (the size the budget's cap counts it with; the vm-hosts' overhead is taken out of the cap), `JobEvent`, `Answer` (with `makeRoomFor`, the size the console names a pool host to make room for), `Command` ...), `enroll`, `Client` (`report`, `deliver`, `leave`; the token is a `Secret`), `Error` (`Unauthorized`, `Refused`, `GithubOff`, `Busy`, `Unavailable`, `Protocol`), GitHub through the console (`github.rs`: `github_read`, `github_jit`, each answer checked: tokens of visible ASCII, lifetimes of at least `MIN_TOKEN_LIFE` and held at most `MAX_TOKEN_LIFE`, unique installation IDs; what the console would refuse is refused unsent; a JIT runner is asked for once and never again by itself; `github_read` names at most `REFUSED_MAX` refused tokens; a 403 with the code `BROKER_OFF` is `Error::GithubOff`, a 429 or 5xx with Retry-After `Error::Busy`, held to `RETRY_AFTER_MAX`), `ConsoleTokens` (`tokens.rs`, the `TokenSource` of the App client: one read for all installations, renewed at half its life and not used `END_MARGIN` before its end on the monotonic clock, concurrent callers sharing one request; a console that fails is asked again after `FAILED_WAIT` doubling to `FAILED_WAIT_MAX` while the last token serves to its end, then every call fails at once with the reason, never asking sooner than the console's Retry-After; a token GitHub refuses is named in the next read so the console replaces it while the other installations' tokens serve, and one handed out again all the same, or new ones GitHub refuses too, are asked for again after a doubling wait; a runner the console answered with a wait is not asked for that repository until it passes (a 429 unsent); only the 403 `broker_off` (the owner's switch for the machine off) fails every call with the console's words for `GATE_RECHECK_ALONE`, and any other 403 is an error with the console's words; installation IDs are mapped by repository, and looked up again after a 404), the scale set calls (`scaleset.rs`: `scaleset_list`, `scaleset_create`, `scaleset_delete`, `scaleset_session_open`, `scaleset_session_refresh`, `scaleset_session_close`, `scaleset_jit`, `scaleset_remove`, each answer checked, the console's refusals by their codes: `not_yours`, `session_exists`, `session_lost`, `name_taken`, `runner_busy`, `rate_limited`; a 503 `github_busy` with its Retry-After is `Error::Busy`) and `Outbox`, the job events that wait for the console in one 0600 file: ids dedup (waiting, and the last 1000 sent), at most 500 kept with the oldest dropped and counted, 100 a report, marked sent only when a valid answer came. A report the console would refuse is refused here without sending. Tests: `tests/contract.rs` (every bound of the protocol against the fake), `client.rs`, `outbox.rs`, `delivery.rs`, `properties.rs`, `tokens.rs` (GitHub through the console end to end, with both fakes). | | `crates/fake-console` (`cf-fake-console`) | A fake CodeFusion Console for tests (dev-dependency only), over `cf_fake_github::httpd`: the checks of the console's `agent.ts` and `protocol.ts` in the same order with the same statuses and bounds (`schema.rs` is protocol.ts field by field), codes, tokens, commands, pause. `withdraw_command` (the console stops sending one). GitHub through the console (`github.rs`, the console's `github.ts`): `serve_github` attaches a `FakeGithub` that makes the tokens, `set_broker` is the owner's switch per machine (off at first; off forgets the kept tokens and revokes the read ones), `scaleset_answer` answers every scale set call with one reply (`no_scaleset_calls`: a console from before them), `set_installations` gives the console's App other installation IDs; kept tokens handed out again while more than `REUSE_MIN_S` is left, a read token named `refused` replaced and revoked (`ignore_refused`: a console that does not read the names), the switch's 403 with the code `broker_off`, `JIT_PER_REPO` and `JIT_PER_MACHINE` runners in `JIT_WINDOW_S` then 429 with Retry-After, a runner name not the machine's 403 without a code, before the Runners App is looked for. The scale set routes (`scaleset.rs`, `github/scaleset/{list,create,delete,session/open,session/refresh,session/close,jit,remove}`): the admin token made in-process and kept while more than `ADMIN_REUSE_S` is left (`forget_admin_tokens`), each call scoped to the machine's own scale sets and Actions runners (403 `not_yours`), rate limits per machine in `RATE_WINDOW_S`, GitHub's 5xx, 429, a drop or a timeout (`GITHUB_DEADLINE`) a 503 `github_busy` with Retry-After (30 s, or `github_busy_wait` for tests), and an audit row per call (`scaleset_audit`). Faults: `down`, `reports_down` (everything but `github/…`), `fail_next`, `fail_next_waiting` (with Retry-After), `drop_before_next`, `accept_then_drop_next`, `accept_then_drop_next_report`, `answer_next` (any answer to the next `github/…` call). What it saw: `reports`, `job_events`, `enrollments`, `tokens`, `calls`, `github_calls`, and `os_of`, the system the console keeps for a machine (`macos`, `linux`, none for anything else, which is never refused). Written from the console's code and not from the client's types. | -| `scripts` | `host-setup.sh`: a Linux host made a Runner of the console, unattended, as root, with no App key: `install` (packages, the Runner's user, user namespaces, the Runner built from its checkout or brought as `--binary`/`--version` with its `--sha256`, the newest image, the configuration with `--settings`; needs no code, runs any number of times, ends on `host-setup: changed` or `unchanged`, `--dry-run` says each change and a diff of the configuration; a new Runner on an enrolled host goes in under the hold `host-setup` once running jobs end, exit 75 when they outlast the wait), `enroll --code-file` (the enrolment and the service), or both at once as a cloud VM's startup script does; `merged_config` keeps the keys the Runner and the console set and takes out the ones a last `--settings` set and this one leaves out; `--class trusted` writes `"class": "trusted"` and no rules, so the Runner's defaults apply; `--sandbox container` needs no KVM, sets up the Runner user's rootless Podman (`place_engine`: subordinate ids, `podman.socket`), installs the newest container-image release into it and writes `sandbox` and `engine`; a `--version` is a minimum: `runs_newer` keeps an enrolled host's Runner that runs a newer commit of main (GitHub's compare), and keeps it when GitHub cannot say; its reads of GitHub's API (`github_get`) send `GITHUB_TOKEN` when set and wait out a rate limit, a 5xx or no answer (`xtask/tests/host_setup.rs`, with the merge and the refusal of each option a step does not take); CI's `fresh-host` job runs it against the fakes, both at once from the source and `install`, again, `--dry-run` and `enroll` with a build brought along. `release-gate.sh`: whether "Runner release" may put out a commit: on main, its push's newest CI, Image (when there is one) and Runner build runs green; `publish `, `wait`, `skip`, or a failure when GitHub cannot say (tests in `xtask/tests/host_setup.rs`). `docker-rollout-mac.sh`: on one Mac, installs an image release and names it and the Docker repositories in the Runner's configuration file, then waits for the Runner to start again on it (or says why it did not). | +| `scripts` | `host-setup.sh`: a Linux host made a Runner of the console, unattended, as root, with no App key: `install` (packages, the Runner's user, user namespaces, the Runner built from its checkout or brought as `--binary`/`--version` with its `--sha256`, the newest image, the configuration with `--settings`; needs no code, runs any number of times, ends on `host-setup: changed` or `unchanged`, `--dry-run` says each change and a diff of the configuration; a new Runner on an enrolled host goes in under the hold `host-setup` once running jobs end, exit 75 when they outlast the wait), `enroll --code-file` (the enrolment and the service), or both at once as a cloud VM's startup script does; `merged_config` keeps the keys the Runner and the console set and takes out the ones a last `--settings` set and this one leaves out; `--class trusted` writes `"class": "trusted"` and no rules, so the Runner's defaults apply; `--sandbox container` needs no KVM, sets up the Runner user's rootless Podman (`place_engine`: subordinate ids, `podman.socket`), installs the newest container-image release into it and writes `sandbox` and `engine`; a `--version` is a minimum: `runs_newer` keeps an enrolled host's Runner that runs a newer commit of main (GitHub's compare), and keeps it when GitHub cannot say; its reads of GitHub's API (`github_get`) send `GITHUB_TOKEN` when set and wait out a rate limit, a 5xx or no answer (`xtask/tests/host_setup.rs`, with the merge and the refusal of each option a step does not take); CI's `fresh-host` job runs it against the fakes, both at once from the source and `install`, again, `--dry-run` and `enroll` with a build brought along. `release-gate.sh`: whether "Runner release" may put out a commit: on main, its push's newest CI, Image (when there is one) and Runner build runs green, read in one list of the push's runs; it uses `github_get`, `main_head` and `compare_status` of `host-setup.sh`; `publish `, `wait`, `skip`, or a failure when GitHub cannot say (tests in `xtask/tests/host_setup.rs`). `docker-rollout-mac.sh`: on one Mac, installs an image release and names it and the Docker repositories in the Runner's configuration file, then waits for the Runner to start again on it (or says why it did not). | | `xtask` | `cargo xtask keygen | guest | release | build | it | fakes serve | kernel check | cloud-hypervisor [check]`. `fakes serve` (`xtask/src/fakes.rs`) runs the fake GitHub and console for `scripts/host-setup.sh` and ends once the host it set up serves. `it` (`xtask/src/it.rs`) runs the vm-host against real machines on this host, one small machine at a time, and writes its numbers to `target/it/report.txt`. Its scenarios: `boot`, `lifecycle`, `exit-code`, `oom`, `timeout`, `stop`, `sigterm`, `sigkill`, `firewall` (what a job reaches), `inside` (what a job's root tries against its own machine), `docker` (a `cf-docker` job: Compose, a build, a published port, and what its containers reach, privileged ones too, with a port open on the host as the target that shows the firewall at work), `docker-oom` (a container that takes all the job's memory ends the job `oom`; one stopped at its own limit, in `docker`, does not), `launchd`, `timers`, `full-memory` (what a machine whose job holds all its memory costs the host, against `Settings::VM_HOST_OVERHEAD`, and that its vm-host reads what its machine holds), and in `it/service.rs` `service` (three jobs through the whole service against the fake GitHub and console: to its end, across a Runner that dies, into an uninstall) and `commands` (the binary's `enroll`, `check --github` and `daemon`). A scenario that needs more memory than the host has free says `SKIPPED` and why. Kernel package pinned by version, size and SHA-256 in `xtask/src/kernel.rs`, read from our copy (the release `kernel-linux-virt-`) and from Alpine only when that is missing; `kernel check` goes red when our copy does not give the pinned bytes and warns when Alpine has a newer kernel. How to move the pin is at the top of `kernel.rs`. `cloud_hypervisor.rs` does the same for the Linux monitor pinned in `crates/vmm-ch/src/pins.rs` (our copy: the release `cloud-hypervisor-`), and `pinned.rs` is what both share: the HTTPS download that checks each redirect's host and the bytes, the check of our copy, and `newest`, the version a release or index names. | | `image/Dockerfile` | The root filesystem a job runs in: GitHub's Actions runner image plus tools, for arm64 and amd64 (`TARGETARCH`, each download pinned for each). For Docker jobs: iptables, pasta and the Compose plugin (the engine, CLI and buildx come with the Actions runner image). Packed by `.github/workflows/image.yml`. | | `.github/workflows` | `ci.yml` (macos-15: fmt, guest, clippy, tests, release build; ubuntu-24.04-arm and ubuntu-24.04: clippy and tests of the crates in `LINUX_CRATES`, clippy for the guest's musl target of that architecture, and the Linux probe; ubuntu: cargo deny), `image.yml` (the image of each architecture, and its container-image release beside it: built and verified on every change to it, signed and published from main, weekly), `pins.yml` (`cargo xtask kernel check` and `cargo xtask cloud-hypervisor check`, weekly and on every change to a pin), `runner-build.yml` (the Runner's Linux program of each architecture, built in Debian 12 from nothing on every push to main, kept as artifacts for a week), `release.yml` (on each of CI, Image and Runner build ending for a push to main, `scripts/release-gate.sh`; once all are green, the build's programs released as `runner-` with the `host-setup.sh` of its commit and their SHA256SUMS, never replaced; by hand with `workflow_dispatch` and a commit). Labelled pull requests are merged by CodeFusion Console. | diff --git a/scripts/host-setup.sh b/scripts/host-setup.sh index 171efc3..13e9cd4 100755 --- a/scripts/host-setup.sh +++ b/scripts/host-setup.sh @@ -145,6 +145,26 @@ github_get() { sleep "$wait" done } +# GitHub's compare of $2 with $3 in the repository whose API is $1, on standard output: ahead, behind, identical or +# diverged as GitHub says it, or nothing when GitHub could not say. +compare_status() { + ( + trap - ERR + github_get "$1/compare/$2...$3?per_page=1" | jq -r '.status // empty' + ) || true +} + +# The whole commit main's branch is at in the repository whose API is $1, read from its ref, which no tag named main +# stands for; nothing when GitHub could not say. +main_head() { + local sha + sha=$( + trap - ERR + github_get "$1/git/ref/heads/main" | jq -r '.object.sha // empty' + ) || sha='' + if [[ $sha =~ ^[0-9a-f]{40}$ ]]; then printf '%s\n' "$sha"; fi +} + # The configuration a run writes, on standard output: the one there now ($1, `{}` for none) without the keys the last # run's settings set ($2) and every key this script owns, then the owned keys this run gives ($3, nulls left out) and # its settings ($4), sorted on one line. The keys the Runner or the console set themselves stay. Fails for settings that @@ -186,16 +206,12 @@ modules_wanted() { # when it runs a commit of main newer than $3, `install` when it runs $3, an older commit, one main does not have or # no commit at all, `unknown` when GitHub could not say. runs_newer() { - local api=$1 running=$2 wanted=$3 status head + local api=$1 running=$2 wanted=$3 head if [[ ! $running =~ ^[0-9a-f]{12}$ || $running == "$wanted" ]]; then echo install return 0 fi - status=$( - trap - ERR - github_get "$api/compare/$wanted...$running?per_page=1" | jq -r '.status // empty' - ) || status='' - case $status in + case $(compare_status "$api" "$wanted" "$running") in ahead) ;; behind | diverged) echo install @@ -206,20 +222,13 @@ runs_newer() { return 0 ;; esac - # Newer than the one asked for, and main's: main's branch by its ref, which no tag named main stands for. - head=$( - trap - ERR - github_get "$api/git/ref/heads/main" | jq -r '.object.sha // empty' - ) || head='' - if [[ ! $head =~ ^[0-9a-f]{40}$ ]]; then + # Newer than the one asked for, and main's. + head=$(main_head "$api") + if [[ -z $head ]]; then echo unknown return 0 fi - status=$( - trap - ERR - github_get "$api/compare/$running...$head?per_page=1" | jq -r '.status // empty' - ) || status='' - case $status in + case $(compare_status "$api" "$running" "$head") in ahead | identical) echo keep ;; behind | diverged) echo install ;; *) echo unknown ;; diff --git a/scripts/release-gate.sh b/scripts/release-gate.sh index dd49b9c..1311ed6 100755 --- a/scripts/release-gate.sh +++ b/scripts/release-gate.sh @@ -31,12 +31,12 @@ if [[ ! $commit =~ ^[0-9a-f]{40}$ ]]; then exit 2 fi -main=$(github_get "$api/git/ref/heads/main" | jq -r '.object.sha // empty') -if [[ ! $main =~ ^[0-9a-f]{40}$ ]]; then - echo "release-gate: main's ref names no commit" >&2 +main=$(main_head "$api") +if [[ -z $main ]]; then + echo "release-gate: GitHub could not say which commit main is at" >&2 exit 1 fi -status=$(github_get "$api/compare/$commit...$main?per_page=1" | jq -r '.status // empty') +status=$(compare_status "$api" "$commit" "$main") case $status in ahead | identical) ;; behind | diverged) @@ -49,13 +49,15 @@ case $status in ;; esac -# The newest run of a workflow for the push of the commit to main: ` `, or nothing for none. +# Every run of the push of the commit to main, of every workflow, in one read. +runs=$(github_get "$api/actions/runs?head_sha=$commit&event=push&branch=main&per_page=100") + +# The newest run of the workflow file $1 among them: ` `, or nothing for none. newest_run() { - github_get "$api/actions/workflows/$1/runs?head_sha=$commit&event=push&branch=main&per_page=100" | - jq -r --arg commit "$commit" ' - [.workflow_runs[] | select(.head_sha == $commit and .event == "push" and .head_branch == "main")] - | if length == 0 then empty else max_by(.run_number, .run_attempt) - | "\(.id) \(.status) \(.conclusion // "none")" end' + jq -r --arg commit "$commit" --arg path ".github/workflows/$1" ' + [.workflow_runs[] | select(.path == $path and .head_sha == $commit and .event == "push" and .head_branch == "main")] + | if length == 0 then empty else max_by(.run_number, .run_attempt) + | "\(.id) \(.status) \(.conclusion // "none")" end' <<<"$runs" } # `wait` or `skip` for a run that is not green; nothing for a green one. $1 names the workflow, $2 its run, $3 whether diff --git a/xtask/tests/host_setup.rs b/xtask/tests/host_setup.rs index 0423d31..9aaeee0 100644 --- a/xtask/tests/host_setup.rs +++ b/xtask/tests/host_setup.rs @@ -806,26 +806,29 @@ fn release_gate(api: &str, commit: &str, env: &[(&str, &str)]) -> (Option, } const PUSHED: &str = "dddddddddddd0000000000000000000000000000"; +const CI: &str = "ci.yml"; +const IMAGE: &str = "image.yml"; +const BUILD: &str = "runner-build.yml"; -/// A run of a workflow as GitHub lists it, of the push of `PUSHED` to main unless changed. -fn run(id: u64, number: u64, status: &str, conclusion: Option<&str>) -> serde_json::Value { +/// A run of the workflow file `workflow` as GitHub lists it, of the push of `PUSHED` to main unless changed. +fn run(workflow: &str, id: u64, number: u64, status: &str, conclusion: Option<&str>) -> serde_json::Value { serde_json::json!({ "id": id, "run_number": number, "run_attempt": 1, "status": status, "conclusion": conclusion, - "head_sha": PUSHED, "event": "push", "head_branch": "main", + "head_sha": PUSHED, "event": "push", "head_branch": "main", "path": format!(".github/workflows/{workflow}"), }) } -fn runs(list: &[serde_json::Value]) -> Reply { - Reply::json(&serde_json::json!({ "total_count": list.len(), "workflow_runs": list })) +fn green(workflow: &str, id: u64) -> serde_json::Value { + run(workflow, id, 1, "completed", Some("success")) } -fn green(id: u64) -> Reply { - runs(&[run(id, 1, "completed", Some("success"))]) +fn runs(list: &[serde_json::Value]) -> Reply { + Reply::json(&serde_json::json!({ "total_count": list.len(), "workflow_runs": list })) } -/// The reads before the runs: main's ref, and main has the commit. -fn on_main() -> Vec { - vec![main_at(), compared("ahead")] +/// Main's ref, main has the commit, and the push's runs. +fn on_main(list: &[serde_json::Value]) -> Vec { + vec![main_at(), compared("ahead"), runs(list)] } fn gate_says(answers: Vec) -> (String, usize) { @@ -838,7 +841,7 @@ fn gate_says(answers: Vec) -> (String, usize) { /// A push whose CI, Image when it made one, and Runner build are green gets its release, with that build's programs. #[test] fn a_green_push_of_main_is_released_with_its_build() { - let api = api([on_main(), vec![green(10), runs(&[]), green(30)]].concat()); + let api = api(on_main(&[green(CI, 10), green(BUILD, 30)])); let (status, said, stderr) = release_gate(&api.server.url(), PUSHED, &[]); assert_eq!((status, said.as_str()), (Some(0), "publish 30"), "{stderr}"); let requests = api.requests(); @@ -848,57 +851,50 @@ fn a_green_push_of_main_is_released_with_its_build() { [ "/git/ref/heads/main", &format!("/compare/{PUSHED}...{MAIN}"), - "/actions/workflows/ci.yml/runs", - "/actions/workflows/image.yml/runs", - "/actions/workflows/runner-build.yml/runs", + "/actions/runs" ] ); - for request in &requests[2..] { - for pair in [("head_sha", PUSHED), ("event", "push"), ("branch", "main")] { - assert!( - request.query.contains(&(pair.0.to_string(), pair.1.to_string())), - "{:?}", - request.query - ); - } + for pair in [("head_sha", PUSHED), ("event", "push"), ("branch", "main")] { + assert!( + requests[2] + .query + .contains(&(pair.0.to_string(), pair.1.to_string())), + "{:?}", + requests[2].query + ); } // Main at the commit itself, and an Image run that is green. - let (said, _) = gate_says(vec![ + let (said, asked) = gate_says(vec![ main_at(), compared("identical"), - green(10), - green(20), - green(30), + runs(&[green(BUILD, 30), green(IMAGE, 20), green(CI, 10)]), ]); - assert_eq!(said, "publish 30"); + assert_eq!((said.as_str(), asked), ("publish 30", 3)); } /// The red commit that went out before: a run that ended other than green keeps its commit's release back, as does /// a commit main does not have. #[test] fn a_push_with_a_red_run_or_off_main_gets_no_release() { - let red = |conclusion| runs(&[run(9, 1, "completed", Some(conclusion))]); + let red = |workflow, conclusion| run(workflow, 9, 1, "completed", Some(conclusion)); for (answers, starts) in [ (vec![main_at(), compared("behind")], "skip "), (vec![main_at(), compared("diverged")], "skip "), ( - [on_main(), vec![red("failure")]].concat(), + on_main(&[red(CI, "failure"), green(BUILD, 30)]), "skip CI run 9 ended failure", ), ( - [on_main(), vec![red("cancelled")]].concat(), + on_main(&[red(CI, "cancelled"), green(BUILD, 30)]), "skip CI run 9 ended cancelled", ), + (on_main(&[red(CI, "timed_out")]), "skip CI run 9 ended timed_out"), ( - [on_main(), vec![red("timed_out")]].concat(), - "skip CI run 9 ended timed_out", - ), - ( - [on_main(), vec![green(10), red("failure")]].concat(), + on_main(&[green(CI, 10), red(IMAGE, "failure"), green(BUILD, 30)]), "skip Image run 9", ), ( - [on_main(), vec![green(10), runs(&[]), red("failure")]].concat(), + on_main(&[green(CI, 10), red(BUILD, "failure")]), "skip Runner build run 9", ), ] { @@ -910,64 +906,67 @@ fn a_push_with_a_red_run_or_off_main_gets_no_release() { /// A run not ended, or not started, makes it wait: the workflow_run of the run that ends starts the check again. #[test] fn a_push_whose_runs_have_not_ended_waits() { - let going = |status| runs(&[run(9, 1, status, None)]); - for (answers, says) in [ + let going = |workflow, status| run(workflow, 9, 1, status, None); + for (list, says) in [ ( - [on_main(), vec![going("in_progress")]].concat(), + vec![going(CI, "in_progress"), green(BUILD, 30)], "wait CI run 9 is in_progress", ), + (vec![going(CI, "queued")], "wait CI run 9 is queued"), + (vec![green(BUILD, 30)], "wait CI has not run for this push"), ( - [on_main(), vec![going("queued")]].concat(), - "wait CI run 9 is queued", - ), - ( - [on_main(), vec![runs(&[])]].concat(), - "wait CI has not run for this push", - ), - ( - [on_main(), vec![green(10), going("in_progress")]].concat(), + vec![green(CI, 10), going(IMAGE, "in_progress"), green(BUILD, 30)], "wait Image run 9 is in_progress", ), ( - [on_main(), vec![green(10), runs(&[]), runs(&[])]].concat(), - "wait Runner build has not run for this push", + vec![green(CI, 10), going(BUILD, "in_progress")], + "wait Runner build run 9 is in_progress", ), + (vec![green(CI, 10)], "wait Runner build has not run for this push"), ] { - let (said, _) = gate_says(answers); + let (said, _) = gate_says(on_main(&list)); assert_eq!(said, says); } } /// The newest run of a workflow counts, as on the console: a re-run that went green releases, a newer red one does -/// not; runs of other commits, events or branches in the answer are none of this push's. +/// not; runs of other workflows, commits, events or branches in the answer are none of this one's. #[test] fn the_newest_run_of_the_push_counts() { - let old_red = run(9, 4, "completed", Some("failure")); - let new_green = run(10, 5, "completed", Some("success")); - let mut rerun_green = run(9, 4, "completed", Some("success")); + let old_red = run(CI, 9, 4, "completed", Some("failure")); + let new_green = run(CI, 10, 5, "completed", Some("success")); + let mut rerun_green = run(CI, 9, 4, "completed", Some("success")); rerun_green["run_attempt"] = 2.into(); - let mut first_attempt_red = run(9, 4, "completed", Some("failure")); - first_attempt_red["run_attempt"] = 1.into(); + let first_attempt_red = run(CI, 9, 4, "completed", Some("failure")); for (list, says) in [ (vec![old_red.clone(), new_green.clone()], "publish 30"), (vec![new_green.clone(), old_red.clone()], "publish 30"), (vec![first_attempt_red, rerun_green], "publish 30"), ( - vec![run(11, 6, "completed", Some("failure")), new_green.clone()], + vec![run(CI, 11, 6, "completed", Some("failure")), new_green.clone()], "skip CI run 11 ended failure", ), + // A red run of another workflow file, Pins, is none of these. + ( + vec![ + new_green.clone(), + run("pins.yml", 12, 9, "completed", Some("failure")), + ], + "publish 30", + ), ] { - let (said, _) = gate_says([on_main(), vec![runs(&list), runs(&[]), green(30)]].concat()); + let (said, _) = gate_says(on_main(&[list.clone(), vec![green(BUILD, 30)]].concat())); assert_eq!(said, says, "{list:?}"); } for (field, value) in [ ("head_sha", "eeeeeeeeeeee0000000000000000000000000000"), ("event", "pull_request"), ("head_branch", "main-copy"), + ("path", ".github/workflows/ci-copy.yml"), ] { - let mut other = run(12, 7, "completed", Some("success")); + let mut other = green(CI, 12); other[field] = value.into(); - let (said, _) = gate_says([on_main(), vec![runs(&[other])]].concat()); + let (said, _) = gate_says(on_main(&[other, green(BUILD, 30)])); assert_eq!(said, "wait CI has not run for this push", "{field}"); } } @@ -980,23 +979,29 @@ fn the_gate_waits_out_429_and_5xx_and_fails_when_github_cannot_say() { main_at(), Reply::empty(502), compared("ahead"), - green(10), Reply::empty(503), - runs(&[]), - green(30), + runs(&[green(CI, 10), green(BUILD, 30)]), ]); - assert_eq!((said.as_str(), asked), ("publish 30", 8)); + assert_eq!((said.as_str(), asked), ("publish 30", 6)); let limits = [("GITHUB_TRIES", "2"), ("GITHUB_WAIT_MAX", "1")]; + let mut odd = green(CI, 10); + odd["id"] = "x".into(); for answers in [ vec![Reply::empty(503)], vec![Reply::json(&serde_json::json!({ "object": { "sha": "main" } }))], vec![main_at(), compared("sideways")], - vec![main_at(), Reply::status_json(404, &serde_json::json!({ "message": "Not Found" }))], - [on_main(), vec![Reply::empty(503)]].concat(), - [on_main(), vec![Reply::text(200, "a proxy")]].concat(), - [on_main(), vec![runs(&[serde_json::json!({ "id": "x", "run_number": 1, "run_attempt": 1, "status": "completed", "conclusion": "success", "head_sha": PUSHED, "event": "push", "head_branch": "main" })])]].concat(), - [on_main(), vec![green(10), runs(&[]), Reply::empty(500)]].concat(), + vec![ + main_at(), + Reply::status_json(404, &serde_json::json!({ "message": "Not Found" })), + ], + vec![main_at(), compared("ahead"), Reply::empty(503)], + vec![ + main_at(), + compared("ahead"), + Reply::text(200, "a proxy"), + ], + on_main(&[odd, green(BUILD, 30)]), ] { let api = api(answers.clone()); let (status, said, _) = release_gate(&api.server.url(), PUSHED, &limits); From f40f3eb18a450d18bd893df9e8e7a0e7a6238d71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krystian=20Paw=C5=82ow?= Date: Thu, 12 Nov 2026 21:15:48 +0100 Subject: [PATCH 4/6] Say what a release update cannot do: no release after a failed build, no signature --- .github/workflows/release.yml | 3 ++- README.md | 12 ++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 21af9f1..3e9814e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,7 +6,8 @@ name: Runner release # for the version `0.1.0+446125c0a1b2` the program says), with `codefusion-runner-linux-amd64`, `-arm64`, the # `host-setup.sh` of the same commit and their SHA256SUMS. A host takes them with `host-setup.sh install --version # --sha256 `, and a Runner installed so updates itself to the release of a commit the console names -# (`update-to`), which it refuses when there is none: so no red commit is ever put out, and each green one is. +# (`update-to`), which it refuses when there is none: so no red commit is ever put out, and each green one whose build +# succeeded is. # # Each of the three workflows ending starts the check (scripts/release-gate.sh); the one that ends last finds them all # green and publishes. A red run, a commit main does not have, or one whose runs have not ended gets no release, and a diff --git a/README.md b/README.md index 394e050..069747c 100644 --- a/README.md +++ b/README.md @@ -182,10 +182,14 @@ the whole commit, on `main` and newer than the commit that runs (GitHub's compar `version`, which must run on this host and name the commit, and then drains, installs and checks the start as a build does. A commit whose release is not out yet is waited for up to 45 minutes, asked every 3; a commit with no release, an older one, a checksum that does not match, a missing file, a program of another architecture or a disk with no -room leaves the running program as it was and says why. Releases are not signed: the checksum guards against a -broken download, and HTTPS to GitHub and the publisher's name against anyone else. "Runner release" puts out a commit -only once its push's CI, Image when it made one, and "Runner build" are green on main (`scripts/release-gate.sh`), the -rule the console rolls out by, so every commit the console names has a release. +room leaves the running program as it was and says why. GitHub answers 60 reads an hour to an address without a +token; a used-up limit is waited out, at most 15 minutes a read. Releases are not signed: the checksum guards against a +broken download, HTTPS against anyone between the host and GitHub, and the publisher's name against a release someone +made by hand; anyone who can change this repository's workflows on main can still publish. "Runner release" puts out +a commit only once its push's CI, Image when it made one, and "Runner build" are green on main +(`scripts/release-gate.sh`), the rule the console rolls out by, plus the build; a commit whose build failed has no +release, and its `update-to` is refused. A Runner installed from a release before this could not update itself: one +`host-setup.sh install --version` of a newer release puts in one that can. A configuration written before the console held the App's key names `app_dir`, a folder with the key (`app.pem`) beside `app.json`. While the key file is there the Runner signs with it and makes its scale set calls itself, as before, From cad5ab6507562e356b0147b41a0f5240c3c802c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krystian=20Paw=C5=82ow?= Date: Thu, 12 Nov 2026 21:19:15 +0100 Subject: [PATCH 5/6] State the release trust boundary as it is: write access to the repository --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 069747c..3f12864 100644 --- a/README.md +++ b/README.md @@ -185,7 +185,7 @@ an older one, a checksum that does not match, a missing file, a program of anoth room leaves the running program as it was and says why. GitHub answers 60 reads an hour to an address without a token; a used-up limit is waited out, at most 15 minutes a read. Releases are not signed: the checksum guards against a broken download, HTTPS against anyone between the host and GitHub, and the publisher's name against a release someone -made by hand; anyone who can change this repository's workflows on main can still publish. "Runner release" puts out +made by hand; anyone with write access to this repository can still publish or replace one. "Runner release" puts out a commit only once its push's CI, Image when it made one, and "Runner build" are green on main (`scripts/release-gate.sh`), the rule the console rolls out by, plus the build; a commit whose build failed has no release, and its `update-to` is refused. A Runner installed from a release before this could not update itself: one From 361e2655e00febb87c08376ac6cd40d9ab9da97a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krystian=20Paw=C5=82ow?= Date: Thu, 12 Nov 2026 21:29:22 +0100 Subject: [PATCH 6/6] Fail an install --version when GitHub cannot say which Runner is newer Kept with status 0, a Runner older than the release asked for stayed while the run ended 'host-setup: unchanged'. A job on the host can use up the address's rate limit of reads without a token, and a commit GitHub does not have answered the same way every run. --- README.md | 4 +- docs/MAP.md | 2 +- scripts/host-setup.sh | 32 ++++++++++----- xtask/tests/host_setup.rs | 84 ++++++++++++++++++++++++++++++++++++++- 4 files changed, 106 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 3f12864..6f491f0 100644 --- a/README.md +++ b/README.md @@ -245,8 +245,8 @@ sudo scripts/host-setup.sh --machine --console --code-file The script's head lists its options (`--home`, `--image`, `--ref`, `--version`, `--check`). It reads GitHub's API for the image with `GITHUB_TOKEN` when that is set, and waits out GitHub's rate limit as the answer says. A `--version` is -a minimum: an enrolled host whose Runner updated itself to a newer commit of main keeps it, and so does one when -GitHub cannot say which is newer; the run says so, and a later run asks again. By hand, once, as root (Ubuntu 24.04 +a minimum: an enrolled host whose Runner updated itself to a newer commit of main keeps it, and the run says so; +when GitHub cannot say which is newer, the run fails and changes nothing. By hand, once, as root (Ubuntu 24.04 also needs the line about user namespaces): ```sh diff --git a/docs/MAP.md b/docs/MAP.md index 92d8ba6..9d4f6e9 100644 --- a/docs/MAP.md +++ b/docs/MAP.md @@ -28,7 +28,7 @@ | `crates/harness` (`cf-harness`) | What every crate's tests start from (dev-dependency only): `cf-fake-github` behind `cf-fake-console`, with the real clients in front of them as a machine has them (`enrolled`, `app`, `app_on`: `ConsoleTokens` and `ScaleSetClient`), `TestClock`, `fast` retries, `until`, `elsewhere` (a host no token may reach), and what a test asserts no machine did: `registrations` (requests for a registration token or traded one) and `admin_requests_from_machines` (service calls not made by the console, `CONSOLE_MARK`); `key_tokens` and `key_app` for a host that still holds the test key. | | `crates/console-client` (`cf-console-client`) | The client of the CodeFusion Console's runner protocol 1 (`/agent/runners/{enroll,report,leave}`): the wire types (`Enroll` and `Report` with the system they run on (`Os`, `macos` or `linux`, left out when none), `Status` with `commands` (the console's commands this Runner does: `pause` and `resume` (the answer's `paused`) on every system, `update` and `update-to` only with somewhere to update from, `test-run` with a console), `memoryForJobsBytes` (the cap of the limits in force), `idleWithJobsWaitingSeconds` and its reason, `heldJobs` (`HeldJobs`: per repository and label set, the jobs GitHub holds back from a scale set with room and the oldest one's wait, left out when none), `appKeyFile` (a GitHub App key file the host still holds, which `app_key::take_out` could not delete or found linked, or the one it signs with, left out when none), `appKeyKept` (why it still signs with it), `docker` (`DockerStatus`: the last Docker check, `ok`, `checkedAt`, `why`, left out before the first) and `sizes`, what one more machine of each size meets: `free` (-1 for a size the host never runs), `why`, `needBytes`, `shortBytes`, `JobLimits`, the report's `limits` that the `settings` command changes, `RunnerStatus` with its machine's `size` and `countedBytes` (the size the budget's cap counts it with; the vm-hosts' overhead is taken out of the cap), `JobEvent`, `Answer` (with `makeRoomFor`, the size the console names a pool host to make room for), `Command` ...), `enroll`, `Client` (`report`, `deliver`, `leave`; the token is a `Secret`), `Error` (`Unauthorized`, `Refused`, `GithubOff`, `Busy`, `Unavailable`, `Protocol`), GitHub through the console (`github.rs`: `github_read`, `github_jit`, each answer checked: tokens of visible ASCII, lifetimes of at least `MIN_TOKEN_LIFE` and held at most `MAX_TOKEN_LIFE`, unique installation IDs; what the console would refuse is refused unsent; a JIT runner is asked for once and never again by itself; `github_read` names at most `REFUSED_MAX` refused tokens; a 403 with the code `BROKER_OFF` is `Error::GithubOff`, a 429 or 5xx with Retry-After `Error::Busy`, held to `RETRY_AFTER_MAX`), `ConsoleTokens` (`tokens.rs`, the `TokenSource` of the App client: one read for all installations, renewed at half its life and not used `END_MARGIN` before its end on the monotonic clock, concurrent callers sharing one request; a console that fails is asked again after `FAILED_WAIT` doubling to `FAILED_WAIT_MAX` while the last token serves to its end, then every call fails at once with the reason, never asking sooner than the console's Retry-After; a token GitHub refuses is named in the next read so the console replaces it while the other installations' tokens serve, and one handed out again all the same, or new ones GitHub refuses too, are asked for again after a doubling wait; a runner the console answered with a wait is not asked for that repository until it passes (a 429 unsent); only the 403 `broker_off` (the owner's switch for the machine off) fails every call with the console's words for `GATE_RECHECK_ALONE`, and any other 403 is an error with the console's words; installation IDs are mapped by repository, and looked up again after a 404), the scale set calls (`scaleset.rs`: `scaleset_list`, `scaleset_create`, `scaleset_delete`, `scaleset_session_open`, `scaleset_session_refresh`, `scaleset_session_close`, `scaleset_jit`, `scaleset_remove`, each answer checked, the console's refusals by their codes: `not_yours`, `session_exists`, `session_lost`, `name_taken`, `runner_busy`, `rate_limited`; a 503 `github_busy` with its Retry-After is `Error::Busy`) and `Outbox`, the job events that wait for the console in one 0600 file: ids dedup (waiting, and the last 1000 sent), at most 500 kept with the oldest dropped and counted, 100 a report, marked sent only when a valid answer came. A report the console would refuse is refused here without sending. Tests: `tests/contract.rs` (every bound of the protocol against the fake), `client.rs`, `outbox.rs`, `delivery.rs`, `properties.rs`, `tokens.rs` (GitHub through the console end to end, with both fakes). | | `crates/fake-console` (`cf-fake-console`) | A fake CodeFusion Console for tests (dev-dependency only), over `cf_fake_github::httpd`: the checks of the console's `agent.ts` and `protocol.ts` in the same order with the same statuses and bounds (`schema.rs` is protocol.ts field by field), codes, tokens, commands, pause. `withdraw_command` (the console stops sending one). GitHub through the console (`github.rs`, the console's `github.ts`): `serve_github` attaches a `FakeGithub` that makes the tokens, `set_broker` is the owner's switch per machine (off at first; off forgets the kept tokens and revokes the read ones), `scaleset_answer` answers every scale set call with one reply (`no_scaleset_calls`: a console from before them), `set_installations` gives the console's App other installation IDs; kept tokens handed out again while more than `REUSE_MIN_S` is left, a read token named `refused` replaced and revoked (`ignore_refused`: a console that does not read the names), the switch's 403 with the code `broker_off`, `JIT_PER_REPO` and `JIT_PER_MACHINE` runners in `JIT_WINDOW_S` then 429 with Retry-After, a runner name not the machine's 403 without a code, before the Runners App is looked for. The scale set routes (`scaleset.rs`, `github/scaleset/{list,create,delete,session/open,session/refresh,session/close,jit,remove}`): the admin token made in-process and kept while more than `ADMIN_REUSE_S` is left (`forget_admin_tokens`), each call scoped to the machine's own scale sets and Actions runners (403 `not_yours`), rate limits per machine in `RATE_WINDOW_S`, GitHub's 5xx, 429, a drop or a timeout (`GITHUB_DEADLINE`) a 503 `github_busy` with Retry-After (30 s, or `github_busy_wait` for tests), and an audit row per call (`scaleset_audit`). Faults: `down`, `reports_down` (everything but `github/…`), `fail_next`, `fail_next_waiting` (with Retry-After), `drop_before_next`, `accept_then_drop_next`, `accept_then_drop_next_report`, `answer_next` (any answer to the next `github/…` call). What it saw: `reports`, `job_events`, `enrollments`, `tokens`, `calls`, `github_calls`, and `os_of`, the system the console keeps for a machine (`macos`, `linux`, none for anything else, which is never refused). Written from the console's code and not from the client's types. | -| `scripts` | `host-setup.sh`: a Linux host made a Runner of the console, unattended, as root, with no App key: `install` (packages, the Runner's user, user namespaces, the Runner built from its checkout or brought as `--binary`/`--version` with its `--sha256`, the newest image, the configuration with `--settings`; needs no code, runs any number of times, ends on `host-setup: changed` or `unchanged`, `--dry-run` says each change and a diff of the configuration; a new Runner on an enrolled host goes in under the hold `host-setup` once running jobs end, exit 75 when they outlast the wait), `enroll --code-file` (the enrolment and the service), or both at once as a cloud VM's startup script does; `merged_config` keeps the keys the Runner and the console set and takes out the ones a last `--settings` set and this one leaves out; `--class trusted` writes `"class": "trusted"` and no rules, so the Runner's defaults apply; `--sandbox container` needs no KVM, sets up the Runner user's rootless Podman (`place_engine`: subordinate ids, `podman.socket`), installs the newest container-image release into it and writes `sandbox` and `engine`; a `--version` is a minimum: `runs_newer` keeps an enrolled host's Runner that runs a newer commit of main (GitHub's compare), and keeps it when GitHub cannot say; its reads of GitHub's API (`github_get`) send `GITHUB_TOKEN` when set and wait out a rate limit, a 5xx or no answer (`xtask/tests/host_setup.rs`, with the merge and the refusal of each option a step does not take); CI's `fresh-host` job runs it against the fakes, both at once from the source and `install`, again, `--dry-run` and `enroll` with a build brought along. `release-gate.sh`: whether "Runner release" may put out a commit: on main, its push's newest CI, Image (when there is one) and Runner build runs green, read in one list of the push's runs; it uses `github_get`, `main_head` and `compare_status` of `host-setup.sh`; `publish `, `wait`, `skip`, or a failure when GitHub cannot say (tests in `xtask/tests/host_setup.rs`). `docker-rollout-mac.sh`: on one Mac, installs an image release and names it and the Docker repositories in the Runner's configuration file, then waits for the Runner to start again on it (or says why it did not). | +| `scripts` | `host-setup.sh`: a Linux host made a Runner of the console, unattended, as root, with no App key: `install` (packages, the Runner's user, user namespaces, the Runner built from its checkout or brought as `--binary`/`--version` with its `--sha256`, the newest image, the configuration with `--settings`; needs no code, runs any number of times, ends on `host-setup: changed` or `unchanged`, `--dry-run` says each change and a diff of the configuration; a new Runner on an enrolled host goes in under the hold `host-setup` once running jobs end, exit 75 when they outlast the wait), `enroll --code-file` (the enrolment and the service), or both at once as a cloud VM's startup script does; `merged_config` keeps the keys the Runner and the console set and takes out the ones a last `--settings` set and this one leaves out; `--class trusted` writes `"class": "trusted"` and no rules, so the Runner's defaults apply; `--sandbox container` needs no KVM, sets up the Runner user's rootless Podman (`place_engine`: subordinate ids, `podman.socket`), installs the newest container-image release into it and writes `sandbox` and `engine`; a `--version` is a minimum: `runs_newer` and `release_stays` keep an enrolled host's Runner that runs a newer commit of main (GitHub's compare), and fail the run, changing nothing, when GitHub cannot say; its reads of GitHub's API (`github_get`) send `GITHUB_TOKEN` when set and wait out a rate limit, a 5xx or no answer (`xtask/tests/host_setup.rs`, with the merge and the refusal of each option a step does not take); CI's `fresh-host` job runs it against the fakes, both at once from the source and `install`, again, `--dry-run` and `enroll` with a build brought along. `release-gate.sh`: whether "Runner release" may put out a commit: on main, its push's newest CI, Image (when there is one) and Runner build runs green, read in one list of the push's runs; it uses `github_get`, `main_head` and `compare_status` of `host-setup.sh`; `publish `, `wait`, `skip`, or a failure when GitHub cannot say (tests in `xtask/tests/host_setup.rs`). `docker-rollout-mac.sh`: on one Mac, installs an image release and names it and the Docker repositories in the Runner's configuration file, then waits for the Runner to start again on it (or says why it did not). | | `xtask` | `cargo xtask keygen | guest | release | build | it | fakes serve | kernel check | cloud-hypervisor [check]`. `fakes serve` (`xtask/src/fakes.rs`) runs the fake GitHub and console for `scripts/host-setup.sh` and ends once the host it set up serves. `it` (`xtask/src/it.rs`) runs the vm-host against real machines on this host, one small machine at a time, and writes its numbers to `target/it/report.txt`. Its scenarios: `boot`, `lifecycle`, `exit-code`, `oom`, `timeout`, `stop`, `sigterm`, `sigkill`, `firewall` (what a job reaches), `inside` (what a job's root tries against its own machine), `docker` (a `cf-docker` job: Compose, a build, a published port, and what its containers reach, privileged ones too, with a port open on the host as the target that shows the firewall at work), `docker-oom` (a container that takes all the job's memory ends the job `oom`; one stopped at its own limit, in `docker`, does not), `launchd`, `timers`, `full-memory` (what a machine whose job holds all its memory costs the host, against `Settings::VM_HOST_OVERHEAD`, and that its vm-host reads what its machine holds), and in `it/service.rs` `service` (three jobs through the whole service against the fake GitHub and console: to its end, across a Runner that dies, into an uninstall) and `commands` (the binary's `enroll`, `check --github` and `daemon`). A scenario that needs more memory than the host has free says `SKIPPED` and why. Kernel package pinned by version, size and SHA-256 in `xtask/src/kernel.rs`, read from our copy (the release `kernel-linux-virt-`) and from Alpine only when that is missing; `kernel check` goes red when our copy does not give the pinned bytes and warns when Alpine has a newer kernel. How to move the pin is at the top of `kernel.rs`. `cloud_hypervisor.rs` does the same for the Linux monitor pinned in `crates/vmm-ch/src/pins.rs` (our copy: the release `cloud-hypervisor-`), and `pinned.rs` is what both share: the HTTPS download that checks each redirect's host and the bytes, the check of our copy, and `newest`, the version a release or index names. | | `image/Dockerfile` | The root filesystem a job runs in: GitHub's Actions runner image plus tools, for arm64 and amd64 (`TARGETARCH`, each download pinned for each). For Docker jobs: iptables, pasta and the Compose plugin (the engine, CLI and buildx come with the Actions runner image). Packed by `.github/workflows/image.yml`. | | `.github/workflows` | `ci.yml` (macos-15: fmt, guest, clippy, tests, release build; ubuntu-24.04-arm and ubuntu-24.04: clippy and tests of the crates in `LINUX_CRATES`, clippy for the guest's musl target of that architecture, and the Linux probe; ubuntu: cargo deny), `image.yml` (the image of each architecture, and its container-image release beside it: built and verified on every change to it, signed and published from main, weekly), `pins.yml` (`cargo xtask kernel check` and `cargo xtask cloud-hypervisor check`, weekly and on every change to a pin), `runner-build.yml` (the Runner's Linux program of each architecture, built in Debian 12 from nothing on every push to main, kept as artifacts for a week), `release.yml` (on each of CI, Image and Runner build ending for a push to main, `scripts/release-gate.sh`; once all are green, the build's programs released as `runner-` with the `host-setup.sh` of its commit and their SHA256SUMS, never replaced; by hand with `workflow_dispatch` and a commit). Labelled pull requests are merged by CodeFusion Console. | diff --git a/scripts/host-setup.sh b/scripts/host-setup.sh index 13e9cd4..dd8db81 100755 --- a/scripts/host-setup.sh +++ b/scripts/host-setup.sh @@ -24,7 +24,7 @@ # running ones (at most an hour) and installs it; when they outlast the wait it exits 75, the hold lifted, and the # Runner of before goes on. A Runner from a release updates itself to newer releases from the console (`update-to`), so # a --version is a minimum: an enrolled host whose Runner runs a newer commit of main, by GitHub's compare of main's -# history, keeps it, and so does one when GitHub cannot say. +# history, keeps it; when GitHub cannot say, the run fails and changes nothing. # # --class shared (the default) or trusted: a host that runs only main's deploy, release and signing jobs of # codefusion-cc/codefusion, by the Runner's default rules (crates/trusted), one job at a time @@ -235,6 +235,25 @@ runs_newer() { esac } +# Whether a run that asks for the release of the commit $3 (named $4) leaves the Runner that runs in place, which says +# it is $2 (its version, `0.1.0+<12 characters>`), by `runs_newer` against the repository's API at $1: status 0, and +# says so, when it runs a newer commit of main; 1 when the release goes in. When GitHub cannot say, the run fails and +# changes nothing: the Runner that runs may be older than the release, and a run that ended `unchanged` would say +# the release is in. +release_stays() { + local api=$1 runs=$2 wanted=$3 named=$4 + case $(runs_newer "$api" "${runs##*+}" "$wanted") in + keep) + say "it runs $runs, newer on main than $named: it stays, and $named waits for none" + return 0 + ;; + unknown) + fail "it runs $runs, and GitHub could not say whether that is newer on main than $named; nothing changed: run it again once GitHub answers, or give the Runner with --binary" + ;; + esac + return 1 +} + # Sourced (the tests), it only defines the above. if (return 0 2>/dev/null); then return 0; fi case ${1-} in @@ -618,16 +637,7 @@ place_binary() { as_user "'$installed' version" 2>/dev/null ) || runs='' runs=${runs#codefusion-runner } - case $(runs_newer "$REPO_API" "${runs##*+}" "${version##*-}") in - keep) - say "it runs $runs, newer on main than $named: it stays, and $named waits for none" - return - ;; - unknown) - say "it runs $runs, and GitHub could not say whether that is newer on main than $named: it stays; a run later asks again" - return - ;; - esac + if release_stays "$REPO_API" "$runs" "${version##*-}" "$named"; then return; fi fi if [[ $(user_sha256 "$staged") != "$sha256" ]]; then change "place the Runner of $named" diff --git a/xtask/tests/host_setup.rs b/xtask/tests/host_setup.rs index 9aaeee0..4206e67 100644 --- a/xtask/tests/host_setup.rs +++ b/xtask/tests/host_setup.rs @@ -341,9 +341,10 @@ fn a_429_or_a_5xx_is_asked_again_before_it_decides() { ); } -/// When GitHub cannot say, at any of the three reads, the Runner that runs stays. +/// When GitHub cannot say, at any of the three reads, the answer is `unknown`, which ends the run +/// (`what_github_cannot_say_fails_the_run_and_changes_nothing`). #[test] -fn what_github_cannot_say_keeps_the_runner_that_runs() { +fn what_github_cannot_say_is_unknown() { let not_found = Reply::status_json(404, &serde_json::json!({ "message": "Not Found" })); for answers in [ vec![Reply::empty(503)], @@ -369,6 +370,85 @@ fn what_github_cannot_say_keeps_the_runner_that_runs() { } } +/// `release_stays` of the script, sourced, against the repository's API at `api`: the Runner that runs says it is +/// `runs`, the run asks for the release of `ASKED`. It prints `stays` or `installs` when it returns. +fn release_stays(api: &str, runs: &str, env: &[(&str, &str)]) -> Output { + let script = concat!(env!("CARGO_MANIFEST_DIR"), "/../scripts/host-setup.sh"); + let mut command = Command::new("bash"); + command + .args([ + "-c", + "source \"$1\" && if release_stays \"$2\" \"$3\" \"$4\" \"release runner-0.1.0-$4\"; then echo stays; \ + else echo installs; fi", + "bash", + script, + api, + runs, + ASKED, + ]) + .env_remove("GITHUB_TOKEN"); + for (name, value) in env { + command.env(name, value); + } + command.output().expect("bash runs") +} + +#[test] +fn a_runner_newer_on_main_stays_and_an_older_one_takes_the_release() { + let newer = api(vec![compared("ahead"), main_at(), compared("ahead")]); + let out = release_stays(&newer.server.url(), &format!("0.1.0+{RUNS}"), &[]); + assert!(out.status.success(), "{out:?}"); + assert_eq!( + String::from_utf8_lossy(&out.stdout), + format!( + "host-setup: arguments: it runs 0.1.0+{RUNS}, newer on main than release runner-0.1.0-{ASKED}: it \ + stays, and release runner-0.1.0-{ASKED} waits for none\nstays\n" + ) + ); + + let older = api(vec![compared("behind")]); + let out = release_stays(&older.server.url(), &format!("0.1.0+{RUNS}"), &[]); + assert!(out.status.success(), "{out:?}"); + assert_eq!(String::from_utf8_lossy(&out.stdout), "installs\n"); +} + +/// When GitHub cannot say, the run ends with a failure and changes nothing. Kept with a status of 0, the Runner that +/// runs could be older than the release asked for (a job on the host can use up the address's rate limit of reads +/// without a token), and the run would end `host-setup: unchanged`, as if the release were in. +#[test] +fn what_github_cannot_say_fails_the_run_and_changes_nothing() { + let not_found = Reply::status_json(404, &serde_json::json!({ "message": "Not Found" })); + for answers in [ + vec![Reply::empty(503)], + vec![rate_limited(403)], + // A commit GitHub does not have, such as a --binary built from a branch never pushed. + vec![not_found.clone()], + vec![compared("ahead"), main_at(), not_found], + ] { + let api = api(answers.clone()); + let out = release_stays( + &api.server.url(), + &format!("0.1.0+{RUNS}"), + &[("GITHUB_TRIES", "2"), ("GITHUB_WAIT_MAX", "1")], + ); + assert_eq!(out.status.code(), Some(1), "{answers:?}: {out:?}"); + assert_eq!(String::from_utf8_lossy(&out.stdout), "", "{answers:?}"); + let stderr = String::from_utf8_lossy(&out.stderr); + assert_eq!( + stderr.lines().last(), + Some( + format!( + "host-setup: arguments: it runs 0.1.0+{RUNS}, and GitHub could not say whether that is newer on \ + main than release runner-0.1.0-{ASKED}; nothing changed: run it again once GitHub answers, or \ + give the Runner with --binary" + ) + .as_str() + ), + "{answers:?}" + ); + } +} + /// A ref that names no whole commit, such as a tag's or a short name, is not compared with: main is not known. #[test] fn a_main_that_names_no_whole_commit_is_not_compared_with() {