From e8216acb642084d677c18efb2d85fd58ed91d3da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krystian=20Paw=C5=82ow?= Date: Thu, 12 Nov 2026 22:40:06 +0100 Subject: [PATCH 1/3] Test that a container's layer is bounded and held to the disk its request names --- crates/vmhost/tests/driver.rs | 51 ++++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/crates/vmhost/tests/driver.rs b/crates/vmhost/tests/driver.rs index e699a62..933b187 100644 --- a/crates/vmhost/tests/driver.rs +++ b/crates/vmhost/tests/driver.rs @@ -1347,6 +1347,16 @@ fn in_a_container_the_job_starts_at_once_with_its_whole_size_and_no_disk() { /// Runs a container job while `daemon` plays the other side: it gets the monitor once the job runs. fn container_job(vmm: &Arc, layout: &Layout, daemon: impl FnOnce(&FakeVmm) + Send) -> JobState { + container_job_of(vmm, layout, request("a"), daemon) +} + +/// [`container_job`] of this request, whose job is `a`. +fn container_job_of( + vmm: &Arc, + layout: &Layout, + request: JobRequest, + daemon: impl FnOnce(&FakeVmm) + Send, +) -> JobState { let mut limits = quick(); limits.heartbeat = Duration::from_secs(30); std::thread::scope(|scope| { @@ -1359,7 +1369,7 @@ fn container_job(vmm: &Arc, layout: &Layout, daemon: impl FnOnce(&FakeV } daemon(vmm); }); - go_within(vmm, layout, request("a"), limits, false).unwrap() + go_within(vmm, layout, request, limits, false).unwrap() }) } @@ -1386,6 +1396,45 @@ fn a_container_that_writes_past_its_disk_size_is_stopped() { assert!(vmm.stops() >= 1); } +/// A job's disk of another size, as a host's settings give it (`job_disk_gb`): the engine is asked to bound the +/// layer to it, and the layer is held to it, to the byte. +#[test] +fn a_containers_layer_is_bounded_and_held_to_the_disk_its_request_names() { + for (written, ends) in [(6 << 30, End::Ok), ((6 << 30) + 1, End::Failed)] { + let dir = tempfile::tempdir().unwrap(); + let layout = Layout::new(dir.path()); + let (go_on, wait) = mpsc::channel::<()>(); + let vmm = Arc::new( + FakeVmm::new([job(move |mut stream, power, _| { + say(&mut stream, GuestMessage::Heartbeat(usage())); + if ends == End::Ok { + let _ = wait.recv(); + finish(stream, summary(GuestEnd::Exited, Some(0))); + } else { + // Only the vm-host's stop ends it. + power.wait_for_the_end(); + } + })]) + .of_containers(), + ); + let six = JobRequest { + scratch_bytes: 6 << 30, + ..request("a") + }; + let state = container_job_of(&vmm, &layout, six, move |vmm| { + vmm.layer(Some(written)); + std::thread::sleep(Duration::from_millis(200)); + let _ = go_on.send(()); + }); + assert_eq!(vmm.started()[0].layer_bytes, Some(6 << 30)); + let ended = state.end.unwrap(); + assert_eq!(ended.end, ends, "{written}"); + if ends == End::Failed { + assert!(ended.message.unwrap().contains("6144 MiB")); + } + } +} + /// A layer of exactly the scratch size is fine, and the most it held is the job's disk peak, whatever the guest, /// which sees no disk of its own, says. #[test] From 6c8ac4f8db92a4184b2005ddf27d4e09b2bea7d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krystian=20Paw=C5=82ow?= Date: Thu, 12 Nov 2026 22:40:06 +0100 Subject: [PATCH 2/3] Make the disk hold, the disk stop and each job's disk host settings disk_hold_gb, disk_stop_gb and job_disk_gb in the settings file become the budget's Limits, as ci_memory_gb and ci_cpus do. The defaults stay 25, 5 and 40 GB. A hold or a job's disk the state folder's file system cannot hold is refused; a container host whose hold and one job's disk do not fit it together starts and says so. --- crates/budget/src/lib.rs | 2 +- crates/budget/src/settings.rs | 170 +++++++++++++++- crates/budget/tests/admission.rs | 5 +- crates/budget/tests/disk.rs | 299 ++++++++++++++++++++++++++++ crates/budget/tests/properties.rs | 18 +- crates/daemon/src/lib.rs | 5 +- crates/daemon/src/machines.rs | 7 +- crates/daemon/src/platform/linux.rs | 4 + crates/daemon/src/platform/mac.rs | 4 + crates/daemon/src/platform/mod.rs | 47 ++++- crates/daemon/tests/machines.rs | 38 ++++ crates/runner/src/config.rs | 117 +++++++++-- crates/runner/src/service.rs | 118 +++++++++-- 13 files changed, 780 insertions(+), 54 deletions(-) create mode 100644 crates/budget/tests/disk.rs diff --git a/crates/budget/src/lib.rs b/crates/budget/src/lib.rs index dd5f126..c6a2675 100644 --- a/crates/budget/src/lib.rs +++ b/crates/budget/src/lib.rs @@ -43,7 +43,7 @@ pub use budget::{ pub use fake::FakeHost; pub use host::{Battery, Host, HostError, HostFacts, HostReading, Millis, Now, Pressure, Thermal}; pub use level::{Hold, Level, Trouble}; -pub use settings::{DEFER_LIMIT, Limits, Mode, Profile, Sandbox, Settings, SettingsError}; +pub use settings::{DEFER_LIMIT, DiskShort, Limits, Mode, Profile, Sandbox, Settings, SettingsError}; pub use size::Size; /// One mebibyte. The plan's "MB" and "GB" are these: a 16 GB Mac reports 16 × 2^30 bytes. diff --git a/crates/budget/src/settings.rs b/crates/budget/src/settings.rs index 29f6608..b6603c3 100644 --- a/crates/budget/src/settings.rs +++ b/crates/budget/src/settings.rs @@ -91,6 +91,8 @@ pub struct Settings { /// Under this much free disk the job with the largest scratch disk is stopped. What the system would free does /// not count: a machine's writes need the space now. pub disk_evict_bytes: u64, + /// The size of each job's disk: a machine's scratch disk, or the most a container may write into its layer. + pub job_disk_bytes: u64, /// How long a reserved slot waits for its job before it is given back. pub reservation_ttl: Millis, /// A reading older than this, or two readings further apart, close the Mac. @@ -113,6 +115,12 @@ pub struct Limits { /// The CPUs CI's machines may take together, from 1 to every core: for a host whose other work leaves CI more /// (or fewer) than the defaults give, such as a game server that takes CI jobs only while it is idle. pub ci_cpus: Option, + /// The free disk under which no job starts: [`Settings::disk_hold_bytes`]. + pub disk_hold_bytes: Option, + /// The free disk under which a job is stopped: [`Settings::disk_evict_bytes`]. + pub disk_evict_bytes: Option, + /// The size of each job's disk: [`Settings::job_disk_bytes`]. + pub job_disk_bytes: Option, } /// A setting the budget refuses, with what is wrong. Settings come from a file, the tray and the console, so a bad @@ -175,8 +183,60 @@ pub enum SettingsError { #[error("memory for CI of {ci_bytes} bytes holds no machine: the smallest needs {least_bytes}")] CiUnderSmallest { ci_bytes: u64, least_bytes: u64 }, /// Jobs would be stopped for a full disk before slots stop opening for it. - #[error("the disk limit that stops a job is above the one that stops new slots")] - DiskLimitsCrossed, + #[error( + "the free disk under which a job is stopped, {evict_bytes} bytes, is above the one under which no job \ + starts, {hold_bytes}" + )] + DiskLimitsCrossed { evict_bytes: u64, hold_bytes: u64 }, + /// A free disk under which no job starts that is under the least a job is stopped at. + #[error("the free disk under which no job starts is under {least_bytes} bytes")] + DiskHoldTooLow { least_bytes: u64 }, + /// A disk limit that leaves the Runner no room for its own records and logs before a job is stopped. + #[error("the free disk under which a job is stopped is under {least_bytes} bytes")] + DiskEvictTooLow { least_bytes: u64 }, + /// A job's disk too small for a checkout and a build, or larger than any machine's disk may be. + #[error("a job's disk of {bytes} bytes is not between {least_bytes} and {most_bytes}")] + JobDisk { + bytes: u64, + least_bytes: u64, + most_bytes: u64, + }, + /// A disk limit no file system of this size ever meets: no job would start. + #[error( + "the free disk under which no job starts, {hold_bytes} bytes, is not under the size of the file system of \ + the state folder, {disk_bytes}: no job would ever start" + )] + DiskHoldOverDisk { hold_bytes: u64, disk_bytes: u64 }, + /// A job's disk larger than the file system it is on. + #[error( + "a job's disk of {job_bytes} bytes is larger than the file system of the state folder, {disk_bytes}" + )] + JobDiskOverDisk { job_bytes: u64, disk_bytes: u64 }, +} + +/// What a container host's disk settings leave short of, though the Runner starts on them: the file system of the +/// state folder holds less than the free disk under which no job starts and one job's whole disk together. A job +/// that writes its whole layer then takes the host under that free disk, and may be stopped for it +/// ([`Settings::check_disk`]). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct DiskShort { + /// [`Settings::disk_hold_bytes`]. + pub hold_bytes: u64, + /// [`Settings::job_disk_bytes`]. + pub job_bytes: u64, + /// The size of the file system of the state folder. + pub disk_bytes: u64, +} + +impl std::fmt::Display for DiskShort { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "the free disk under which no job starts ({} bytes) and one job's disk ({}) do not fit the file system \ + of the state folder ({}): a job that writes its whole disk may be stopped for a full disk", + self.hold_bytes, self.job_bytes, self.disk_bytes + ) + } } impl Settings { @@ -213,6 +273,25 @@ impl Settings { /// No Mac keeps less than this for itself, whatever a setting asks for. pub const LEAST_FLOOR: u64 = GIB; + /// The free disk under which no job starts unless the host's settings name another + /// ([`Settings::disk_hold_bytes`]). + pub const DISK_HOLD: u64 = 25 * GIB; + + /// The free disk under which a job is stopped unless the host's settings name another + /// ([`Settings::disk_evict_bytes`]). + pub const DISK_EVICT: u64 = 5 * GIB; + + /// The size of each job's disk unless the host's settings name another ([`Settings::job_disk_bytes`]). + pub const JOB_DISK: u64 = 40 * GIB; + + /// The sizes a job's disk may have: a checkout, its dependencies and a build need a few gigabytes, and the + /// vm-host refuses a disk over 512 GiB. + pub const JOB_DISK_RANGE: std::ops::RangeInclusive = 4 * GIB..=512 * GIB; + + /// The least free disk under which a job is stopped ([`Settings::disk_evict_bytes`]): the Runner's records, + /// its outbox and its log need room while a job is stopped. + pub const LEAST_DISK_EVICT: u64 = GIB; + /// The defaults for a Mac of this size. /// /// - Cap: all memory but a reserve of 25% (at least 3 GB) on a dedicated Mac, 40% (at least 4 GB) on a @@ -274,8 +353,9 @@ impl Settings { swap_growth_window: Millis::secs(60), evict_min_held_bytes: GIB, evict_every: Millis::secs(30), - disk_hold_bytes: 25 * GIB, - disk_evict_bytes: 5 * GIB, + disk_hold_bytes: Self::DISK_HOLD, + disk_evict_bytes: Self::DISK_EVICT, + job_disk_bytes: Self::JOB_DISK, reservation_ttl: Millis::mins(2), stale_after: Millis::secs(5), } @@ -439,8 +519,23 @@ impl Settings { /// floor stays the higher of the two. A cap that would leave the host less than its floor is lowered to what is left. /// - The memory for CI ([`Settings::ci_bytes`]), when set: the cap becomes what it leaves after one machine's /// overhead, and a floor not set as a share becomes what it leaves the host, at least [`Settings::LEAST_FLOOR`]. + /// - The free disk under which no job starts and the one under which a job is stopped, when set. A hold set + /// without the other lowers the default stop to it, never raises it: a hold of 3 GiB stops jobs under 3 GiB. + /// - The size of each job's disk, when set. pub fn limited(mut self, limits: &Limits) -> Result { self.max_machines = limits.max_machines.or(self.max_machines); + if let Some(hold) = limits.disk_hold_bytes { + self.disk_hold_bytes = hold; + if limits.disk_evict_bytes.is_none() { + self.disk_evict_bytes = self.disk_evict_bytes.min(hold); + } + } + if let Some(evict) = limits.disk_evict_bytes { + self.disk_evict_bytes = evict; + } + if let Some(job) = limits.job_disk_bytes { + self.job_disk_bytes = job; + } if let Some(idle_after) = limits.idle_after { self.idle_after = idle_after; } @@ -603,12 +698,77 @@ impl Settings { }); } } + // Said of the hold first: one set alone takes the stop down with it, and the owner set the hold. + if self.disk_hold_bytes < Self::LEAST_DISK_EVICT { + return Err(SettingsError::DiskHoldTooLow { + least_bytes: Self::LEAST_DISK_EVICT, + }); + } + if self.disk_evict_bytes < Self::LEAST_DISK_EVICT { + return Err(SettingsError::DiskEvictTooLow { + least_bytes: Self::LEAST_DISK_EVICT, + }); + } if self.disk_evict_bytes > self.disk_hold_bytes { - return Err(SettingsError::DiskLimitsCrossed); + return Err(SettingsError::DiskLimitsCrossed { + evict_bytes: self.disk_evict_bytes, + hold_bytes: self.disk_hold_bytes, + }); + } + if !Self::JOB_DISK_RANGE.contains(&self.job_disk_bytes) { + return Err(SettingsError::JobDisk { + bytes: self.job_disk_bytes, + least_bytes: *Self::JOB_DISK_RANGE.start(), + most_bytes: *Self::JOB_DISK_RANGE.end(), + }); } Ok(()) } + /// These settings against the file system of the state folder, of `disk_bytes` (0 when the system will not + /// say, and then nothing is checked). Only what the owner set (`limits`) is refused, so a host on the defaults + /// starts as it always did: + /// + /// - a free disk under which no job starts that is not under the file system's size, since no file system is + /// ever that free and no job would start; + /// - a job's disk larger than the file system. + /// + /// In containers, where every job writes onto that file system, a hold and one job's disk that do not fit it + /// together are [`DiskShort`]: the Runner starts and says so. Not refused, since a job's disk is the most it may + /// write, not what it takes: most jobs write a fraction of it, and the free disk under which a job is stopped + /// still keeps the host from filling up. + pub fn check_disk( + &self, + limits: &Limits, + sandbox: Sandbox, + disk_bytes: u64, + ) -> Result, SettingsError> { + if disk_bytes == 0 { + return Ok(None); + } + if limits.disk_hold_bytes.is_some() && self.disk_hold_bytes >= disk_bytes { + return Err(SettingsError::DiskHoldOverDisk { + hold_bytes: self.disk_hold_bytes, + disk_bytes, + }); + } + if limits.job_disk_bytes.is_some() && self.job_disk_bytes > disk_bytes { + return Err(SettingsError::JobDiskOverDisk { + job_bytes: self.job_disk_bytes, + disk_bytes, + }); + } + let fits = self + .disk_hold_bytes + .checked_add(self.job_disk_bytes) + .is_some_and(|both| both <= disk_bytes); + Ok((sandbox == Sandbox::Container && !fits).then_some(DiskShort { + hold_bytes: self.disk_hold_bytes, + job_bytes: self.job_disk_bytes, + disk_bytes, + })) + } + /// Whether this Mac could run a VM of this size at all: alone, with all its memory available and its person /// away. Sizes that fail are not worth offering to GitHub from this Mac. pub fn can_run(&self, size: Size) -> Result<(), Refusal> { diff --git a/crates/budget/tests/admission.rs b/crates/budget/tests/admission.rs index 4c8ffce..90add6c 100644 --- a/crates/budget/tests/admission.rs +++ b/crates/budget/tests/admission.rs @@ -195,7 +195,10 @@ fn every_bad_setting_is_refused_by_name() { ); assert_eq!( bad(|s| s.disk_evict_bytes = s.disk_hold_bytes + 1), - SettingsError::DiskLimitsCrossed + SettingsError::DiskLimitsCrossed { + evict_bytes: 25 * GIB + 1, + hold_bytes: 25 * GIB + } ); let durations: [fn(&mut Settings); 8] = [ |s| s.idle_after = Millis::ZERO, diff --git a/crates/budget/tests/disk.rs b/crates/budget/tests/disk.rs new file mode 100644 index 0000000..a244130 --- /dev/null +++ b/crates/budget/tests/disk.rs @@ -0,0 +1,299 @@ +//! The disk limits a host's settings may give: the free disk under which no job starts (the hold), the one under +//! which a job is stopped, and the size of each job's disk. A small Linux host that runs containers lowers them; every +//! other host keeps the defaults. + +mod common; + +use cf_budget::{ + DiskShort, Eviction, EvictionCause, GIB, HostFacts, Limits, Refusal, Sandbox, Settings, SettingsError, + Size, Trouble, Usage, +}; +use common::*; + +/// heimforge's dal1: 11 GiB, 12 cores alike, no KVM, jobs in containers. +const DAL1: HostFacts = HostFacts { + total_bytes: 11 * GIB, + cores: 12, +}; + +fn dal1() -> Settings { + Settings::for_headless_in(DAL1, 0, Sandbox::Container) +} + +fn disk(hold: Option, evict: Option, job: Option) -> Limits { + Limits { + disk_hold_bytes: hold, + disk_evict_bytes: evict, + job_disk_bytes: job, + ..Limits::default() + } +} + +#[test] +fn every_kind_of_host_keeps_the_disk_defaults() { + let hosts = [ + dedicated(), + workstation(), + Settings::for_headless(DAL1, 0), + dal1(), + Settings::for_headless(DAL1, 0).one_at_a_time(0), + ]; + for settings in hosts { + assert_eq!( + ( + settings.disk_hold_bytes, + settings.disk_evict_bytes, + settings.job_disk_bytes + ), + (25 * GIB, 5 * GIB, 40 * GIB) + ); + let limited = settings.clone().limited(&Limits::default()).unwrap(); + assert_eq!(limited, settings, "no limits change nothing"); + } + assert_eq!( + (Settings::DISK_HOLD, Settings::DISK_EVICT, Settings::JOB_DISK), + (25 * GIB, 5 * GIB, 40 * GIB) + ); +} + +#[test] +fn a_hold_alone_lowers_the_stop_to_it_and_never_raises_it() { + let low = dal1().limited(&disk(Some(3 * GIB), None, None)).unwrap(); + assert_eq!((low.disk_hold_bytes, low.disk_evict_bytes), (3 * GIB, 3 * GIB)); + let high = dal1().limited(&disk(Some(10 * GIB), None, None)).unwrap(); + assert_eq!((high.disk_hold_bytes, high.disk_evict_bytes), (10 * GIB, 5 * GIB)); + let both = dal1() + .limited(&disk(Some(3 * GIB), Some(GIB), Some(8 * GIB))) + .unwrap(); + assert_eq!( + (both.disk_hold_bytes, both.disk_evict_bytes, both.job_disk_bytes), + (3 * GIB, GIB, 8 * GIB) + ); + // A stop set alone keeps the default hold. + let stop = dal1().limited(&disk(None, Some(2 * GIB), None)).unwrap(); + assert_eq!((stop.disk_hold_bytes, stop.disk_evict_bytes), (25 * GIB, 2 * GIB)); +} + +#[test] +fn the_disk_limits_at_their_edges() { + let too_low = Err(SettingsError::DiskEvictTooLow { least_bytes: GIB }); + let hold_too_low = Err(SettingsError::DiskHoldTooLow { least_bytes: GIB }); + // A hold of nothing, or of a byte, is said of the hold, though it takes the stop down with it. + assert_eq!(dal1().limited(&disk(Some(0), None, None)), hold_too_low); + assert_eq!(dal1().limited(&disk(Some(1), None, None)), hold_too_low); + assert_eq!(dal1().limited(&disk(Some(GIB - 1), None, None)), hold_too_low); + assert_eq!( + dal1().limited(&disk(Some(GIB - 1), Some(GIB - 1), None)), + hold_too_low + ); + assert_eq!(dal1().limited(&disk(None, Some(0), None)), too_low); + assert_eq!(dal1().limited(&disk(None, Some(GIB - 1), None)), too_low); + // The least of both, equal. + let least = dal1().limited(&disk(Some(GIB), Some(GIB), None)).unwrap(); + assert_eq!((least.disk_hold_bytes, least.disk_evict_bytes), (GIB, GIB)); + // A stop above the hold, by a byte. + assert_eq!( + dal1().limited(&disk(Some(3 * GIB), Some(3 * GIB + 1), None)), + Err(SettingsError::DiskLimitsCrossed { + evict_bytes: 3 * GIB + 1, + hold_bytes: 3 * GIB + }) + ); + // A stop set alone above the default hold. + assert_eq!( + dal1().limited(&disk(None, Some(26 * GIB), None)), + Err(SettingsError::DiskLimitsCrossed { + evict_bytes: 26 * GIB, + hold_bytes: 25 * GIB + }) + ); + // The largest numbers do not overflow anything. + assert_eq!( + dal1() + .limited(&disk(Some(u64::MAX), Some(u64::MAX), None)) + .map(|s| s.disk_hold_bytes), + Ok(u64::MAX) + ); +} + +#[test] +fn a_jobs_disk_is_between_four_and_512_gib() { + let job = |bytes| dal1().limited(&disk(None, None, Some(bytes))); + let refused = |bytes| { + Err(SettingsError::JobDisk { + bytes, + least_bytes: 4 * GIB, + most_bytes: 512 * GIB, + }) + }; + for bytes in [0, 1, 4 * GIB - 1, 512 * GIB + 1, u64::MAX] { + assert_eq!(job(bytes), refused(bytes), "{bytes}"); + } + for bytes in [4 * GIB, 8 * GIB, 512 * GIB] { + assert_eq!(job(bytes).map(|s| s.job_disk_bytes), Ok(bytes), "{bytes}"); + } +} + +#[test] +fn a_hold_that_no_file_system_of_this_size_meets_is_refused() { + let limits = disk(Some(12 * GIB), Some(GIB), Some(4 * GIB)); + let settings = dal1().limited(&limits).unwrap(); + let check = |disk_bytes| settings.check_disk(&limits, Sandbox::Container, disk_bytes); + let over = |disk_bytes| { + Err(SettingsError::DiskHoldOverDisk { + hold_bytes: 12 * GIB, + disk_bytes, + }) + }; + // Larger than the disk, and exactly the disk: no file system is ever all free. + assert_eq!(check(11 * GIB), over(11 * GIB)); + assert_eq!(check(12 * GIB), over(12 * GIB)); + assert!( + over(12 * GIB) + .unwrap_err() + .to_string() + .contains("no job would ever start") + ); + // A byte more disk than the hold is not refused, only short of a job's disk. + assert_eq!( + check(12 * GIB + 1), + Ok(Some(DiskShort { + hold_bytes: 12 * GIB, + job_bytes: 4 * GIB, + disk_bytes: 12 * GIB + 1 + })) + ); + // A system that does not say its disk's size: nothing is checked. + assert_eq!(check(0), Ok(None)); +} + +#[test] +fn a_jobs_disk_larger_than_the_file_system_is_refused() { + let limits = disk(Some(2 * GIB), None, Some(16 * GIB)); + let settings = dal1().limited(&limits).unwrap(); + assert_eq!( + settings.check_disk(&limits, Sandbox::Container, 16 * GIB - 1), + Err(SettingsError::JobDiskOverDisk { + job_bytes: 16 * GIB, + disk_bytes: 16 * GIB - 1 + }) + ); + assert_eq!( + settings.check_disk(&limits, Sandbox::Vm, 16 * GIB - 1), + Err(SettingsError::JobDiskOverDisk { + job_bytes: 16 * GIB, + disk_bytes: 16 * GIB - 1 + }), + "a machine's disk is a file on it too" + ); +} + +#[test] +fn a_container_host_whose_hold_and_one_jobs_disk_do_not_fit_starts_and_says_so() { + let limits = disk(Some(4 * GIB), Some(GIB), Some(8 * GIB)); + let settings = dal1().limited(&limits).unwrap(); + // To the byte: 12 GiB holds both. + assert_eq!( + settings.check_disk(&limits, Sandbox::Container, 12 * GIB), + Ok(None) + ); + let short = settings.check_disk(&limits, Sandbox::Container, 12 * GIB - 1); + assert_eq!( + short, + Ok(Some(DiskShort { + hold_bytes: 4 * GIB, + job_bytes: 8 * GIB, + disk_bytes: 12 * GIB - 1 + })) + ); + assert!( + short + .unwrap() + .unwrap() + .to_string() + .contains("do not fit the file system"), + "it says what is short" + ); + // Machines are left as they are. + let vm = Settings::for_headless(DAL1, 0).limited(&limits).unwrap(); + assert_eq!(vm.check_disk(&limits, Sandbox::Vm, 12 * GIB - 1), Ok(None)); +} + +#[test] +fn the_defaults_are_never_refused_for_a_small_disk() { + // A host on the defaults starts on any disk as it did; a container host on them is told why it may take nothing. + for settings in [dedicated(), Settings::for_headless(DAL1, 0)] { + assert_eq!( + settings.check_disk(&Limits::default(), Sandbox::Vm, 20 * GIB), + Ok(None) + ); + } + assert_eq!( + dal1().check_disk(&Limits::default(), Sandbox::Container, 20 * GIB), + Ok(Some(DiskShort { + hold_bytes: 25 * GIB, + job_bytes: 40 * GIB, + disk_bytes: 20 * GIB + })) + ); +} + +/// A budget that has seen a minute of readings with this much free disk. +fn watched_with_free(settings: Settings, free: u64) -> cf_budget::Budget { + let mut budget = cf_budget::Budget::new(settings).unwrap(); + watch(&mut budget, 0, 60, |r| { + r.available_bytes = 10 * GIB; + r.disk_free_bytes = free; + }); + budget +} + +#[test] +fn a_small_disk_takes_a_job_once_its_hold_is_lowered_and_holds_under_it() { + // dal1's home: about 12 GiB, 4 GiB of it free. + let free = 4 * GIB; + let mut defaults = watched_with_free(dal1(), free); + assert_eq!( + defaults.claim(secs(60), Size::G2).map(|_| ()), + Err(Refusal::Trouble(Trouble::DiskLow)), + "today's 25 GiB hold takes nothing" + ); + let lowered = dal1() + .limited(&disk(Some(3 * GIB), Some(GIB), Some(6 * GIB))) + .unwrap(); + let mut budget = watched_with_free(lowered.clone(), free); + let grant = budget.claim(secs(60), Size::G2).expect("a job starts"); + budget.confirm(grant.slot).unwrap(); + // To the byte: at the hold a job still starts, a byte under it none does. + let mut at_hold = watched_with_free(lowered.clone(), 3 * GIB); + assert!(at_hold.claim(secs(60), Size::G2).is_ok()); + let mut reading = calm(secs(61)); + reading.available_bytes = 10 * GIB; + reading.disk_free_bytes = 3 * GIB - 1; + let observed = budget.observe(secs(61), Some(reading)); + assert_eq!(observed.evict, None, "above the stop, the job goes on"); + assert_eq!( + budget.claim(secs(61), Size::G2).map(|_| ()), + Err(Refusal::Trouble(Trouble::DiskLow)) + ); + // Under the stop, the job that wrote is stopped. + budget + .usage( + secs(62), + grant.slot, + Usage { + resident_bytes: 0, + scratch_bytes: 5 * GIB, + }, + ) + .unwrap(); + reading.at = secs(62); + reading.disk_free_bytes = GIB - 1; + assert_eq!( + budget.observe(secs(62), Some(reading)).evict, + Some(Eviction { + slot: grant.slot, + cause: EvictionCause::Disk + }) + ); +} diff --git a/crates/budget/tests/properties.rs b/crates/budget/tests/properties.rs index bb69882..72ce69d 100644 --- a/crates/budget/tests/properties.rs +++ b/crates/budget/tests/properties.rs @@ -6,8 +6,8 @@ mod common; use cf_budget::{ - Assigned, Budget, Charge, EvictionCause, GIB, Hold, HostFacts, HostReading, Level, MIB, Millis, Mode, - Now, Pressure, Profile, Settings, Size, SlotId, Thermal, Usage, admit, + Assigned, Budget, Charge, EvictionCause, GIB, Hold, HostFacts, HostReading, Level, Limits, MIB, Millis, + Mode, Now, Pressure, Profile, Settings, Size, SlotId, Thermal, Usage, admit, }; use common::*; use proptest::prelude::*; @@ -162,6 +162,20 @@ fn op() -> impl Strategy { } fn settings() -> impl Strategy { + // Disk limits a small host's settings give, in GiB: the hold and the stop, the lower of the two. + (host_settings(), prop::option::of((1..30u64, 1..30u64))).prop_map(|(settings, disk)| match disk { + None => settings, + Some((a, b)) => settings + .limited(&Limits { + disk_hold_bytes: Some(a.max(b) * GIB), + disk_evict_bytes: Some(a.min(b) * GIB), + ..Limits::default() + }) + .expect("disk limits in order"), + }) +} + +fn host_settings() -> impl Strategy { ( prop::sample::select(vec![8u64, 16, 32]), 4..16u32, diff --git a/crates/daemon/src/lib.rs b/crates/daemon/src/lib.rs index 95ceb0e..c4ed068 100644 --- a/crates/daemon/src/lib.rs +++ b/crates/daemon/src/lib.rs @@ -39,7 +39,10 @@ pub use machines::{ OFFER_STANDS, RESULT_WAIT, Registration, Reports, SizeRoom, Snapshot, }; #[cfg(any(target_os = "macos", target_os = "linux"))] -pub use platform::{HostMachines, HostService, HostSetup, Platform, This, host_service, records_dir}; +pub use platform::{ + HostMachines, HostService, HostSetup, Platform, This, budget_settings, host_service, records_dir, + state_disk_bytes, +}; pub use records::{ Disagrees, DiskStore, Ending, GithubJob, MemoryStore, RULES, Record, SavedSlot, Store, TestRunFacts, Upgrade, diff --git a/crates/daemon/src/machines.rs b/crates/daemon/src/machines.rs index 8add690..7b8e266 100644 --- a/crates/daemon/src/machines.rs +++ b/crates/daemon/src/machines.rs @@ -43,8 +43,6 @@ pub struct MachineConfig { pub host: String, /// The version of the installed image jobs run in. pub image: String, - /// The size of a job's disk. - pub scratch_bytes: u64, /// The longest a job may run. pub deadline: Duration, /// The lifecycle's waits. One vm-host boots a machine twice by itself, so `starts` is 1 and `start` covers @@ -72,7 +70,6 @@ impl MachineConfig { MachineConfig { host: host.to_string(), image: image.to_string(), - scratch_bytes: 40 << 30, deadline, limits: Limits { // Two boots of the vm-host's own, each with its forced stop, and the disk's copy before them. @@ -1587,7 +1584,7 @@ impl Machines { host: self.config.host.clone(), memory_bytes: grant.vm_bytes, cpus: grant.cpus, - scratch_bytes: self.config.scratch_bytes, + scratch_bytes: inner.budget.settings().job_disk_bytes, image: self.config.image.clone(), workload: own.workload, env: own.env, @@ -2328,7 +2325,7 @@ impl Hosts for Machines { host: self.config.host.clone(), memory_bytes: grant.vm_bytes, cpus: grant.cpus, - scratch_bytes: self.config.scratch_bytes, + scratch_bytes: inner.budget.settings().job_disk_bytes, image: self.config.image.clone(), workload: self.config.stand_in.clone().unwrap_or(Workload::ActionsRunner { jit_config: spec.jit_config, diff --git a/crates/daemon/src/platform/linux.rs b/crates/daemon/src/platform/linux.rs index 3b09594..9e35cce 100644 --- a/crates/daemon/src/platform/linux.rs +++ b/crates/daemon/src/platform/linux.rs @@ -53,6 +53,10 @@ impl Platform for Linux { } } + fn disk_bytes(folder: &Path) -> Option { + LinuxHost::new(folder).ok()?.disk_total_bytes() + } + fn boot_session() -> Option { boot_session(Path::new(ROOT)) } diff --git a/crates/daemon/src/platform/mac.rs b/crates/daemon/src/platform/mac.rs index 6595b5c..5f713ce 100644 --- a/crates/daemon/src/platform/mac.rs +++ b/crates/daemon/src/platform/mac.rs @@ -47,6 +47,10 @@ impl Platform for Mac { } } + fn disk_bytes(folder: &Path) -> Option { + MacHost::new(folder).ok()?.disk_total_bytes() + } + fn boot_session() -> Option { boot_session() } diff --git a/crates/daemon/src/platform/mod.rs b/crates/daemon/src/platform/mod.rs index 0d2ba44..996d9d2 100644 --- a/crates/daemon/src/platform/mod.rs +++ b/crates/daemon/src/platform/mod.rs @@ -15,7 +15,7 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; -use cf_budget::{Host, HostError, HostFacts, Limits, Mode, Sandbox, Settings}; +use cf_budget::{DiskShort, Host, HostError, HostFacts, Limits, Mode, Sandbox, Settings}; use cf_console_client::{Client, OUTBOX_CAPACITY, Outbox}; use cf_ghapp::{Clock, GitHubApp, Http, RetryPolicy, SystemClock}; use cf_guest_proto::Workload; @@ -70,6 +70,10 @@ pub trait Platform { /// costs the host less ([`Settings::in_containers`]). fn settings(facts: HostFacts, mode: Mode, trusted: bool, sandbox: Sandbox) -> Settings; + /// The size of the file system `folder` is on, in bytes; `None` when the system will not say or the folder is + /// not there. + fn disk_bytes(folder: &Path) -> Option; + /// A name for this start of the host: another one after every start, the same until the next, whatever the /// clock does meanwhile. `None` when the system will not say. fn boot_session() -> Option; @@ -115,6 +119,32 @@ pub type HostService = ServiceOf; /// carries what waits, and meanwhile the service's loop goes on. const CONSOLE_ANSWER: Duration = Duration::from_secs(15); +/// The size of the file system the state folder is on, in bytes; 0 when the system will not say. A state folder not +/// made yet (`check` and `install` before the first start) will be on its nearest folder's file system. +pub fn state_disk_bytes(state_dir: &Path) -> u64 { + state_dir.ancestors().find_map(P::disk_bytes).unwrap_or(0) +} + +/// The budget's numbers on a host of this platform: its defaults with what the owner set on top (`limits`), checked, +/// and checked against the state folder's file system of `disk_bytes` ([`state_disk_bytes`], 0 to check nothing +/// against it: [`Settings::check_disk`]). With the settings comes what the disk settings leave short of, which the +/// Runner starts on and says. +pub fn budget_settings( + facts: HostFacts, + mode: Mode, + trusted: bool, + sandbox: Sandbox, + limits: &Limits, + disk_bytes: u64, +) -> Result<(Settings, Option), String> { + let said = |error: cf_budget::SettingsError| format!("the limits for jobs: {error}"); + let settings = P::settings(facts, mode, trusted, sandbox) + .limited(limits) + .map_err(said)?; + let short = settings.check_disk(limits, sandbox, disk_bytes).map_err(said)?; + Ok((settings, short)) +} + /// Where the records of machines are kept in a state folder. pub fn records_dir(state_dir: &Path) -> PathBuf { state_dir.join("records") @@ -202,6 +232,17 @@ pub fn host_service(setup: HostSetup) -> Result<(Arc> Some(console) => Arc::new(OutboxReports(console.outbox.clone())), None => Arc::new(LoggedReports), }; + let (settings, short) = budget_settings::

( + facts, + setup.mode, + setup.policy.trusted.is_some(), + setup.sandbox.0, + &setup.limits, + state_disk_bytes::

(state), + )?; + if let Some(short) = short { + tracing::warn!(%short, "the disk settings leave a job's disk short"); + } let mut config = MachineConfig::new(&setup.machine, &setup.image, &boot, setup.deadline); config.stand_in = setup.stand_in; config.docker_max_jobs = setup.policy.docker_max_jobs; @@ -216,9 +257,7 @@ pub fn host_service(setup: HostSetup) -> Result<(Arc> (Sandbox::Container, None) => return Err("a container host names its engine's socket".into()), (Sandbox::Vm, _) => P::vm_hosts(&setup.program, state), }, - settings: P::settings(facts, setup.mode, setup.policy.trusted.is_some(), setup.sandbox.0) - .limited(&setup.limits) - .map_err(|error| format!("the limits for jobs: {error}"))?, + settings, config, roster: roster.clone(), store: Arc::new(store), diff --git a/crates/daemon/tests/machines.rs b/crates/daemon/tests/machines.rs index 71fa7d8..cd030e2 100644 --- a/crates/daemon/tests/machines.rs +++ b/crates/daemon/tests/machines.rs @@ -421,6 +421,44 @@ async fn when_the_mac_runs_out_of_memory_the_newest_job_is_stopped_and_only_one_ assert_eq!(w.slots(), 1); } +/// A small host whose settings lower the disk limits: its jobs get the disk the settings give, a job starts with +/// free disk down to the hold and none under it, where the defaults took none at all. +#[tokio::test] +async fn a_small_disk_takes_jobs_down_to_the_hold_its_settings_give_with_the_disk_they_give() { + let free = |w: &World, bytes: u64| { + w.m.with_host(|host| host.set(|r| r.disk_free_bytes = bytes)); + pass(&w.m, 61); + }; + let defaults = World::new(2); + free(&defaults, 4 * GIB); + assert!( + matches!(defaults.m.claim(SIZE, None), Claim::Waits { why, .. } if why == "disk"), + "the default hold of 25 GiB" + ); + + let limits = cf_budget::Limits { + disk_hold_bytes: Some(3 * GIB), + disk_evict_bytes: Some(GIB), + job_disk_bytes: Some(6 * GIB), + ..cf_budget::Limits::default() + }; + let mut w = World::on(cf_daemon::testing::settings_for(2).limited(&limits).unwrap()); + free(&w, 3 * GIB); + let first = w.running().await; + assert_eq!( + w.parts + .vms + .started() + .iter() + .map(|request| (request.job.as_str(), request.scratch_bytes)) + .collect::>(), + [(job_name(&first), 6 * GIB)] + ); + free(&w, 3 * GIB - 1); + assert!(matches!(w.m.claim(SIZE, None), Claim::Waits { why, .. } if why == "disk")); + assert!(w.parts.vms.stopped().is_empty(), "above the stop the job goes on"); +} + #[tokio::test] async fn when_the_disk_runs_out_the_job_with_the_most_on_it_is_stopped() { let mut w = World::new(2); diff --git a/crates/runner/src/config.rs b/crates/runner/src/config.rs index ec381de..fea6328 100644 --- a/crates/runner/src/config.rs +++ b/crates/runner/src/config.rs @@ -107,6 +107,20 @@ pub struct Config { /// defaults (all but two on a host of its own). The sizes' vCPUs follow it unless `vcpus` names them. #[serde(default)] pub ci_cpus: Option, + /// The free disk under which no job starts, in GB of 2^30 bytes (`3`), at the file system of the state folder: + /// at least 1 and under the file system's size. Left out for 25. Set alone, it also lowers `disk_stop_gb` to + /// itself when that is higher. + #[serde(default)] + pub disk_hold_gb: Option, + /// The free disk under which the job that wrote the most is stopped, in GB: at least 1 and no more than + /// `disk_hold_gb`. Left out for 5, or `disk_hold_gb` when that is less. + #[serde(default)] + pub disk_stop_gb: Option, + /// The size of each job's disk, in GB: a machine's scratch disk, or the most a container writes into its layer + /// (the engine's `StorageOpt` size where it bounds one). 4 to 512 and no larger than the file system of the state + /// folder; left out for 40. + #[serde(default)] + pub job_disk_gb: Option, /// vCPUs of a machine of each size named, by its label (`{"cf-4g": 3}`), 1 to 64 and no more than CI may use; /// the sizes left out keep the host's defaults. #[serde(default)] @@ -286,21 +300,26 @@ impl Config { Settings::FLOOR_PERCENT.end() )); } - if let Some(gb) = &self.ci_memory_gb - && !gb - .as_f64() - .is_some_and(|gb| gb.is_finite() && gb > 0.0 && gb <= CI_MEMORY_GB_MAX) - { - return Err(format!( - "ci_memory_gb: more than 0 and at most {CI_MEMORY_GB_MAX}" - )); - } if self .ci_cpus .is_some_and(|cpus| !(1..=CI_CPUS_MAX).contains(&cpus)) { return Err(format!("ci_cpus: between 1 and {CI_CPUS_MAX}")); } + for (key, gb, most) in [ + ("ci_memory_gb", &self.ci_memory_gb, CI_MEMORY_GB_MAX), + ("disk_hold_gb", &self.disk_hold_gb, DISK_GB_MAX), + ("disk_stop_gb", &self.disk_stop_gb, DISK_GB_MAX), + ("job_disk_gb", &self.job_disk_gb, JOB_DISK_GB_MAX), + ] { + if let Some(gb) = gb + && !gb + .as_f64() + .is_some_and(|gb| gb.is_finite() && gb > 0.0 && gb <= most) + { + return Err(format!("{key}: more than 0 and at most {most}")); + } + } if self.labels.len() > 8 { return Err("labels: 8 at most".into()); } @@ -460,12 +479,11 @@ impl Config { idle_after: self.idle_minutes.map(|minutes| Millis::mins(minutes.into())), floor_percent: self.min_free_memory_percent, vcpus: Size::ALL.map(|size| self.vcpus.get(size.label()).copied()), - ci_bytes: self - .ci_memory_gb - .as_ref() - .and_then(serde_json::Number::as_f64) - .map(ci_bytes), + ci_bytes: gb_bytes(self.ci_memory_gb.as_ref()), ci_cpus: self.ci_cpus, + disk_hold_bytes: gb_bytes(self.disk_hold_gb.as_ref()), + disk_evict_bytes: gb_bytes(self.disk_stop_gb.as_ref()), + job_disk_bytes: gb_bytes(self.job_disk_gb.as_ref()), } } @@ -561,17 +579,25 @@ const CI_CPUS_MAX: u32 = 1024; /// The most `ci_memory_gb` may say, far above any host's memory: the budget refuses more than the host has. const CI_MEMORY_GB_MAX: f64 = 4096.0; -/// `ci_memory_gb` in bytes, to the byte below. +/// The bytes of a number of GB of 2^30 bytes in the file, whole bytes down. #[allow( clippy::cast_possible_truncation, clippy::cast_sign_loss, clippy::cast_precision_loss )] -fn ci_bytes(gb: f64) -> u64 { - // In range by the check: a finite number above 0 and at most CI_MEMORY_GB_MAX. - (gb * cf_budget::GIB as f64).floor() as u64 +fn gb_bytes(gb: Option<&serde_json::Number>) -> Option { + // In range by the check: a finite number above 0 and at most DISK_GB_MAX, far inside a u64. + gb.and_then(serde_json::Number::as_f64) + .map(|gb| (gb * cf_budget::GIB as f64).floor() as u64) } +/// The most a disk setting may say, in GB: larger than any disk the Runner is on; the budget and the disk's size +/// refuse what does not fit. +const DISK_GB_MAX: f64 = 65536.0; + +/// The most `job_disk_gb` may say: the largest disk a job may have, which the budget checks again. +const JOB_DISK_GB_MAX: f64 = (*Settings::JOB_DISK_RANGE.end() / cf_budget::GIB) as f64; + #[cfg(test)] mod tests { use super::*; @@ -1181,6 +1207,61 @@ mod tests { ); } + #[test] + fn the_disk_settings_are_gb_above_0_and_left_out_keep_the_defaults() { + let gib = cf_budget::GIB; + let whole = Config::parse(WHOLE).expect("config").limits(); + assert_eq!( + ( + whole.disk_hold_bytes, + whole.disk_evict_bytes, + whole.job_disk_bytes + ), + (None, None, None) + ); + type Read = fn(&Limits) -> Option; + let keys: [(&str, Read, u64); 3] = [ + ("disk_hold_gb", |l| l.disk_hold_bytes, 65536), + ("disk_stop_gb", |l| l.disk_evict_bytes, 65536), + ("job_disk_gb", |l| l.job_disk_bytes, 512), + ]; + for (key, of, most) in keys { + for (gb, bytes) in [ + (serde_json::json!(3), Some(3 * gib)), + (serde_json::json!(2.5), Some(5 * gib / 2)), + (serde_json::json!(1e-9), Some(1)), + (serde_json::json!(most), Some(most * gib)), + (serde_json::Value::Null, None), + ] { + assert_eq!( + with(|v| v[key] = gb.clone()).map(|c| of(&c.limits())), + Ok(bytes), + "{key} {gb}" + ); + } + for gb in [ + serde_json::json!(0), + serde_json::json!(-0.0), + serde_json::json!(-1), + serde_json::json!(-1e300), + serde_json::json!(most as f64 + 0.5), + serde_json::json!(1e300), + ] { + let error = with(|v| v[key] = gb.clone()).expect_err("refused"); + assert_eq!(error, format!("{key}: more than 0 and at most {most}"), "{gb}"); + } + for gb in [ + serde_json::json!("3"), + serde_json::json!([3]), + serde_json::json!({"gb": 3}), + serde_json::json!(true), + ] { + let error = with(|v| v[key] = gb.clone()).expect_err("refused"); + assert!(error.contains("invalid type"), "{key} {gb}: {error}"); + } + } + } + #[test] fn cpus_for_ci_are_whole_and_within_reason() { assert_eq!(Config::parse(WHOLE).expect("config").limits().ci_cpus, None); diff --git a/crates/runner/src/service.rs b/crates/runner/src/service.rs index 5d58a29..7d39155 100644 --- a/crates/runner/src/service.rs +++ b/crates/runner/src/service.rs @@ -7,11 +7,13 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; -use cf_budget::{Budget, Host as _, HostFacts, HostReading, Settings, Size, admit}; +use cf_budget::{Budget, DiskShort, Host as _, HostFacts, HostReading, Settings, Size, admit}; use cf_console_client::ConsoleTokens; use cf_daemon::names::ServePolicy; use cf_daemon::update::{STAYS_UP, SelfUpdate, SettingsChange, SettingsFile, SettingsLook, UpdateFile}; -use cf_daemon::{HostMachines, HostService, HostSetup, Lines, Platform, ROLL_AT, This, host_service}; +use cf_daemon::{ + HostMachines, HostService, HostSetup, Lines, Platform, ROLL_AT, This, host_service, state_disk_bytes, +}; use cf_ghapp::{AppCredentials, GitHubApp, Http, KeyTokens, RetryPolicy, SystemClock, SystemMonotonic}; use cf_secret::Secret; @@ -134,16 +136,22 @@ fn console(config: &Config) -> Result)>, String> Ok(Some((address.clone(), Secret::new(token)))) } -/// The budget's numbers on this host under the configuration: the defaults with what the file sets on top. -fn budget_of(facts: HostFacts, config: &Config) -> Result { - This::settings( +/// The budget's numbers on this host under the configuration: the defaults with what the file sets on top, checked +/// against the state folder's file system of `disk_bytes` (0 checks nothing against it), with what its disk settings +/// leave short of. +fn budget_of( + facts: HostFacts, + config: &Config, + disk_bytes: u64, +) -> Result<(Settings, Option), String> { + cf_daemon::budget_settings::( facts, config.mode(), config.class == Class::Trusted, config.sandbox, + &config.limits(), + disk_bytes, ) - .limited(&config.limits()) - .map_err(|error| format!("the limits for jobs: {error}")) } /// The settings in `path`, when a Runner can start on them: the file is one it takes, its limits are ones the budget @@ -157,7 +165,7 @@ fn startable(path: &Path) -> Result { let facts = This::host(Path::new("/")) .and_then(|host| host.facts()) .map_err(|error| format!("{}: {error}", This::SYSTEM))?; - budget_of(facts, &config)?; + budget_of(facts, &config, state_disk_bytes::(&config.state_dir))?; installed_image(&config)?; app(&config)?; console(&config)?; @@ -719,12 +727,18 @@ pub fn enroll(config: &Path) -> Result<(), String> { Ok(()) } -/// What this host gives jobs under the configuration: the sizes it runs, with their CPUs, and how many of the -/// commonest fit side by side. -fn room(config: &Config) -> Result { - let facts = This::host(Path::new("/")) +/// What this host gives jobs under the configuration, as lines of `check`: the sizes it runs, with their CPUs, and +/// how many of the commonest fit side by side ([`room_line`]); then the disk limits they run under ([`disk_line`]). +fn room(config: &Config) -> Vec> { + let disk = state_disk_bytes::(&config.state_dir); + let read = This::host(Path::new("/")) .and_then(|host| host.facts()) - .map_err(|error| format!("{}: {error}", This::SYSTEM))?; + .map_err(|error| format!("{}: {error}", This::SYSTEM)); + let (facts, (settings, short)) = match read.and_then(|facts| Ok((facts, budget_of(facts, config, disk)?))) + { + Ok(read) => read, + Err(error) => return vec![Err(error)], + }; #[cfg(target_os = "macos")] let (whose, when) = match config.mac_use { crate::config::Use::Dedicated => ("nobody works here", ""), @@ -732,7 +746,29 @@ fn room(config: &Config) -> Result { }; #[cfg(not(target_os = "macos"))] let (whose, when) = ("nobody sits at it", ""); - Ok(room_line(facts, &budget_of(facts, config)?, whose, when)) + vec![ + Ok(room_line(facts, &settings, whose, when)), + Ok(disk_line(&settings, disk, short)), + ] +} + +/// [`room`]'s line of the disk limits on a state folder's file system of `disk_bytes` (0: not known), with what they +/// leave short of. +fn disk_line(settings: &Settings, disk_bytes: u64, short: Option) -> String { + let disk = match disk_bytes { + 0 => "the size of its file system cannot be read".to_string(), + bytes => format!("its file system has {}", gib(bytes)), + }; + let line = format!( + "no job starts under {} free disk, a job is stopped under {}, each job's disk is {}; {disk}", + gib(settings.disk_hold_bytes), + gib(settings.disk_evict_bytes), + gib(settings.job_disk_bytes), + ); + match short { + Some(short) => format!("{line}; but {short}"), + None => line, + } } /// [`room`]'s line for these facts and settings, in GiB as [`memory_now`]'s. @@ -814,8 +850,8 @@ fn now_line(config: &Config) -> String { Ok((facts, host.read()?)) }); match read { - Ok((facts, reading)) => match budget_of(facts, config) { - Ok(settings) => memory_now(&settings, &reading), + Ok((facts, reading)) => match budget_of(facts, config, 0) { + Ok((settings, _)) => memory_now(&settings, &reading), Err(error) => error, }, Err(error) => format!("{} could not be read: {error}", This::SYSTEM), @@ -870,8 +906,8 @@ fn common_needs(config: &Config, program: &Path) -> Vec> } }), This::fit(program).map(|()| format!("{} can start machines", program.display())), - room(config), ]; + lines.extend(room(config)); // Right after the image: the engine must hold it too. if let Some(socket) = &config.engine { lines.insert(1, cf_vmm_oci::probe(socket, &config.image)); @@ -1280,6 +1316,54 @@ mod tests { assert_eq!(file.look(), SettingsLook::Same, "put back as it was"); } + /// The disk settings a small host's file gives reach the budget a Runner starts with, and one that no file system + /// of the state folder's size could meet makes the file one no Runner starts on. + #[test] + fn the_disk_settings_reach_the_budget_and_a_hold_past_the_state_folders_disk_is_refused() { + let here = Startable::new(); + let gib = cf_budget::GIB; + here.write(&here.text("2026.10.1", "")); + let file = here.file(); + let facts = This::host(Path::new("/")) + .and_then(|host| host.facts()) + .expect("facts"); + let budget = |more: &str| { + here.write(&here.text("2026.10.1", more)); + let config = Config::load(&here.path).expect("a file"); + budget_of(facts, &config, state_disk_bytes::(&config.state_dir)).map(|(settings, _)| { + ( + settings.disk_hold_bytes, + settings.disk_evict_bytes, + settings.job_disk_bytes, + ) + }) + }; + assert_eq!(budget(""), Ok((25 * gib, 5 * gib, 40 * gib)), "the defaults"); + assert_eq!( + budget(r#", "disk_hold_gb": 3, "disk_stop_gb": 1.5, "job_disk_gb": 6"#), + Ok((3 * gib, 3 * gib / 2, 6 * gib)) + ); + assert!(matches!(file.look(), SettingsLook::Changed { .. })); + let unusable = |more: &str, why: &str| { + here.write(&here.text("2026.10.1", more)); + match file.look() { + SettingsLook::Unusable { why: said } => assert!(said.contains(why), "{more}: {said}"), + other => panic!("{more}: {other:?}"), + } + }; + let disk = This::disk_bytes(&here.state).expect("the state folder's disk"); + let past = (disk / gib + 1).to_string(); + unusable(&format!(r#", "disk_hold_gb": {past}"#), "no job would ever start"); + unusable(r#", "disk_hold_gb": 0.5"#, "no job starts is under"); + unusable(r#", "disk_stop_gb": 0.5"#, "a job is stopped is under"); + unusable( + r#", "disk_hold_gb": 3, "disk_stop_gb": 4"#, + "is above the one under which no job starts", + ); + unusable(r#", "job_disk_gb": 3"#, "a job's disk of"); + unusable(r#", "disk_stop_gb": -1"#, "disk_stop_gb: more than 0"); + } + /// The console's `settings` command, written by the Runner into its settings file: each setting is set or /// taken out, everything else stays as it was, and a Runner started on the file runs with those limits. #[test] From ed6abd2f36aaedacc44cbf67b3f8ed647b953268 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krystian=20Paw=C5=82ow?= Date: Thu, 12 Nov 2026 22:40:06 +0100 Subject: [PATCH 3/3] Document the disk settings for a small container host --- README.md | 4 ++++ docs/MAP.md | 6 +++--- docs/containers.md | 37 +++++++++++++++++++++++++++++++++---- 3 files changed, 40 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 6f491f0..dd81357 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,10 @@ A configuration file says what the Mac is: on a Mac, `["cf-linux"]` on Linux) and `only_labels` (labels of `labels` the host is kept for: it then serves only the jobs that ask for one of them, so a host with `"labels": ["cf-linux", "cf-ops"], "only_labels": ["cf-ops"]` is always free for `runs-on: [cf-2g, cf-ops]`, and the tests go elsewhere). +- Optional, for a host with a small disk, in GB of 2^30 bytes at the file system of `state_dir`: `disk_hold_gb` (no + job starts with less free; 25), `disk_stop_gb` (with less free the job that wrote the most is stopped; 5, and never + above `disk_hold_gb`, which lowers it when set alone) and `job_disk_gb` (each job's disk, 4 to 512; 40). A hold the + file system can never meet, or a job's disk larger than it, is refused. `check` prints the disk line. - The file names where the console's token goes, so one that others can write is refused (`chmod go-w` it). Keep it in a folder that stays and is not `state_dir` (which `--purge` deletes): the agent reads it at every start. diff --git a/docs/MAP.md b/docs/MAP.md index 9d4f6e9..d16e364 100644 --- a/docs/MAP.md +++ b/docs/MAP.md @@ -13,14 +13,14 @@ | `crates/nft` (`cf-nft`) | nf_tables rules without the `nft` program. `lib.rs` is the model (`Match`, `Rule` and `Rule::new`, `Chain` on a `Hook`, `Table` in a `Family`: inet, bridge, netdev) and `decide`, what a rule set does with a packet (pure). `encode.rs` turns tables into one netlink batch (`batch`, `Load::Replace` swaps the whole ruleset in one transaction) and refuses what the kernel would; `decode.rs` reads the kernel's messages back (`read`). `apply` and `list` (Linux) talk to the kernel through `cf-netlink`. `BLOCKED` is the one list of private and special ranges, used by the guest's firewall and by the host's in front of a Linux machine. Tests: `tests/nft.rs` (bytes `nft` 1.0.6 wrote, recorded in `tests/nft/`), `tests/model.rs` (a model of the kernel's expressions against `decide`, round trips), `tests/kernel.rs` (Linux: applied and listed back in a fresh user and network namespace). | | `crates/netlink` (`cf-netlink`) | Netlink itself, for any protocol: `message`, `attribute`, `attribute_str` (a NUL-ended text), `nested`, `messages` and `attributes` to read them back, and on Linux the `Socket` that sends requests and reads every answer (`request`, `dump`), with its own `Error`. Users: `cf-nft` (nf_tables), `cf-vmm-ch` (rtnetlink). Tests in `tests/wire.rs`. | | `crates/vmm-ch` (`cf-vmm-ch`) | `ChVmm`: machines on Linux, cloud-hypervisor under KVM, one process per machine (`machine.rs`, Linux). Each machine has a cgroup of its own under the part of the tree delegated to the Runner (`cgroup.rs`: memory at its size and `overhead`, no swap (a kernel without a swap account per cgroup only on a host that swaps to nothing), `pids.max`, the cpuset of its cores, `cgroup.kill`, `wait_empty` woken by `cgroup.events`; `memory.current` is `Vm::resident`; `own` and `enter_host` for a test suite that shares its delegated cgroup), its vCPUs on the host's fast cores that it may use (`cpus.rs` with `cf_host_linux::fast_cpus`, `--host-cpus` overrides). With a network it runs under the system's pasta (`programs::pasta`: found on `PATH`, refused when another user could change it or what it links to, tried when the first machine with a network starts; `apt install passt`, 2023-03 or newer) in a user and network namespace; the hidden `vm-sandbox` subcommand of the binary (`sandbox.rs`) makes the guest's tap, loads `firewall.rs` onto it (netdev ingress: ARP, DHCP and DNS to pasta on the gateway, IPv4 to the internet; drops `BLOCKED`, 127/8 and every network of the host, IPv6 and everything else), bridges it with pasta's device (`link.rs`, rtnetlink), and starts the monitor without capabilities and without user namespaces in a PID, cgroup, mount and empty network namespace of its own (the tap's descriptor its only way out), no file it writes growing past the largest it is shown, its root a read-only tmpfs with only what `Sandbox::for_machine` shows it (its program, the guest's files, its console and folder, `/dev/kvm`, `/dev/urandom`) and a `/proc` of its own, then reports how it ended (`ending.rs`). `command.rs` builds both command lines, `events.rs` reads `--event-monitor`, `vsock.rs` the CONNECT handshake. cloud-hypervisor is pinned in `pins.rs` for each architecture, the host's fetched into `target/vmm-programs` and embedded by `build.rs` (Linux on aarch64 or x86_64), unpacked into `/programs` and checked before each machine (`programs.rs`). Tests: unit tests of everything pure, `tests/isolation.rs` (Linux: a stand-in guest on a veth pair behind the real pasta, guard and firewall; each blocked target a test of its own), `tests/monitor_root.rs` (Linux, no harness: a stand-in monitor in the real sandbox tries the host's files, processes and mounts, and under pasta tries to send past its tap), `tests/kvm.rs` (Linux with KVM, no harness: how a real machine ends, a panic, a stop, a kill, its cgroup's OOM and a dead monitor; it skips without `/dev/kvm` and the guest, `CF_REQUIRE_KVM=1` makes that a failure). | -| `crates/runner` (`codefusion-runner`) | The binary. `commands.rs`: `version` (the package's version and the commit, which `build.rs` reads from git or `CF_COMMIT`), `image verify`, `image install`, `pause`, `resume` and `status` (`holds.rs`: a named hold on the machine, `--wait` until the Runner saw it and runs nothing, exit 75 when the wait runs out; `status --json`), `vm` (the vm-host, `src/vm.rs`), `boot-proof`, `vmm` (Linux: the cloud-hypervisor the build carries, which `install` and an update ask of a program), and the service's commands in `service.rs` (macOS and Linux): `daemon` (what the service manager runs; ends on SIGTERM; one per state folder, held by a lock on `/runner.lock`), `enroll` (the console's one-time code on standard input, read with the terminal's echo off by `hidden.rs`, which gives the terminal back on every way out, a signal included; the token to `/console-token`, 0600, never printed), `check [--github]` (what an install needs, and the memory at that moment against the budget's floor with the sizes that fit; with `--github` what GitHub shows, changing nothing there and nothing in the state folder), `install` (copies the program to `/bin`, writes the service manager's entry, refuses while a machine stands), `uninstall [--leave] [--purge]` (refused while a Runner started by hand holds the state folder; a Mac without a usable token goes without telling the console). `config.rs` is the configuration file (refused when others can write it; `console` required unless the file names `app_dir` (absolute, not through `..`), the folder of a GitHub App key the host held before the console held it: while its `app.pem` is there `service.rs` `app_and_key` signs with it (`held_key`: `KeyTokens`, and `cf_daemon::HeldKey` for the service), else the GitHub tokens come from the console (`ConsoleTokens`); `check` and `uninstall` sign with a held key and never take it out (`looking`); `use` required on a Mac; on Linux `dedicated` or left out; `max_jobs`, `idle_minutes` (a workstation's only), `min_free_memory_percent` and `vcpus` (a size's vCPUs by its label, which `Settings::limited` checks against the CPUs CI may use), `ci_memory_gb` (the memory CI's machines hold at once, each with its overhead, in GB of 2^30 bytes) and `ci_cpus` (the CPUs they take together) are the owner's `Limits`, which `budget_of` and the service lay over the host's settings; `ConfigFile::change` writes the console's `settings` command into the file through `.json.next`, which must start a Runner before it replaces the file) and, on a Mac, the refusal of the cf-runners manager's own name and of external volumes; `manager.rs` is `ServiceManager`, the seam to the service manager; `launchd.rs` the agent `cc.codefusion.vm-runner` in the person's session (`plist`, `Agent`; `stop` waits past `ExitTimeOut` and decides by whether launchd still has the agent, not by what `bootout` says); `systemd.rs` (Linux) the user unit `codefusion-runner.service` (`unit`, with every path quoted for systemd or refused; `UserUnit` over an injected `systemctl --user`, the real one ended after 100 s; `stop` waits past `TimeoutStopSec` and decides by `ActiveState`); `linux.rs` the lines `check` adds on Linux (`Facts`, `lines`: KVM, lingering, delegated controllers, user namespaces, pasta, `noexec`, each with the root command that fixes it). `update.rs`: `CheckoutUpdate`, the `SelfUpdate` of a host that built the Runner, for the console's `update` and `update-to`: in the checkout the program was built from (`build.rs` writes it down, with `CARGO_HOME`, `RUSTUP_HOME` and `CARGO_BUILD_JOBS`), only when it is the config's `build_folder`, clean and at a commit (`own`), fetches `origin refs/heads/main` (a tag named `main` is not it), takes main's head or the commit `update-to` names when that resolves to one commit of the main fetched (`on_main`), refuses a target that does not contain the running commit (both looked up among the objects alone by `commit_named`, so a branch or tag named like a commit never stands for it), checks out the peeled commit, builds with `cargo xtask build` at low priority into `logs/update.log` and ends a build that takes longer than 45 minutes or outlasts the daemon, and keeps the program it checked as `bin/codefusion-runner.next`; `install` places that one, keeps the running program as `bin/codefusion-runner.prev` and writes `update.json` first; `guard`, run before the daemon reads its configuration, counts the new program's starts and puts the one before back after `MAX_STARTS`, and `ended_in_order` takes back the count of a start that ended as asked. `Installer` is what both updates share: `offer` asks a program its `version` (it must run here and name the commit) and keeps it as `.next`, `install` places it. `release_update.rs` (Linux and macOS builds; Linux hosts only by asset): `ReleaseUpdate`, the `SelfUpdate` of a Runner with no `build_folder`, installed from a release: `updates_from` is `Releases`, so it lists only `update-to`; `build` finds the commit's release (`runner--<12 hex>` by tag, else the newest page), waits `RELEASE_WAIT` for one not out yet, asked every `ASK_EVERY` by its tag and every `PAGE_EVERY`th look among the newest too, ended by `end`; refuses a draft, another publisher than `github-actions[bot]`, one not made at the whole commit, not on main or not newer than the running commit (`cf_ghapp::Releases::compare`); fetches `SHA256SUMS` and this host's asset (`asset_for_this_host`) into `.download` (0600), checks its SHA-256 against `SHA256SUMS` (`cf_image::parse_sums`), and removes the download whatever happens. `service.rs` `self_update` picks `CheckoutUpdate` with a `build_folder`, else `ReleaseUpdate`. `app_key.rs`: `take_out`: a file that still names `app_dir` has `/app.pem` deleted (a symlink: the link alone, never what it points to), `app_dir` taken out (the file without it checked as a configuration before anything is deleted, then written through `.json.next`) and the deleted file remembered first in `/app-key-file` (0600; a link's target on a second line; `remembered`), which the report's `appKeyFile` names while something is still there; refused, changing nothing, while the file names no console or the host holds no token for it (`KEPT_NO_CONSOLE`, `KEPT_NOT_ENROLLED`). A key deleted whose file could not be written is no failure: `TakenOut::app_dir_kept` says why the file still names the folder. `one_change` is held from the read to the write, by the take-out and by `ConfigFile::change` (the console's settings). `When::Answered` is the service's, once a scale set call of the console this host made was answered (`HeldKey::take_out`); `When::KeyGone` is what `daemon` runs (`start_on`) after `guard` and the state folder's lock (`only_runner`) and before it checks its configuration: it deletes nothing, and takes out only a folder whose key file this Runner's own take-out remembered; a key file missing otherwise keeps the folder (`NOT_TAKEN_OUT_HERE`), and the report's `appKeyKept` says so (`app_key_said`). `runner.entitlements` is signed into it. `config.rs`: `class` (`shared` or `trusted`) and `trusted` (the rules); `check_class` refuses a trusted host that would serve more than one job of one repository, or that holds the GitHub App's key (`app_dir`) or has no `console` to register through. | -| `crates/budget` (`cf-budget`) | Admission, with no I/O: `Budget` (`observe`, `claim`, `confirm`, `assign` and `withdraw` for a job GitHub assigned without asking (`assign_in_turn` and `confirm_in_turn` with its `Turn`: `line` puts main-branch and deploy jobs first, repositories in turn, none passed over past `DEFER_LIMIT`; `set_turn` when what a waiting job stands for changed; a `Turn::refusable` job, a test run, gets room only as a claim would, under the limits in force, and waiting is passed over without holding up those behind it; `make_room` holds back every other size, `making-room`, and puts the size first in line), `adopt`, `usage`, `release`, `hold`, and `status`, which names every slot with how it started and every waiting job with what keeps it), the pure `admit`, `Settings` (every number, `for_host` has the defaults; `limited` lays the owner's `Limits` over them: `max_machines`, refused past it with `Refusal::Machines`, `idle_after`, the floor as a share of memory, which lowers the cap, `ci_bytes`, the memory CI's machines hold at once with each one's overhead (`admit` counts them; `tests/ci_memory.rs`), and `ci_cpus`, the CPUs they take together; `for_headless` those of a Linux host nobody sits at, which reserves what the host needs, `HEADLESS_HOST_USE`, and caps the sizes so that each machine fits with its overhead; `on_fast_cores` gives CI only the fast kind of core of a host whose cores differ; `one_at_a_time` those of a trusted host, which runs one machine on every core it has), `Size` and its labels, the `Host` trait with `HostReading`, `Battery`, what a host's page shows of its battery, and `FakeHost`, the host tests script (`read_takes` lets a read take time before and after the reading gets its time; `tests/fake_host.rs`). Tests: `admission`, `assigned`, `headless`, `limits` (the rule at 4, 7.75, 8, 16 and 32 GiB and its boundaries), `levels`, `making_room`, `races`, `properties`, `turns`. | +| `crates/runner` (`codefusion-runner`) | The binary. `commands.rs`: `version` (the package's version and the commit, which `build.rs` reads from git or `CF_COMMIT`), `image verify`, `image install`, `pause`, `resume` and `status` (`holds.rs`: a named hold on the machine, `--wait` until the Runner saw it and runs nothing, exit 75 when the wait runs out; `status --json`), `vm` (the vm-host, `src/vm.rs`), `boot-proof`, `vmm` (Linux: the cloud-hypervisor the build carries, which `install` and an update ask of a program), and the service's commands in `service.rs` (macOS and Linux): `daemon` (what the service manager runs; ends on SIGTERM; one per state folder, held by a lock on `/runner.lock`), `enroll` (the console's one-time code on standard input, read with the terminal's echo off by `hidden.rs`, which gives the terminal back on every way out, a signal included; the token to `/console-token`, 0600, never printed), `check [--github]` (what an install needs, and the memory at that moment against the budget's floor with the sizes that fit; with `--github` what GitHub shows, changing nothing there and nothing in the state folder), `install` (copies the program to `/bin`, writes the service manager's entry, refuses while a machine stands), `uninstall [--leave] [--purge]` (refused while a Runner started by hand holds the state folder; a Mac without a usable token goes without telling the console). `config.rs` is the configuration file (refused when others can write it; `console` required unless the file names `app_dir` (absolute, not through `..`), the folder of a GitHub App key the host held before the console held it: while its `app.pem` is there `service.rs` `app_and_key` signs with it (`held_key`: `KeyTokens`, and `cf_daemon::HeldKey` for the service), else the GitHub tokens come from the console (`ConsoleTokens`); `check` and `uninstall` sign with a held key and never take it out (`looking`); `use` required on a Mac; on Linux `dedicated` or left out; `max_jobs`, `idle_minutes` (a workstation's only), `min_free_memory_percent` and `vcpus` (a size's vCPUs by its label, which `Settings::limited` checks against the CPUs CI may use), `ci_memory_gb` (the memory CI's machines hold at once, each with its overhead, in GB of 2^30 bytes), `ci_cpus` (the CPUs they take together), `disk_hold_gb`, `disk_stop_gb` (the free disk under which no job starts, and under which a job is stopped) and `job_disk_gb` (each job's disk) are the owner's `Limits`, which `budget_of` and the service lay over the host's settings through `cf_daemon::budget_settings`; `ConfigFile::change` writes the console's `settings` command into the file through `.json.next`, which must start a Runner before it replaces the file) and, on a Mac, the refusal of the cf-runners manager's own name and of external volumes; `manager.rs` is `ServiceManager`, the seam to the service manager; `launchd.rs` the agent `cc.codefusion.vm-runner` in the person's session (`plist`, `Agent`; `stop` waits past `ExitTimeOut` and decides by whether launchd still has the agent, not by what `bootout` says); `systemd.rs` (Linux) the user unit `codefusion-runner.service` (`unit`, with every path quoted for systemd or refused; `UserUnit` over an injected `systemctl --user`, the real one ended after 100 s; `stop` waits past `TimeoutStopSec` and decides by `ActiveState`); `linux.rs` the lines `check` adds on Linux (`Facts`, `lines`: KVM, lingering, delegated controllers, user namespaces, pasta, `noexec`, each with the root command that fixes it). `update.rs`: `CheckoutUpdate`, the `SelfUpdate` of a host that built the Runner, for the console's `update` and `update-to`: in the checkout the program was built from (`build.rs` writes it down, with `CARGO_HOME`, `RUSTUP_HOME` and `CARGO_BUILD_JOBS`), only when it is the config's `build_folder`, clean and at a commit (`own`), fetches `origin refs/heads/main` (a tag named `main` is not it), takes main's head or the commit `update-to` names when that resolves to one commit of the main fetched (`on_main`), refuses a target that does not contain the running commit (both looked up among the objects alone by `commit_named`, so a branch or tag named like a commit never stands for it), checks out the peeled commit, builds with `cargo xtask build` at low priority into `logs/update.log` and ends a build that takes longer than 45 minutes or outlasts the daemon, and keeps the program it checked as `bin/codefusion-runner.next`; `install` places that one, keeps the running program as `bin/codefusion-runner.prev` and writes `update.json` first; `guard`, run before the daemon reads its configuration, counts the new program's starts and puts the one before back after `MAX_STARTS`, and `ended_in_order` takes back the count of a start that ended as asked. `Installer` is what both updates share: `offer` asks a program its `version` (it must run here and name the commit) and keeps it as `.next`, `install` places it. `release_update.rs` (Linux and macOS builds; Linux hosts only by asset): `ReleaseUpdate`, the `SelfUpdate` of a Runner with no `build_folder`, installed from a release: `updates_from` is `Releases`, so it lists only `update-to`; `build` finds the commit's release (`runner--<12 hex>` by tag, else the newest page), waits `RELEASE_WAIT` for one not out yet, asked every `ASK_EVERY` by its tag and every `PAGE_EVERY`th look among the newest too, ended by `end`; refuses a draft, another publisher than `github-actions[bot]`, one not made at the whole commit, not on main or not newer than the running commit (`cf_ghapp::Releases::compare`); fetches `SHA256SUMS` and this host's asset (`asset_for_this_host`) into `.download` (0600), checks its SHA-256 against `SHA256SUMS` (`cf_image::parse_sums`), and removes the download whatever happens. `service.rs` `self_update` picks `CheckoutUpdate` with a `build_folder`, else `ReleaseUpdate`. `app_key.rs`: `take_out`: a file that still names `app_dir` has `/app.pem` deleted (a symlink: the link alone, never what it points to), `app_dir` taken out (the file without it checked as a configuration before anything is deleted, then written through `.json.next`) and the deleted file remembered first in `/app-key-file` (0600; a link's target on a second line; `remembered`), which the report's `appKeyFile` names while something is still there; refused, changing nothing, while the file names no console or the host holds no token for it (`KEPT_NO_CONSOLE`, `KEPT_NOT_ENROLLED`). A key deleted whose file could not be written is no failure: `TakenOut::app_dir_kept` says why the file still names the folder. `one_change` is held from the read to the write, by the take-out and by `ConfigFile::change` (the console's settings). `When::Answered` is the service's, once a scale set call of the console this host made was answered (`HeldKey::take_out`); `When::KeyGone` is what `daemon` runs (`start_on`) after `guard` and the state folder's lock (`only_runner`) and before it checks its configuration: it deletes nothing, and takes out only a folder whose key file this Runner's own take-out remembered; a key file missing otherwise keeps the folder (`NOT_TAKEN_OUT_HERE`), and the report's `appKeyKept` says so (`app_key_said`). `runner.entitlements` is signed into it. `config.rs`: `class` (`shared` or `trusted`) and `trusted` (the rules); `check_class` refuses a trusted host that would serve more than one job of one repository, or that holds the GitHub App's key (`app_dir`) or has no `console` to register through. | +| `crates/budget` (`cf-budget`) | Admission, with no I/O: `Budget` (`observe`, `claim`, `confirm`, `assign` and `withdraw` for a job GitHub assigned without asking (`assign_in_turn` and `confirm_in_turn` with its `Turn`: `line` puts main-branch and deploy jobs first, repositories in turn, none passed over past `DEFER_LIMIT`; `set_turn` when what a waiting job stands for changed; a `Turn::refusable` job, a test run, gets room only as a claim would, under the limits in force, and waiting is passed over without holding up those behind it; `make_room` holds back every other size, `making-room`, and puts the size first in line), `adopt`, `usage`, `release`, `hold`, and `status`, which names every slot with how it started and every waiting job with what keeps it), the pure `admit`, `Settings` (every number, `for_host` has the defaults; `limited` lays the owner's `Limits` over them: `max_machines`, refused past it with `Refusal::Machines`, `idle_after`, the floor as a share of memory, which lowers the cap, `ci_bytes`, the memory CI's machines hold at once with each one's overhead (`admit` counts them; `tests/ci_memory.rs`), and `ci_cpus`, the CPUs they take together, and the disk limits: `disk_hold_bytes`, `disk_evict_bytes` and `job_disk_bytes`, each job's disk (`tests/disk.rs`); `check_disk` checks them against the state folder's file system, refusing a hold or a job's disk the owner set that it cannot hold and saying `DiskShort` of a container host whose hold and one job's disk do not fit it together; `for_headless` those of a Linux host nobody sits at, which reserves what the host needs, `HEADLESS_HOST_USE`, and caps the sizes so that each machine fits with its overhead; `on_fast_cores` gives CI only the fast kind of core of a host whose cores differ; `one_at_a_time` those of a trusted host, which runs one machine on every core it has), `Size` and its labels, the `Host` trait with `HostReading`, `Battery`, what a host's page shows of its battery, and `FakeHost`, the host tests script (`read_takes` lets a read take time before and after the reading gets its time; `tests/fake_host.rs`). Tests: `admission`, `assigned`, `headless`, `limits` (the rule at 4, 7.75, 8, 16 and 32 GiB and its boundaries), `levels`, `making_room`, `races`, `properties`, `turns`. | | `crates/clock` (`cf-clock`) | `now`: the two clocks the budget counts time by, read from the host (Linux: `CLOCK_MONOTONIC` and `CLOCK_BOOTTIME`; macOS: `CLOCK_UPTIME_RAW` and `CLOCK_MONOTONIC_RAW`). Both host crates' `Host::now`. | | `crates/host-macos` (`cf-host-macos`) | `MacHost` reads macOS behind `cf_budget::Host` (page counts, pressure, swap, idle time, thermal, power, disk; what the volume frees on demand, Time Machine's local snapshots and purgeable caches, is its capacity for important use beyond the free space, asked at most every `IMPORTANT_EVERY`, 30 s, for the budget's disk hold and never for an eviction); `Raw` and `reading` are the arithmetic: available memory is what is ready now (free, file cache, purgeable) and, while the Mac is calm (pressure normal, `kern.memorystatus_level` at least `CALM_LEVEL`, the compressor under `HEAVY_COMPRESSOR_SHARE` of memory), what compressing its inactive anonymous pages would free, never past the memorystatus level's share (`tests/reclaim.rs`). For the console's page: `boot_time`, `model` (chip and model, `model_name`), `battery` (what `pmset -g batt` prints, read by `battery_in`, as `cf_budget::Battery`), `host_name`, `load`, `cpu_percent`. `performance_cores` (`hw.perflevel0.logicalcpu` of a chip with two kinds of core, for the budget's CPUs). `program_of` says which program a process runs (the daemon's `is_ours`, `Holder`). `backup`: `exclude_from_backup` marks a folder the Runner made as skipped by Time Machine (the sticky exclusion `tmutil addexclusion` sets, an extended attribute), `exclude_with` is the decision behind the `Attributes` edge; the daemon, `enroll`, `install` and `image install` mark the state folder (the daemon's `Mac::keep_state`, a failure is a logged line, never fatal), and a cache folder to come goes through the same call when it is made. `examples/host-probe.rs` prints a Mac's readings. | | `crates/host-linux` (`cf-host-linux`) | `LinuxHost` reads Linux behind `cf_budget::Host`: MemAvailable, swap, swap-outs (`swapouts_of`: `pswpout`, but with zram in `/proc/swaps` only what is in use on swap that is not zram, since a page in zram stays in memory and PSI says when memory is short; `pswpout` again when `/proc/swaps` cannot be read), PSI (`/proc/pressure/memory`, Normal when the kernel has none), the thermal zones by their own trip points, `/sys/class/power_supply` (a host with no battery runs on the mains; a device's own battery, such as a mouse's, is left out unread), the state folder's disk by `statvfs`; idle time is none on a headless host. Every reader takes a root folder, so tests run against trees of files on any OS. `parse` reads each file, never more than `MAX_FILE_BYTES` (`counter` is one value of a `name value` file, also a cgroup's); `Raw` and `reading` are the arithmetic, with what each field means in the doc comment of `reading`. `fast_cpus` (the kind of core, by `midr_el1`, of the highest `cpu_capacity`, for `Settings::for_headless`). For the console's page: `boot_session` (`boot_id`), `boot_time`, `model` (device tree, else DMI), `battery`, `host_name`, `load`, `cpu_percent`. `process` and `program_of` say which program a process runs and when it started, read twice around the program so a number that went to another process in between names none; `started_at` moves with a step of the wall clock, `started_ticks` with `boot_session` does not. Tests: `fixtures.rs` (an RK3588 board's files, broken in every way), `parse.rs` (with properties), `real.rs` (Linux only: this machine against `free`, `df`, `getconf`). `examples/linux-probe.rs` prints a Linux host's readings and the headless budget. | | `crates/lifecycle` (`cf-lifecycle`) | `Slot`: the state machine of one slot and its job (events in, effects out, watchdogs, `saved` and `restore` for a daemon restart; `on` answers every phase and event with no catch-all arm, keeps a word of a job in every phase, and ends a slot as `no-job` only on `Event::Unregistered`, GitHub's 204; `Asking` is the phase of a slot that asks for its registration's removal, entered at the idle limit or on `Retire`; `Slot::hear` replays GitHub's stored word on a saved slot), and `StartBreaker` (`admit` before a claim, `holds` for the budget's hold). `tests/model.rs` searches every state it can reach. | | `crates/scaleset` (`cf-scaleset`) | The scale set protocol: `ScaleSetClient` (scale sets, sessions, one-job JIT configs, `generate_plain_jit_config` for one outside any scale set, removal of an Actions runner by id and by name (`remove_runner_named`); every administrator's call is the console's `github/scaleset/*`, which holds the Runners App's key and the service's admin token, except on a host that still holds the key (`with_key`, `holds_key`; `key.rs` `KeyAdmin`: its own registration token and admin token, the admin token kept until 120 s before it ends and replaced once on 401, and only the sets named `--…` listed), and on a host whose console is older than those calls (a 404 without a code: `KeyAdmin` with the console's `github/registration` token, `Registrar::Console`, until the Runner starts again; a Runner never needs a console newer than the one that runs; `tests/older_console.rs`); the session keeps the `serviceUrl` the console gave), `runner_busy`, the REST view of whether an Actions runner holds a job, `MessageSession` (long poll, ack, acquire), `listen` with the `Scaler` trait (handle, then ack; duplicates acked only; shutdown interrupts only the wait), `Backoff` and `supervise`. A token goes only to the API's origin and to hosts of `ACTIONS_HOST`: an address the service hands back that is elsewhere is refused. A registration whose answer was lost, or that names no runner, is removed again. Every status and malformed body is an `Error`. `examples/github_probe.rs` is the real check against a throwaway repository (prints no token). Tests in `tests/`, run against `cf-fake-github`; `github_rules.rs` holds the fake to what GitHub was seen to do (`docs/github-protocol.md`). | -| `crates/daemon` (`cf-daemon`) | The service. `holds.rs`: holds set on the machine itself, one file each in `/holds` (`set`, `lift`, `read`, which holds on a damaged file or an unreadable folder), and `/status.json` (`LocalStatus`: the holds the Runner saw, its machines and the ones coming, written on a change and every `STATUS_EVERY`); `Service::look_at_holds`, each tick, takes no new job while any hold or the console's pause is set and reports them as `status.holds`. `machines.rs`: `Machines` joins the budget, the lifecycle of each slot and the vm-hosts: its scale sets share the room their polls offer GitHub (`offer_room`: a free slot one poll offers no other does until that poll ends, `OFFER_STANDS` at most, and sets with jobs queued, `set_demand`, go before those without), so GitHub never assigns more jobs than it can start; a job GitHub assigned is counted (`assign`), started when memory allows (a machine counts with what its vm-host read it holds of the Mac's memory, `count_held`, its job's peak only when that is not known; the log says once when it is not, or is more than the machine is counted with, `Budget::host_bytes`), registered just in time, watched every tick, and ended once with an `EndKind` (`ends.rs`); `restore` adopts what a Runner before left from the records (`records.rs`, `DiskStore`; a record stores facts, `up`, `ran_job`, `completed`, `unregistered`, and carries the `RULES` it was written under; `Record::agrees` says its facts agree, and `Record::upgrade` is the one place that reads an older Runner's record: its beliefs cleared, GitHub's stored word replayed; a current record that disagrees is an error) and the vm-hosts that still answer, in the same start of the Mac (`kern.bootsessionuuid`, not the boot time, which moves with the clock); one that is silent or wrote nothing yet is killed, and so is what has no record. A tick looks at the vm-hosts without the machines' lock, so a vm-host that hangs keeps no listener waiting. What the polls in flight may still offer GitHub is kept (`Offer`): every answer a listener got since all bells last rang, since many scale sets of one size each ask at their own moment. The last slot taken rings their bells at once, room that went or came back with the Mac's memory at a look, at most every 5 s, so no poll in flight goes on offering a slot that is gone, or none while there is one. An open Mac whose other programs leave no memory for a size it serves says so in the log and is closed for `memory` in the console's report (`Snapshot::short_of`); one that would carry the size without its own machines is full, not short. `Snapshot::rooms` is what one more machine of each size meets, for the report's `sizes` and each scale set's capacity: how many fit, the budget's word when none does, `never` for a size the settings never run, and for memory what it needs and lacks. `making.rs`: `Making`, room made for a larger size. A job that waits for the room the Mac's own machines hold, all of them smaller, while the empty Mac would run it, has every other size held back until it starts, for `MAKE_ROOM_LIMIT` (30 min) at most and then none for as long; its scale set offers a place in line while GitHub has the job back, for `TAKEN_BACK_WAIT` (2 min); an empty Mac whose other programs hold the room gives up after `NO_ROOM_WAIT`. A size whose polls found no room for `STARVED_AFTER` (10 min) is offered a place before any of its jobs is assigned; a size the empty Mac cannot run never is. A headless host, one of a pool, makes room and offers a place only for the size the console's answer names it (`makeRoomFor`, `Making::name`, `Machines::make_room_for`), at once. `vms.rs`: `Vms`, the seam to `codefusion-runner vm`, and `VmHosts`, the real one. It writes each vm-host's process and start time to `run/.pid` (0600) before it hands it the job, so the Runner that starts next can end one that hangs before its first state. A signal goes only to a child whose exit was not collected, to a process that answers on the job's socket, or to one that runs the Runner's binary, is not the Runner and started no later than the time written down for it (`is_ours`, through `Platform::process`): a number that went to a later process is sent nothing. `service.rs`: `Service` (`discover` repositories and labels and, on a host that still holds the App's key (`HeldKey`), asks the console's scale set calls about each served repository in turn (the next after a refusal of one, none after a console that cannot be reached or has no such calls): once one answered, `HeldKey::take_out` and a start again, else the key serves and the report's `appKeyKept` says why, one supervised listener per scale set, `sweep` of registrations nothing stands behind, `report` to the console with the job events of the outbox and how long jobs handed here waited while nothing ran, with a WARN after `STALL_WARN_AFTER`; the sweep is the one place that removes a registration: those of slots in `Asking`, and of machines that ended while theirs stands, after `RESULT_WAIT` for one whose guest came up with no word of a job; each answer goes to `Machines::answered(runner, Answer)`: `Removed` (204) is no job, `Gone` (404) under a guest that came up is a job over, `Refused` (422) a job that runs, a job implied so is stored as inferred, `discover` and `look_at_queue` run in a task beside `run`'s loop, so a slow GitHub holds up no report and no stop; `look_at_queue`, about once a minute, serves the label sets of queued and running jobs that the default branches' workflows do not name, under `queued.rs`'s `Queued::select`: each for `queued_keep` (30 min) after a job of it was last seen, `queued_bounds` (8 a repository, 32 a host) the newest first, none of a repository it does not serve, none of a fork's pull request on a public repository, and none dropped while that repository's queue cannot be read (for up to `LONGEST_WAIT`, 24 h); a change lists the scale sets of only the repositories whose sets changed; `survey` for `check`, `retire` for `uninstall`, `run`). Each look at the queue also finds the jobs GitHub holds back from a scale set with room (`held.rs`, pure: queued two minutes while every poll of the set offered room beyond GitHub's assigned count for as long, `SetState::room_since`), names them in the report's `heldJobs`, and gives the set one Actions runner of its own at a time, one in five minutes at most and none while a queued job of the repository asks for only some of its labels (`held::narrower_waits`), in a slot the budget gives: registered outside the scale set (`SetRegistry::own`, scale set number `held::OWN`, 0) with exactly its labels and started by `scaler::start_runner`, as a scale set's are; it is retired once its set's queue holds no job it could take, or after five minutes. What the Runner believes about each machine and its job, which evidence moves it and what is written before what is done, is `docs/job-state.md`. `scaler.rs`: `Scaler` (reserve before acquire, release for jobs taken elsewhere, converge on assigned jobs, each place in line with the turn of the assigned job it stands for (`Slots::set_turn`), offers taken overdue first, then main and deploy jobs; `capacity` offers a paused or held Mac's polls only what runs and what was promised (`Slots::held`), and no room to a job in line `PLACE_LIMIT` (10 min) after it was first assigned here, taken back and handed here again or not (one away as long is a re-run, and new), so GitHub's take-back sends it to another machine, retire surplus after `IDLE_GRACE` (`Hosts::retire`: idle machines ask for their registration's removal, which the sweep does); before a session opens, `reconcile` asks GitHub's REST view of each Actions runner it believes waits, since messages go with a closed session, and a busy one took a job (`Answer::Busy`); a session opens only at start, after GitHub forgot one, after a listener failed, or after a restart), `Roster` (an Actions runner belongs to a repository's scale set: GitHub numbers scale sets within a repository, so the number alone names none), `turns.rs`: `turn_of`, whether a job is the main branch's or a deploy, from its workflow's ref, name and event. `names` (runner and scale set names, `ServePolicy`; a scale set's name ends in a hash of its repository and labels, so two repositories that ask for the same labels each get their own). `update.rs`: `SettingsFile` (what the Runner's configuration file says now against what it started with, `SettingsLook`; `change` writes the console's `settings` command, read by `SettingsChange::from_args`, which names each of the cf-runners manager's settings the Runner does not take; `Service::look_at_settings` takes a new one up as an update does, waiting in `drained`, the same wait, for no machine to stand, then starting again), `SelfUpdate` (`build` takes the commit `update-to` names, or none for main's head; `updates_from`: `UpdateFrom::Checkout` by default, `Releases` lists no plain `update`, refuses one with `NO_HEAD` and shows `fetching its release` in place of `building` (`takes_head`, `first_step`); `end` stops a build or a wait when the daemon ends), `commit_from_args`, the console's `update-to` arguments (`{"commit"}` alone, `is_commit`: 7 to 40 lowercase hex characters), `Pending` in `update.json`, `at_start` (`Trying`, `GoBack` after `MAX_STARTS` starts that did not stay up for `STAYS_UP`, `Failed`, also when a program before could not be put back: `Pending::failed`); the service builds while jobs run, then holds new jobs until no machine stands and no slot promised to GitHub is left (`Machines::coming`), installs and ends (`update`, `restart`; one at a time, the result said once by the Runner that starts next). `platform/`: `host_service`, the service on a real host, which the binary and `cargo xtask it` both build, over a `Platform` (`This` is this build's): what differs between systems and nothing else: the host's reader and the budget's settings (`Settings::for_host` on the performance cores, `on_fast_cores` with `cf_host_macos::performance_cores`; on Linux `for_headless` with the fast cores), the boot session, host name, model and what the console's page shows (`extras`), `keep_state` (Time Machine; nothing on Linux), `fit` (the virtualization entitlement; on Linux `codefusion-runner vmm` says the build carries cloud-hypervisor), `process` (program and start, for `is_ours`; on Linux `cf_host_linux::process`), `program` (the program alone, for a kill's wait), `vm_hosts` and `SERVICE_LOG`. `mac.rs` is macOS, `linux.rs` Linux, where each vm-host starts in a systemd scope of its own (`VmHosts::in_scopes`, `systemd-run --user --scope -p Delegate=yes`, `codefusion-runner-.scope`), whose delegated cgroup it claims for its machines, and `user_bus_env` finds the user's systemd under `sudo -iu`, and not another user's that `su` left in the environment (`this_user` names the user from the user database). `tests/scopes.rs` (Linux, no harness) starts a stand-in vm-host so and checks it claims the scope's cgroup. `test_run/`: the console's `test-run` command (`TestRun::from_args`, its arguments checked, the run's `url` on the Runner's own console; `admit`, an urgent claim of the budget's as `test-run` that goes before the CI jobs in line, busy while the host takes no new job, a test run is on it or a Docker run finds the Docker jobs full; `wait_turn` for `waitS`, withdrawn as soon as the console's answer no longer carries the command), started by `Machines::start_test_run` as a machine with no Actions runner: its record is `unregistered` from the start with `test_run` set, its job starts when its guest is up and ends with the machine, and `Service::test_run` answers `started tr-…`, `busy: …` or `never: …`, later for a run that waits. `agent.mjs` is the workload (`workload()`: `node --input-type=module -e `, env `CF_RUN_URL`, `CF_RUN_TOKEN`, `CF_RUN_TIMEOUT_S`), the agent that fetches the bundles, runs the suite in its own process group, spools its output decoded as UTF-8 (U+FFFD for what is not) and sends it to the console; `agent.test.mjs` its `node --test` tests against a fake console (CI's Linux job). Tests: `tests/test_run.rs`, and in `tests/service.rs` the command end to end. `docker_check.rs`: whether Docker starts in this host's job machines (`DockerCheck`: `of_end` reads the check machine's end and its console log's "Docker sandbox ended" line, `due` at the start and every `RECHECK_AFTER` (6 h) while it fails, `None` from `of_end` for a machine that ended for something else (slept, evicted, stopped), checked again after `AGAIN_AFTER_NO_ANSWER` (10 min, `Machines::docker_check_due`), `refusal` while it failed; a failure is kept across starts by the `Store`, `docker_failure` and `keep_docker_failure`, in `DiskStore`'s `docker-check.failed`, a pass is not); `Machines::start_docker_probe` boots the check's machine in a claimed slot (no Actions runner, `Record::probe`, nothing reported), `Service::look_at_docker` on every tick, and a failed check drops the `cf-docker` sets (`size_here`) and answers a test run with Docker `never:`; tests `tests/docker_check.rs` and in `tests/service.rs`. `log.rs`: `Lines`, the log file that is rolled over. `testing` has the fakes, and `CrashingStore`, records at which a test makes the daemon die between any two writes. Tests: `machines.rs` (every end kind, a crash at every step), `records.rs` (every record main and earlier builds could have written, read by `Record::upgrade`), `interleavings.rs` (the property test), `schedules.rs` (the property test of the job facts against a GitHub that keeps the truth: crashes at any write, every answer to a removal, lost and late messages, records left in older formats; with pinned schedules), `offers.rs` (the property test of what several listeners of one size offer), `making_room.rs` (the mini's cf-6g job against the cf-2g ones that keep coming), `calm.rs` (the calm minute against time that passes within a look, and against everything that is really wrong), `service.rs` (GitHub's and the console's failures; the label sets of queued jobs: served, kept 30 min, bounded, refused, unreadable, 304, a fork's on a public repository; the property test that no job gets two machines, a session opens only for a cause, a job that waits while a machine fits moves at GitHub's next look, and every job that ran is heard, start and end once each, however many messages went with a session, with a stale or failing REST view, a crash between any two writes, five minutes that take an assignment back, removals that fail until GitHub recovers, and an uninstall in the middle; machines that boot slowly, so a start or an end comes before the Runner saw one up; no VM whose job runs is stopped, and every single write's facts agree (`CrashingStore::disagreements`); a second property aims the same checks at surplus removals), `scaler.rs`, `listening.rs`, `docker.rs` (a Docker job's daemon flag, the cap of `docker_max_jobs` in `Machines`, and `ServePolicy::serves_docker`: listed and private only). A trusted host (`ServePolicy::trusted`) makes no scale set: `Service::ask_trusted` asks for one Actions runner with the label `trusted` at a time (`trusted_every`, 15 s), in the largest machine it runs, whose `JobRequest::trusted` carries the rules; a 409 with `Retry-After` (the console's `no_trusted_job`) is a wait. It takes no test run, boots no Docker check and reports `class: trusted`. | +| `crates/daemon` (`cf-daemon`) | The service. `holds.rs`: holds set on the machine itself, one file each in `/holds` (`set`, `lift`, `read`, which holds on a damaged file or an unreadable folder), and `/status.json` (`LocalStatus`: the holds the Runner saw, its machines and the ones coming, written on a change and every `STATUS_EVERY`); `Service::look_at_holds`, each tick, takes no new job while any hold or the console's pause is set and reports them as `status.holds`. `machines.rs`: `Machines` joins the budget, the lifecycle of each slot and the vm-hosts: its scale sets share the room their polls offer GitHub (`offer_room`: a free slot one poll offers no other does until that poll ends, `OFFER_STANDS` at most, and sets with jobs queued, `set_demand`, go before those without), so GitHub never assigns more jobs than it can start; a job GitHub assigned is counted (`assign`), started when memory allows (a machine counts with what its vm-host read it holds of the Mac's memory, `count_held`, its job's peak only when that is not known; the log says once when it is not, or is more than the machine is counted with, `Budget::host_bytes`), registered just in time, watched every tick, and ended once with an `EndKind` (`ends.rs`); `restore` adopts what a Runner before left from the records (`records.rs`, `DiskStore`; a record stores facts, `up`, `ran_job`, `completed`, `unregistered`, and carries the `RULES` it was written under; `Record::agrees` says its facts agree, and `Record::upgrade` is the one place that reads an older Runner's record: its beliefs cleared, GitHub's stored word replayed; a current record that disagrees is an error) and the vm-hosts that still answer, in the same start of the Mac (`kern.bootsessionuuid`, not the boot time, which moves with the clock); one that is silent or wrote nothing yet is killed, and so is what has no record. A tick looks at the vm-hosts without the machines' lock, so a vm-host that hangs keeps no listener waiting. What the polls in flight may still offer GitHub is kept (`Offer`): every answer a listener got since all bells last rang, since many scale sets of one size each ask at their own moment. The last slot taken rings their bells at once, room that went or came back with the Mac's memory at a look, at most every 5 s, so no poll in flight goes on offering a slot that is gone, or none while there is one. An open Mac whose other programs leave no memory for a size it serves says so in the log and is closed for `memory` in the console's report (`Snapshot::short_of`); one that would carry the size without its own machines is full, not short. `Snapshot::rooms` is what one more machine of each size meets, for the report's `sizes` and each scale set's capacity: how many fit, the budget's word when none does, `never` for a size the settings never run, and for memory what it needs and lacks. `making.rs`: `Making`, room made for a larger size. A job that waits for the room the Mac's own machines hold, all of them smaller, while the empty Mac would run it, has every other size held back until it starts, for `MAKE_ROOM_LIMIT` (30 min) at most and then none for as long; its scale set offers a place in line while GitHub has the job back, for `TAKEN_BACK_WAIT` (2 min); an empty Mac whose other programs hold the room gives up after `NO_ROOM_WAIT`. A size whose polls found no room for `STARVED_AFTER` (10 min) is offered a place before any of its jobs is assigned; a size the empty Mac cannot run never is. A headless host, one of a pool, makes room and offers a place only for the size the console's answer names it (`makeRoomFor`, `Making::name`, `Machines::make_room_for`), at once. `vms.rs`: `Vms`, the seam to `codefusion-runner vm`, and `VmHosts`, the real one. It writes each vm-host's process and start time to `run/.pid` (0600) before it hands it the job, so the Runner that starts next can end one that hangs before its first state. A signal goes only to a child whose exit was not collected, to a process that answers on the job's socket, or to one that runs the Runner's binary, is not the Runner and started no later than the time written down for it (`is_ours`, through `Platform::process`): a number that went to a later process is sent nothing. `service.rs`: `Service` (`discover` repositories and labels and, on a host that still holds the App's key (`HeldKey`), asks the console's scale set calls about each served repository in turn (the next after a refusal of one, none after a console that cannot be reached or has no such calls): once one answered, `HeldKey::take_out` and a start again, else the key serves and the report's `appKeyKept` says why, one supervised listener per scale set, `sweep` of registrations nothing stands behind, `report` to the console with the job events of the outbox and how long jobs handed here waited while nothing ran, with a WARN after `STALL_WARN_AFTER`; the sweep is the one place that removes a registration: those of slots in `Asking`, and of machines that ended while theirs stands, after `RESULT_WAIT` for one whose guest came up with no word of a job; each answer goes to `Machines::answered(runner, Answer)`: `Removed` (204) is no job, `Gone` (404) under a guest that came up is a job over, `Refused` (422) a job that runs, a job implied so is stored as inferred, `discover` and `look_at_queue` run in a task beside `run`'s loop, so a slow GitHub holds up no report and no stop; `look_at_queue`, about once a minute, serves the label sets of queued and running jobs that the default branches' workflows do not name, under `queued.rs`'s `Queued::select`: each for `queued_keep` (30 min) after a job of it was last seen, `queued_bounds` (8 a repository, 32 a host) the newest first, none of a repository it does not serve, none of a fork's pull request on a public repository, and none dropped while that repository's queue cannot be read (for up to `LONGEST_WAIT`, 24 h); a change lists the scale sets of only the repositories whose sets changed; `survey` for `check`, `retire` for `uninstall`, `run`). Each look at the queue also finds the jobs GitHub holds back from a scale set with room (`held.rs`, pure: queued two minutes while every poll of the set offered room beyond GitHub's assigned count for as long, `SetState::room_since`), names them in the report's `heldJobs`, and gives the set one Actions runner of its own at a time, one in five minutes at most and none while a queued job of the repository asks for only some of its labels (`held::narrower_waits`), in a slot the budget gives: registered outside the scale set (`SetRegistry::own`, scale set number `held::OWN`, 0) with exactly its labels and started by `scaler::start_runner`, as a scale set's are; it is retired once its set's queue holds no job it could take, or after five minutes. What the Runner believes about each machine and its job, which evidence moves it and what is written before what is done, is `docs/job-state.md`. `scaler.rs`: `Scaler` (reserve before acquire, release for jobs taken elsewhere, converge on assigned jobs, each place in line with the turn of the assigned job it stands for (`Slots::set_turn`), offers taken overdue first, then main and deploy jobs; `capacity` offers a paused or held Mac's polls only what runs and what was promised (`Slots::held`), and no room to a job in line `PLACE_LIMIT` (10 min) after it was first assigned here, taken back and handed here again or not (one away as long is a re-run, and new), so GitHub's take-back sends it to another machine, retire surplus after `IDLE_GRACE` (`Hosts::retire`: idle machines ask for their registration's removal, which the sweep does); before a session opens, `reconcile` asks GitHub's REST view of each Actions runner it believes waits, since messages go with a closed session, and a busy one took a job (`Answer::Busy`); a session opens only at start, after GitHub forgot one, after a listener failed, or after a restart), `Roster` (an Actions runner belongs to a repository's scale set: GitHub numbers scale sets within a repository, so the number alone names none), `turns.rs`: `turn_of`, whether a job is the main branch's or a deploy, from its workflow's ref, name and event. `names` (runner and scale set names, `ServePolicy`; a scale set's name ends in a hash of its repository and labels, so two repositories that ask for the same labels each get their own). `update.rs`: `SettingsFile` (what the Runner's configuration file says now against what it started with, `SettingsLook`; `change` writes the console's `settings` command, read by `SettingsChange::from_args`, which names each of the cf-runners manager's settings the Runner does not take; `Service::look_at_settings` takes a new one up as an update does, waiting in `drained`, the same wait, for no machine to stand, then starting again), `SelfUpdate` (`build` takes the commit `update-to` names, or none for main's head; `updates_from`: `UpdateFrom::Checkout` by default, `Releases` lists no plain `update`, refuses one with `NO_HEAD` and shows `fetching its release` in place of `building` (`takes_head`, `first_step`); `end` stops a build or a wait when the daemon ends), `commit_from_args`, the console's `update-to` arguments (`{"commit"}` alone, `is_commit`: 7 to 40 lowercase hex characters), `Pending` in `update.json`, `at_start` (`Trying`, `GoBack` after `MAX_STARTS` starts that did not stay up for `STAYS_UP`, `Failed`, also when a program before could not be put back: `Pending::failed`); the service builds while jobs run, then holds new jobs until no machine stands and no slot promised to GitHub is left (`Machines::coming`), installs and ends (`update`, `restart`; one at a time, the result said once by the Runner that starts next). `platform/`: `host_service`, the service on a real host, which the binary and `cargo xtask it` both build, over a `Platform` (`This` is this build's): what differs between systems and nothing else: the host's reader and the budget's settings (`Settings::for_host` on the performance cores, `on_fast_cores` with `cf_host_macos::performance_cores`; on Linux `for_headless` with the fast cores; `budget_settings` lays the owner's `Limits` over them and checks them against the state folder's file system, `disk_bytes`), the boot session, host name, model and what the console's page shows (`extras`), `keep_state` (Time Machine; nothing on Linux), `fit` (the virtualization entitlement; on Linux `codefusion-runner vmm` says the build carries cloud-hypervisor), `process` (program and start, for `is_ours`; on Linux `cf_host_linux::process`), `program` (the program alone, for a kill's wait), `vm_hosts` and `SERVICE_LOG`. `mac.rs` is macOS, `linux.rs` Linux, where each vm-host starts in a systemd scope of its own (`VmHosts::in_scopes`, `systemd-run --user --scope -p Delegate=yes`, `codefusion-runner-.scope`), whose delegated cgroup it claims for its machines, and `user_bus_env` finds the user's systemd under `sudo -iu`, and not another user's that `su` left in the environment (`this_user` names the user from the user database). `tests/scopes.rs` (Linux, no harness) starts a stand-in vm-host so and checks it claims the scope's cgroup. `test_run/`: the console's `test-run` command (`TestRun::from_args`, its arguments checked, the run's `url` on the Runner's own console; `admit`, an urgent claim of the budget's as `test-run` that goes before the CI jobs in line, busy while the host takes no new job, a test run is on it or a Docker run finds the Docker jobs full; `wait_turn` for `waitS`, withdrawn as soon as the console's answer no longer carries the command), started by `Machines::start_test_run` as a machine with no Actions runner: its record is `unregistered` from the start with `test_run` set, its job starts when its guest is up and ends with the machine, and `Service::test_run` answers `started tr-…`, `busy: …` or `never: …`, later for a run that waits. `agent.mjs` is the workload (`workload()`: `node --input-type=module -e `, env `CF_RUN_URL`, `CF_RUN_TOKEN`, `CF_RUN_TIMEOUT_S`), the agent that fetches the bundles, runs the suite in its own process group, spools its output decoded as UTF-8 (U+FFFD for what is not) and sends it to the console; `agent.test.mjs` its `node --test` tests against a fake console (CI's Linux job). Tests: `tests/test_run.rs`, and in `tests/service.rs` the command end to end. `docker_check.rs`: whether Docker starts in this host's job machines (`DockerCheck`: `of_end` reads the check machine's end and its console log's "Docker sandbox ended" line, `due` at the start and every `RECHECK_AFTER` (6 h) while it fails, `None` from `of_end` for a machine that ended for something else (slept, evicted, stopped), checked again after `AGAIN_AFTER_NO_ANSWER` (10 min, `Machines::docker_check_due`), `refusal` while it failed; a failure is kept across starts by the `Store`, `docker_failure` and `keep_docker_failure`, in `DiskStore`'s `docker-check.failed`, a pass is not); `Machines::start_docker_probe` boots the check's machine in a claimed slot (no Actions runner, `Record::probe`, nothing reported), `Service::look_at_docker` on every tick, and a failed check drops the `cf-docker` sets (`size_here`) and answers a test run with Docker `never:`; tests `tests/docker_check.rs` and in `tests/service.rs`. `log.rs`: `Lines`, the log file that is rolled over. `testing` has the fakes, and `CrashingStore`, records at which a test makes the daemon die between any two writes. Tests: `machines.rs` (every end kind, a crash at every step), `records.rs` (every record main and earlier builds could have written, read by `Record::upgrade`), `interleavings.rs` (the property test), `schedules.rs` (the property test of the job facts against a GitHub that keeps the truth: crashes at any write, every answer to a removal, lost and late messages, records left in older formats; with pinned schedules), `offers.rs` (the property test of what several listeners of one size offer), `making_room.rs` (the mini's cf-6g job against the cf-2g ones that keep coming), `calm.rs` (the calm minute against time that passes within a look, and against everything that is really wrong), `service.rs` (GitHub's and the console's failures; the label sets of queued jobs: served, kept 30 min, bounded, refused, unreadable, 304, a fork's on a public repository; the property test that no job gets two machines, a session opens only for a cause, a job that waits while a machine fits moves at GitHub's next look, and every job that ran is heard, start and end once each, however many messages went with a session, with a stale or failing REST view, a crash between any two writes, five minutes that take an assignment back, removals that fail until GitHub recovers, and an uninstall in the middle; machines that boot slowly, so a start or an end comes before the Runner saw one up; no VM whose job runs is stopped, and every single write's facts agree (`CrashingStore::disagreements`); a second property aims the same checks at surplus removals), `scaler.rs`, `listening.rs`, `docker.rs` (a Docker job's daemon flag, the cap of `docker_max_jobs` in `Machines`, and `ServePolicy::serves_docker`: listed and private only). A trusted host (`ServePolicy::trusted`) makes no scale set: `Service::ask_trusted` asks for one Actions runner with the label `trusted` at a time (`trusted_every`, 15 s), in the largest machine it runs, whose `JobRequest::trusted` carries the rules; a 409 with `Retry-After` (the console's `no_trusted_job`) is a wait. It takes no test run, boots no Docker check and reports `class: trusted`. | | `crates/ghapp` (`cf-ghapp`) | `release.rs`: `Releases`, a public repository's releases read with no token (`by_tag`, `page`, `branch_head`, `compare`), and `download`, a release's file streamed into a `Sink` and hashed as it is written, followed across GitHub's redirect to its storage with no token, begun again from the first byte after a break, a 429, a used-up rate limit or a 5xx, ended at once by a full disk (`Error::Local`), a size over its limit or the stop flag, also while a connection stalls or a retry waits (`sleep_unless`) (`tests/release.rs`). `http.rs`: `retry_after` reads `Retry-After`, else a used-up rate limit's `x-ratelimit-reset`; `RetryPolicy::wait_after` is the one rule of when and how long both `Http::send` and a download try again; `file_client` follows a release file's redirect to another host and never carries a token. The GitHub App client. Its tokens come from a `TokenSource` (`source.rs`), the console's (`cf_console_client::ConsoleTokens`), through `GitHubApp::with_source`: no machine holds the App's key, a registration token or the Actions service's admin token, except one that still holds the key until the console's scale set calls answered it (`KeyTokens`, `key.rs`: the App JWT of `jwt.rs`, installation tokens and `registration_token`; `AppCredentials::load` in `credentials.rs`, `app.json` and `app.pem`, which must be 0600; tests in `tests/key.rs`); a call about a repository passes it, so a source whose installation IDs differ finds the one that covers it; `Monotonic` is the clock a token's lifetime is counted on. `rest.rs` is the REST plumbing both share (a 401 tells the source and tries once more). The clock skew is learned from the `Date` header (`clock.rs`). Repositories, workflow files (the caller names the largest file to read), repository variables, `waiting_jobs` (the jobs of a repository's queued and running runs of the label sets served, `cf_workflows::LabelSet::served`, from the REST view, each saying whether its run is a fork's, under bounds: `WAITING_RUNS` of the newest `RUNS_LISTED` runs, queued ones before those under way and each the oldest first, `JOBS_PAGE` jobs of one, `MAX_RUN_SETS` distinct sets of one run; the listings asked for again with the ETag of their last answer, so a 304 reuses it, and each run's jobs asked for again every look with theirs (a run's `updated_at` stays put while its jobs start and end), and `forget_queues` drops what repositories no longer served last read as; each set names its queued jobs, the oldest first), `generate_jit_config` (an Actions runner for one job outside any scale set, with exactly the labels given, sent once); `Http::new(timeout, retry)` with retries (429 and 5xx with jitter, `Retry-After`), HTTPS only (or this machine itself), redirects only to the same origin, at most `MAX_BODY_BYTES` of an answer and 100 pages of a listing. No error and no `Debug` carries an answer's text. Tests in `tests/app.rs`, run against `cf-fake-github`. | | `crates/workflows` (`cf-workflows`) | `self_hosted` / `self_hosted_with`: the label sets a repository's workflows ask for (port of the Go `workflows` package, tests in `tests/labels.rs`), `size_gb` for the `cf-2g/4g/6g/8g` labels, `self-hosted` = 4 GB, `DOCKER_LABEL` (`cf-docker`, served alone or with a size), `is_host_label` and `LabelSet::host_labels` (`cf-macos` and the like, served only by hosts whose config `labels` has them, and a host whose `only_labels` names some of them serves only the sets that ask for one: `ServePolicy::serves_labels` and `labels_refusal` in `crates/daemon/src/names.rs`), the `vars.*` expressions evaluated with the repository's variables by `expression.rs`, a reader of GitHub's expressions over variables and literals: `||`, parentheses, `format` (`{0}`, `{{`/`}}` escapes), `fromJSON` and `toJSON`, nested at most `MAX_DEPTH` deep, no value over `MAX_VALUE_BYTES` (a JSON list under `fromJSON`, with no `toJSON` anywhere in the expression, is several labels; what it cannot read stands for the pool label), `LabelSet::served`, the rules every set to serve keeps, a workflow's or a queued job's (one label of ours, no other platform, at most `MAX_SET_LABELS` labels of at most `MAX_LABEL_BYTES`, letters, digits, `-`, `_` and `.` only), and `MAX_FILE_BYTES`, the largest workflow file read. `LabelSet::served` refuses every set with `TRUSTED_LABEL` in any spelling GitHub's case-blind match takes for it (`is_trusted_label`: `TRUSTED`, `truſted`), from a list or a variable: no shared host takes a trusted job. | | `crates/secret` (`cf-secret`) | `Secret`: a value whose `Debug` prints `Secret(..)`. Every token and key is wrapped in it. `replace_private`: a file for this user alone (0600), put in place in one step; `replace_private_as` with a mode of the user's own bits (0500 for a program). `private_dir`: a folder for this user alone (0700). | diff --git a/docs/containers.md b/docs/containers.md index 3c6b6dc..2299243 100644 --- a/docs/containers.md +++ b/docs/containers.md @@ -26,9 +26,10 @@ daemon, the budget. `cf-vmm-oci` stands where cloud-hypervisor stands, and a "ma - held to its size's memory with no swap, its CPUs and 4096 processes, so a job that runs out ends `oom`; an engine that answers the create with any warning (a limit it could not set) gets its container removed and the job refused, and the supervisor refuses to start a job whose control group reads `max` for `memory.max` or `pids.max`; -- held to the size's scratch disk: the vm-host reads the container's writable layer (`SizeRw`) every 2 seconds, ends - the job when it holds more, and reports its peak as the job's disk peak; where the engine's storage can bound a - layer (overlay on xfs, an engine with root) the create also asks for `StorageOpt.size`; +- held to its disk, `job_disk_gb` (40 GB unless the host's settings say otherwise): the vm-host reads the + container's writable layer (`SizeRw`) every 2 seconds, ends the job when it holds more, and reports its peak as the + job's disk peak; where the engine's storage can bound a layer (overlay on xfs, an engine with root) the create also + asks for `StorageOpt.size` of that disk; - on a network of its own, which reaches no other job's container. The container's first process is the supervisor, as the container's root with only `NET_ADMIN`, `SETGID`, `SETPCAP` @@ -88,6 +89,30 @@ The budget is the same as a machine host's, so a size's count is set by memory a | 6 | 11.4 GiB | 2 | 2 | 1 | 1 | | 4 | 7.6 GiB | 1 | 1 | 0 | 0 | +## A small disk + +Every job's layer and the engine's images are on the file system of the state folder (with rootless Podman, the +Runner user's home). The defaults are a Mac's: no job starts with less than 25 GB free, the job that wrote the most is +stopped under 5 GB, and each job may write 40 GB. A host whose home is a file system of 12 GB never has 25 GB free and +takes nothing, so its settings lower them: + +```json +{ "disk_hold_gb": 3, "disk_stop_gb": 1, "job_disk_gb": 8 } +``` + +`scripts/host-setup.sh install --settings ` writes them into the configuration like any other key, and a +running Runner takes them up as it takes up a changed file. The Runner refuses to start on + +- `disk_stop_gb` under 1 or above `disk_hold_gb` (a `disk_hold_gb` set alone lowers the default stop to itself); +- `job_disk_gb` under 4 or over 512; +- a `disk_hold_gb` that is not under the file system's size, since no file system is ever all free and no job would + start, or a `job_disk_gb` larger than the file system; + +and starts, saying so in its log and in `check`'s disk line, when `disk_hold_gb` and one job's disk together are +more than the file system: a job's disk is the most it may write, not what it takes, so most jobs still run, and +`disk_stop_gb` still stops the job that wrote the most before the disk fills. Settings left out are checked as before, +so a host on the defaults starts on any disk. + ## Tests - `crates/vmm-oci/tests/engine.rs`, against a real engine (CI's Linux jobs, with Docker and with rootless Podman): a @@ -97,7 +122,11 @@ The budget is the same as a machine host's, so a size's count is set by memory a release installed into the engine runs a job. - `crates/supervisor/tests/container.rs`: the firewall's inputs and what it lets out, the limits check, the seccomp filter on each architecture (with a property test against its specification) and the look at the kernel's account. -- `crates/vmhost/src/driver.rs`: a container that writes past its disk is stopped, and its disk peak is reported. +- `crates/vmhost/tests/driver.rs`: a container that writes past its disk, of whatever size its request names, is + stopped, and its disk peak is reported. +- `crates/budget/tests/disk.rs`: the disk settings' defaults, their edges, the checks against the file system, and a + small disk that takes a job once its hold is lowered and holds under it; `crates/daemon/tests/machines.rs`: a job + gets the disk the settings give. - `crates/image/tests/store.rs`: the image goes whole into the engine, each host takes only its kind, and an import that stops early, cannot be named or is refused leaves nothing. - `xtask/tests/host_setup.rs`: `--sandbox` and the keys it writes.