Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions crates/daemon/tests/service.proptest-regressions
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,4 @@ cc fb9c128abdd7bdbd0d6fcaae50eebf935ef713ec7e24674758fe5173e00c1813 # shrinks to
cc bacbc6c251aa735fc9978902f1f174fb01abc6cc92dbdfd1e645db8baa1b7ccb # shrinks to script = [Up(69), WaitingCancelled, GithubLooks, GithubLooks, Queued(true), SlowBoot(true), Minutes(149), Completed(183, false), RestStale(false), Minutes(67), Up(80), RemovalsFail(false), Cancelled(166, true), Queued(false), Minutes(59), Cancelled(55, true), LostWith(15, Both), Queued(false), RestFails(223), LostWith(7, End), Up(165), Wait(5), LostWith(27, Both), Queued(false)]
cc ea010f76c2e880d30ca781918590884ccd3ef4c59678c1da478f430d6ee4fa1c
cc 81df8993801b917c71760523b745143086350fbd4a3bd8da5bf6fb90dd5b0b94 # mac-b never ran a job, yet the console heard of one
cc a8f1c1f8125296ffc5aaa0f518886ae98c4fb9791fedbd341c16b6a3397d9f53 # sessions opened [1, 3] times, with cause for [2, 2]
48 changes: 47 additions & 1 deletion crates/daemon/tests/service.rs
Original file line number Diff line number Diff line change
Expand Up @@ -291,18 +291,25 @@ impl Rig {

/// The new Runner of [`Rig::restart_after`], not yet doing its rounds.
async fn start_again(&mut self, edit: impl FnOnce(&Rig)) {
// From this moment nothing the dying Runner asks reaches GitHub: an answer it got is the last it gets.
// From this moment nothing the dying Runner asks reaches GitHub, itself or through the console: an answer it
// got is the last it gets.
self.github.set_down(true);
let console_down = self.console.is_down();
self.console.down(true);
self.stop_pump().await;
self.service.kill();
if self.store.crashed() {
// The listeners end at their next wait; until then GitHub and the vm-hosts keep refusing them.
tokio::time::sleep(Duration::from_millis(50)).await;
self.parts.vms.freeze(false);
}
// A call the console took from the dying Runner, a session it opens at GitHub say, ends while GitHub is down:
// it does not land once the next Runner started.
self.console.quiet().await;
self.store.revive();
edit(self);
self.dead.store(false, Ordering::Relaxed);
self.console.down(console_down);
self.github.set_down(false);
let host = self.machines.with_host(|host| host.clone());
self.parts.roster = Arc::new(Roster::new());
Expand Down Expand Up @@ -2777,6 +2784,45 @@ fn running(rig: &Rig, jobs: &[i64]) -> Vec<String> {
.collect()
}

/// A session the Runner asked the console for just before it died, which the console opens at GitHub only once the
/// next Runner started, would be the dead Runner's: GitHub refuses the next one's while it lasts. The rig's Runner that
/// starts again waits for the console to end what the dying one asked, against a GitHub that is down.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn a_session_the_dying_runner_asked_the_console_for_is_not_opened_once_the_next_one_started() {
let mut rig = Rig::new(2).await;
rig.service.discover().await;
until("the scale set's session", || {
rig.github.scale_sets().iter().any(|set| set.has_session)
})
.await;
let set = rig.github.scale_sets()[0].id;
rig.console.slow_next("session/open", Duration::from_millis(300));
rig.github.lose_session(set);
until("the listener's open waits at the console", || {
rig.console.slowed() == 1
})
.await;
let before = session_opens(&rig)[0];
rig.restart().await;
// The dying Runner's open is over, whatever the console made of it.
rig.console.quiet().await;
rig.service.discover().await;
until("the next Runner's session", || {
rig.github
.scale_sets()
.iter()
.any(|s| s.id == set && s.has_session)
})
.await;
// A session that was there first would have the open refused, and tried again every 20 to 100 ms.
tokio::time::sleep(Duration::from_millis(400)).await;
assert_eq!(
session_opens(&rig)[0] - before,
1,
"the next Runner opened its session once, and nothing else did"
);
}

proptest! {
#![proptest_config(ProptestConfig { cases: 32, ..ProptestConfig::default() })]

Expand Down
1 change: 1 addition & 0 deletions crates/fake-console/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ cf-fake-github = { path = "../fake-github" }
rand = "0.9"
reqwest = { version = "0.13", default-features = false }
serde_json.workspace = true
tokio = { version = "1", features = ["time"] }

[dev-dependencies]
tokio = { version = "1", features = ["macros", "rt-multi-thread", "time"] }
Expand Down
77 changes: 74 additions & 3 deletions crates/fake-console/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
//! order, with the same statuses and bounds.
//!
//! Tests drive it ([`FakeConsole::add_code`], [`FakeConsole::pause`], [`FakeConsole::queue_command`]), make it
//! misbehave ([`FakeConsole::down`], [`FakeConsole::fail_next`], [`FakeConsole::accept_then_drop_next`],
//! misbehave ([`FakeConsole::down`], [`FakeConsole::slow_next`], [`FakeConsole::fail_next`], [`FakeConsole::accept_then_drop_next`],
//! [`FakeConsole::answer_next`]) and read what it saw ([`FakeConsole::reports`], [`FakeConsole::job_events`],
//! [`FakeConsole::github_calls`]). It is written from the console's code and not
//! from the client's types, so the client's wire format is checked against the contract and not against itself.
Expand All @@ -24,6 +24,7 @@ pub use scaleset::{

use std::collections::HashSet;
use std::sync::{Arc, Mutex, MutexGuard};
use std::time::Duration;

use cf_fake_github::FakeGithub;
use cf_fake_github::httpd::{Handler, Reply, Request, Server};
Expand Down Expand Up @@ -75,6 +76,8 @@ struct Faults {
drop_after_reports: u32,
/// What every scale set call is answered instead, while set.
scaleset: Option<Reply>,
/// The next scale set call to this route, and how long the console takes before it asks GitHub.
slow: Option<(String, Duration)>,
}

#[derive(Default)]
Expand All @@ -91,6 +94,10 @@ struct State {
/// Commands the console stopped sending (a test run that ended or was cancelled), by id.
withdrawn: HashSet<String>,
calls: usize,
/// Calls taken and not yet answered.
in_flight: usize,
/// Scale set calls [`Faults::slow`] held.
slowed: usize,
paused: bool,
/// The size the answers name the Mac to make room for.
make_room_for: Option<String>,
Expand Down Expand Up @@ -126,6 +133,22 @@ fn coded(status: u16, message: &str, code: &str) -> Reply {
Reply::status_json(status, &json!({ "error": message, "code": code }))
}

/// A call the console took, counted until it is answered or its caller went away.
struct InFlight(Arc<Mutex<State>>);

impl InFlight {
fn new(state: &Arc<Mutex<State>>) -> InFlight {
state.lock().unwrap_or_else(|e| e.into_inner()).in_flight += 1;
InFlight(state.clone())
}
}

impl Drop for InFlight {
fn drop(&mut self) {
self.0.lock().unwrap_or_else(|e| e.into_inner()).in_flight -= 1;
}
}

fn take(counter: &mut u32) -> bool {
if *counter == 0 {
return false;
Expand Down Expand Up @@ -362,7 +385,9 @@ impl FakeConsole {
}

async fn handle(state: &Arc<Mutex<State>>, request: Request) -> Reply {
let _flying = InFlight::new(state);
let github_route = request.path.starts_with(&format!("{PREFIX}github/"));
let mut slow = None;
let step = {
let mut state = state.lock().unwrap_or_else(|e| e.into_inner());
state.calls += 1;
Expand All @@ -386,14 +411,30 @@ impl FakeConsole {
return reply.clone();
}
match request.path.strip_prefix(scaleset::PREFIX) {
Some(route) if request.method == "POST" => state.scaleset_begin(route, &request),
Some(route) if request.method == "POST" => {
if state
.faults
.slow
.as_ref()
.is_some_and(|(slowed, _)| slowed == route)
{
slow = state.faults.slow.take().map(|(_, wait)| wait);
state.slowed += 1;
}
state.scaleset_begin(route, &request)
}
_ => scaleset::Step::Done(state.route(&request)),
}
};
// GitHub is asked with no lock held.
let reply = match step {
scaleset::Step::Done(reply) => reply,
scaleset::Step::Call(call) => scaleset::run(state, *call).await,
scaleset::Step::Call(call) => {
if let Some(wait) = slow {
tokio::time::sleep(wait).await;
}
scaleset::run(state, *call).await
}
};
let mut state = state.lock().unwrap_or_else(|e| e.into_inner());
let taken = matches!(&reply, Reply::Response { status: 200, .. });
Expand Down Expand Up @@ -439,6 +480,11 @@ impl FakeConsole {
self.lock().faults.down = down;
}

/// Whether [`FakeConsole::down`] has the console down.
pub fn is_down(&self) -> bool {
self.lock().faults.down
}

/// While down, every call but a `github/…` one is answered 503: reports and job events wait while the console
/// still gives the host its GitHub tokens and makes its scale set calls.
pub fn reports_down(&self, down: bool) {
Expand All @@ -452,6 +498,31 @@ impl FakeConsole {
self.lock().faults.scaleset = answer;
}

/// The next scale set call to `route` (such as `session/open`) waits `wait` before the console asks GitHub: a
/// console that is slow while the Runner that asked dies and the next one starts.
pub fn slow_next(&self, route: &str, wait: Duration) {
self.lock().faults.slow = Some((route.to_string(), wait));
}

/// How many scale set calls [`FakeConsole::slow_next`] held so far.
pub fn slowed(&self) -> usize {
self.lock().slowed
}

/// Waits until the console answered every call it took: what it still asks GitHub for a caller ends first.
/// Panics after a minute, as a console that never ends a call would hang the test.
pub async fn quiet(&self) {
let began = std::time::Instant::now();
while self.lock().in_flight > 0 {
assert!(
began.elapsed() < Duration::from_secs(60),
"the console still answers {} calls after a minute",
self.lock().in_flight
);
tokio::time::sleep(Duration::from_millis(5)).await;
}
}

/// What a console from before the scale set calls answers them: the 404 of a route it does not know, with no
/// code (agent.ts `handleAgent`).
pub fn no_scaleset_calls() -> Reply {
Expand Down
Loading
Loading