From 8265a59a25bd619ed8e5e93441c6a208199a2f67 Mon Sep 17 00:00:00 2001 From: kmatasfp <33095685+kmatas@users.noreply.github.com> Date: Thu, 20 Aug 2026 12:32:10 -0700 Subject: [PATCH 01/40] Add S10 chaos scenario driver and scheduled-fire account --- golem-test-framework/src/benchmark/config.rs | 2 + .../chaos_suites/cloud-chaos.yaml | 75 ++ integration-tests/src/benchmarks/all.rs | 4 + integration-tests/src/chaos/fires.rs | 894 ++++++++++++++++++ integration-tests/src/chaos/mod.rs | 71 +- integration-tests/src/chaos/pinned.rs | 60 +- integration-tests/src/chaos/result.rs | 297 +++++- integration-tests/src/chaos/scenarios/mod.rs | 7 + integration-tests/src/chaos/scenarios/s1.rs | 1 + integration-tests/src/chaos/scenarios/s10.rs | 665 +++++++++++++ integration-tests/src/chaos/scenarios/s12.rs | 1 + integration-tests/src/chaos/scenarios/s13.rs | 1 + integration-tests/src/chaos/scenarios/s5.rs | 1 + integration-tests/src/chaos/scenarios/s8.rs | 1 + integration-tests/src/chaos/scheduled.rs | 546 +++++++++++ integration-tests/src/chaos/summary.rs | 27 + integration-tests/src/chaos/workload.rs | 55 +- test-components/agent-counters-v2/src/lib.rs | 79 +- test-components/agent-counters/src/lib.rs | 79 +- 19 files changed, 2830 insertions(+), 36 deletions(-) create mode 100644 integration-tests/src/chaos/fires.rs create mode 100644 integration-tests/src/chaos/scenarios/s10.rs create mode 100644 integration-tests/src/chaos/scheduled.rs diff --git a/golem-test-framework/src/benchmark/config.rs b/golem-test-framework/src/benchmark/config.rs index be3cceeb25..653d281a5f 100644 --- a/golem-test-framework/src/benchmark/config.rs +++ b/golem-test-framework/src/benchmark/config.rs @@ -242,6 +242,8 @@ pub enum ChaosScenarioArg { S12, /// Rolling executor restarts under load. S13, + /// Executor pod kill while scheduled actions are between claim and fire. + S10, } /// Density subcommand action. diff --git a/integration-tests/chaos_suites/cloud-chaos.yaml b/integration-tests/chaos_suites/cloud-chaos.yaml index 00ed926cfc..a3883ff25a 100644 --- a/integration-tests/chaos_suites/cloud-chaos.yaml +++ b/integration-tests/chaos_suites/cloud-chaos.yaml @@ -318,6 +318,81 @@ scenarios: signalTimeoutSecs: 1800 + # S10 — executor pod kill while scheduled actions are pending (GOL-378). + # + # The only scenario whose workload the driver is not holding a connection to. + # Every target registers an action ten seconds ahead, over and over, so a few + # hundred actions are always accepted and not yet run. Killing the executor + # that owns their shards asks whether the platform still runs each of them, + # exactly once, after the shards move. + # + # Like S8 the driver names the pod: it picks the executor owning the largest + # share of targets and reports its address in `baseline-ready.json`. Unlike + # S8, the targets on the *other* executor keep registering too — they are the + # control group, and the report splits the two. + - code: S10 + name: executor-crash-during-scheduled-fire + enabled: true + + fault: + kind: pod-kill + target: worker-executor + # The workflow narrows the selector to the pod the driver named; `one` + # stays as the belt-and-braces bound, exactly as in S8. + mode: one + durationSecs: 60 + + phases: + # Eighteen leads' worth of registrations, so the baseline measures a + # cluster that has been firing scheduled actions steadily rather than one + # that has just started. + baselineSecs: 180 + # Covers the kill, the reschedule, and the shard reassignment that has to + # happen before anything claims the dead executor's due actions. + faultSecs: 120 + # Registrations continue throughout. Long enough that the actions + # registered during the fault also fall due, fire and are accounted for + # inside the run rather than after it. + recoverySecs: 300 + + scheduled: + # 100 targets, each with its own emitter. Also the resolution of the + # report: a lost action localises to one target out of a hundred. + targets: 100 + # 50 registrations a second overall, which is half what the mixed workload + # offers. The scheduled path is the whole experiment here rather than one + # stream of four, so the rate is set by what keeps the pending population + # large rather than by a share of a budget. + intervalMillis: 2000 + # 500 actions accepted and not yet run at any instant. It also has to + # comfortably exceed the workflow's inject-and-verify path — signal poll + # (5s) plus `kubectl apply` plus waiting for `AllInjected` — or everything + # registered before the kill would already have fired by the time the pod + # died. + leadSecs: 10 + # What recovering a pending action may cost, and the number the fire-delay + # percentiles are reported against. Derived rather than picked: the + # executor's scheduler holds a 30s lease and refreshes every 2s, and a + # shard reassignment has to complete before the surviving executor claims + # anything at all. 60s covers both without being so generous that a + # regression would sit inside it. + # + # Recorded, not asserted. A p99 past it is an attention line and a number + # in the result, because how much a reassignment may cost is a judgement. + leaseBudgetSecs: 60 + + retryPolicy: + # Identical to the others and load-bearing for the same reason, with one + # extra consequence here: the retry goes out under the original key, and + # that key is what the scheduled action carries back to the target. So a + # retried registration that registered twice shows up as one token firing + # twice, not as an arithmetic argument about totals. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + # S13 — rolling executor restarts under load (GOL-367). # # One executor killed every 60 seconds for five minutes while the mixed diff --git a/integration-tests/src/benchmarks/all.rs b/integration-tests/src/benchmarks/all.rs index af889b5156..ae6711b38c 100644 --- a/integration-tests/src/benchmarks/all.rs +++ b/integration-tests/src/benchmarks/all.rs @@ -592,6 +592,7 @@ async fn run_chaos( ChaosScenarioArg::S5 => chaos::ScenarioCode::S5, ChaosScenarioArg::S12 => chaos::ScenarioCode::S12, ChaosScenarioArg::S13 => chaos::ScenarioCode::S13, + ChaosScenarioArg::S10 => chaos::ScenarioCode::S10, }; let config = suite .scenario(code, allow_disabled) @@ -623,6 +624,9 @@ async fn run_chaos( chaos::ScenarioCode::S13 => { chaos::scenarios::s13::run(&config, &manifest, &deps, &signals, &outputs).await } + chaos::ScenarioCode::S10 => { + chaos::scenarios::s10::run(&config, &manifest, &deps, &signals, &outputs).await + } }; deps.kill_all().await; diff --git a/integration-tests/src/chaos/fires.rs b/integration-tests/src/chaos/fires.rs new file mode 100644 index 0000000000..b18f38ea29 --- /dev/null +++ b/integration-tests/src/chaos/fires.rs @@ -0,0 +1,894 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The scheduled-fire account (GOL-378). +//! +//! Every other oracle in this suite reduces to arithmetic over counts: the +//! driver submitted N, the durable state says M, and the gap between them is +//! the finding. That works because an increment is interchangeable with every +//! other increment. A scheduled action is not. The question S10 asks is whether +//! *this* action, claimed by an executor that then died, fired — once — and how +//! long the lease took to hand it to somebody else. +//! +//! So the target agent records a token per fire rather than a tally, and this +//! module pairs those tokens against the registrations the driver made. Pairing +//! is what makes the verdicts facts about a named action rather than a +//! judgement about a distribution, which is the same reason S8 probes keys +//! individually. +//! +//! ## What fails a run, and what only gets reported +//! +//! Three things fail it, and all three are statements about one token: +//! +//! - a **confirmed** registration whose action never fired +//! - a token that fired **more than once** +//! - a registration the platform **refused** that fired anyway +//! +//! Everything else is reported. In particular an *indeterminate* registration +//! that never fired is not a finding: the driver never learned whether the +//! registration landed, so an action that never fired is one of the two +//! legitimate answers. Those are counted, so a clean verdict over many of them +//! reads as weaker than a clean verdict with none. +//! +//! The same care applies to the read itself. A target whose fire log could not +//! be read, or whose log hit the component's cap, cannot testify about its own +//! registrations — those become unverifiable rather than lost. Reporting an +//! unreadable agent as lost work would turn a failed read into a correctness +//! defect, which is the exact mistake this suite exists to avoid. +//! +//! ## Delay, and why it is grouped the way it is +//! +//! Fire delay is `observed - scheduled`: how far past its due time the platform +//! actually ran the action. It is grouped two ways at once, because either +//! alone is misleading. +//! +//! By **window**, because an action due while the executor was gone is the only +//! one whose delay says anything about recovery. By **group**, because on a +//! two-executor cluster roughly half the targets were never on the pod that +//! died: mixing them in drags the percentile down until a lease recovery that +//! took its full TTL looks like a healthy p99. +//! +//! Delays are measured across two clocks — the driver mints the due time, the +//! executor stamps the fire — so a small negative delay is skew rather than an +//! action that fired early. `minDelayMs` is reported per group so that skew is +//! visible instead of silently folded into the percentiles. + +use crate::chaos::history::{OperationRecord, Outcome, Stream}; +use crate::chaos::summary::LatencyStats; +use chrono::{DateTime, TimeDelta, Utc}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; +use std::time::Duration; + +/// Ceiling on how many findings the report carries. +/// +/// A scenario that lost every action would otherwise produce tens of thousands +/// of them and an artifact nobody can open. The count is reported separately, +/// so truncation is stated rather than inferred from a suspiciously round +/// number of findings. +const MAX_FINDINGS: usize = 200; + +/// One fire, as the target agent recorded it. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FireRecord { + /// The registering invocation's idempotency key. + pub token: String, + /// When the action was due, as the driver asked for it. + pub scheduled_at: DateTime, + /// When the platform ran it, as the executor's clock saw it. + pub observed_at: DateTime, +} + +impl FireRecord { + /// How far past its due time the action ran. Negative means clock skew + /// between the driver and the executor, not an action that fired early. + pub fn delay_ms(&self) -> i64 { + (self.observed_at - self.scheduled_at).num_milliseconds() + } +} + +/// Everything read back from one target agent. +#[derive(Debug, Clone)] +pub struct TargetFireLog { + pub agent: String, + /// `ScheduleCounter.polls`, which keeps counting past the log's cap and is + /// therefore what says whether the log below is complete. + pub polls: Option, + pub fires: Vec, + /// Why the agent could not be read, when it could not be. + pub error: Option, +} + +impl TargetFireLog { + /// Whether this log can testify about its own registrations. + /// + /// Two ways it cannot: the read failed outright, or the component's fire log + /// hit its cap and dropped entries. Both leave an absent fire ambiguous. + pub fn is_complete(&self) -> bool { + match (self.error.is_some(), self.polls) { + (true, _) => false, + (false, Some(polls)) => self.fires.len() as u64 >= polls, + // No `polls` read means no way to tell whether the log is whole. + (false, None) => false, + } + } +} + +/// The fault window, as the workflow reported it. +#[derive(Debug, Clone, Copy)] +pub struct FaultWindow { + pub injected_at: DateTime, + /// Absent for a run that never saw the fault clear. + pub recovered_at: Option>, +} + +/// Which side of the fault an action was due on. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum FireWindow { + BeforeFault, + DuringFault, + AfterFault, + /// The run never learned when the fault was injected, so no action can be + /// placed relative to it. + Unknown, +} + +impl FireWindow { + pub fn as_str(self) -> &'static str { + match self { + FireWindow::BeforeFault => "before-fault", + FireWindow::DuringFault => "during-fault", + FireWindow::AfterFault => "after-fault", + FireWindow::Unknown => "unknown", + } + } + + fn of(due: DateTime, fault: Option) -> Self { + match fault { + None => FireWindow::Unknown, + Some(window) if due < window.injected_at => FireWindow::BeforeFault, + Some(FaultWindow { + recovered_at: Some(recovered), + .. + }) if due >= recovered => FireWindow::AfterFault, + Some(_) => FireWindow::DuringFault, + } + } +} + +impl std::fmt::Display for FireWindow { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// Whether an action's target was on the executor the fault killed. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum TargetGroup { + /// Owned by the killed executor when the driver signalled readiness. + OnKilledExecutor, + /// Owned by an executor the fault left alone: the run's own control group. + Elsewhere, +} + +impl TargetGroup { + pub fn as_str(self) -> &'static str { + match self { + TargetGroup::OnKilledExecutor => "on-killed-executor", + TargetGroup::Elsewhere => "elsewhere", + } + } +} + +impl std::fmt::Display for TargetGroup { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// What a token did that it should not have. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum FireViolation { + /// A registration the platform accepted whose action never ran. + NeverFired, + /// One registration, two or more fires. + FiredMoreThanOnce, + /// A registration the platform definitively refused, whose action ran + /// anyway. + FiredDespiteRejection, +} + +impl FireViolation { + pub fn as_str(self) -> &'static str { + match self { + FireViolation::NeverFired => "never-fired", + FireViolation::FiredMoreThanOnce => "fired-more-than-once", + FireViolation::FiredDespiteRejection => "fired-despite-rejection", + } + } +} + +impl std::fmt::Display for FireViolation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// One violation, against one token. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FireFinding { + pub violation: FireViolation, + pub token: String, + pub agent: String, + pub window: FireWindow, + pub detail: String, +} + +/// Fire delay for one (group, window) cell. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FireDelayStats { + pub group: TargetGroup, + pub window: FireWindow, + /// Percentiles over delays clamped at zero, so skew cannot flatter them. + pub delay: LatencyStats, + /// The most negative delay seen, which is the clock skew between the driver + /// and the executor rather than an action that fired early. + pub min_delay_ms: i64, + /// Fires whose delay exceeded the configured lease budget. + pub over_budget: u64, +} + +/// The scheduled-fire account. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ScheduleFireReport { + /// What a lease recovery is allowed to cost, from the suite YAML. Recorded + /// so a percentile in an archived result can be read years later against + /// the number it was judged by rather than against today's config. + pub lease_budget_ms: u64, + pub registrations_confirmed: u64, + pub registrations_indeterminate: u64, + pub registrations_rejected: u64, + /// Fires the targets recorded, including any whose token is unknown. + pub fires_recorded: u64, + /// Accepted registrations paired with exactly one fire. + pub fired_once: u64, + /// Registrations the driver was never sure of that fired anyway — doubt the + /// platform resolved in its own favour. + pub indeterminate_that_fired: u64, + /// Registrations the driver was never sure of that never fired. Not a + /// finding: the registration may never have landed. + pub inconclusive: u64, + /// Registrations whose target could not testify, because its log was + /// unreadable or truncated. + pub unverifiable: u64, + /// Fires whose token no registration claims. Zero on a healthy run: agent + /// names carry the run nonce, so nothing from an earlier run can appear. + pub unknown_tokens: u64, + /// Targets the read-back could not reach at all. + pub targets_unreadable: Vec, + /// Targets whose fire log hit the component's cap. + pub targets_truncated: Vec, + pub delay: Vec, + pub findings: Vec, + /// Findings past [`MAX_FINDINGS`], which the report drops rather than + /// carries. Non-zero means `findings` is a sample. + pub findings_omitted: u64, +} + +impl ScheduleFireReport { + /// Pairs registrations against fires. + /// + /// `records` is the whole history; only the scheduled stream is considered. + /// `lead` is how far ahead registrations were made, used to say when an + /// action that never fired was due — the fire log is the only place the + /// exact due time survives, and an action that never fired left no entry. + pub fn build( + records: &[OperationRecord], + logs: &[TargetFireLog], + lead: Duration, + fault: Option, + killed_targets: &BTreeSet, + lease_budget: Duration, + ) -> Self { + let lead = TimeDelta::from_std(lead).unwrap_or(TimeDelta::zero()); + let budget_ms = lease_budget.as_millis().min(u64::MAX as u128) as u64; + + let mut fires_by_token: BTreeMap<&str, Vec<&FireRecord>> = BTreeMap::new(); + let mut complete: BTreeMap<&str, bool> = BTreeMap::new(); + let mut targets_unreadable = Vec::new(); + let mut targets_truncated = Vec::new(); + let mut fires_recorded = 0u64; + + for log in logs { + complete.insert(log.agent.as_str(), log.is_complete()); + if log.error.is_some() { + targets_unreadable.push(log.agent.clone()); + } else if !log.is_complete() { + targets_truncated.push(log.agent.clone()); + } + for fire in &log.fires { + fires_recorded += 1; + fires_by_token + .entry(fire.token.as_str()) + .or_default() + .push(fire); + } + } + + let mut report = Self { + lease_budget_ms: budget_ms, + registrations_confirmed: 0, + registrations_indeterminate: 0, + registrations_rejected: 0, + fires_recorded, + fired_once: 0, + indeterminate_that_fired: 0, + inconclusive: 0, + unverifiable: 0, + unknown_tokens: 0, + targets_unreadable, + targets_truncated, + delay: Vec::new(), + findings: Vec::new(), + findings_omitted: 0, + }; + + let mut claimed: BTreeSet<&str> = BTreeSet::new(); + let mut findings: Vec = Vec::new(); + + for record in records.iter().filter(|r| r.stream == Stream::Scheduled) { + let token = record.idempotency_key.as_str(); + claimed.insert(token); + let fires = fires_by_token.get(token).map(Vec::as_slice).unwrap_or(&[]); + // The due time comes from the fire itself when there is one, because + // that is what the platform was actually told. Without a fire the + // driver only knows what it asked for. + let due = fires + .first() + .map(|f| f.scheduled_at) + .unwrap_or(record.submitted_at + lead); + let window = FireWindow::of(due, fault); + let can_testify = complete + .get(record.agent.as_str()) + .copied() + .unwrap_or(false); + + match record.outcome { + Outcome::Confirmed => report.registrations_confirmed += 1, + Outcome::Indeterminate => report.registrations_indeterminate += 1, + Outcome::Rejected => report.registrations_rejected += 1, + } + + match (record.outcome, fires.len()) { + (_, count) if count > 1 => findings.push(FireFinding { + violation: FireViolation::FiredMoreThanOnce, + token: token.to_string(), + agent: record.agent.clone(), + window, + detail: format!( + "one registration, {count} fires at {} — the action ran more than once", + fires + .iter() + .map(|f| f.observed_at.to_rfc3339()) + .collect::>() + .join(", ") + ), + }), + (Outcome::Rejected, 1) => findings.push(FireFinding { + violation: FireViolation::FiredDespiteRejection, + token: token.to_string(), + agent: record.agent.clone(), + window, + detail: "the platform refused the registration, then ran the action" + .to_string(), + }), + (Outcome::Confirmed, 1) => report.fired_once += 1, + (Outcome::Indeterminate, 1) => { + report.fired_once += 1; + report.indeterminate_that_fired += 1; + } + (Outcome::Confirmed, 0) if !can_testify => report.unverifiable += 1, + (Outcome::Confirmed, 0) => findings.push(FireFinding { + violation: FireViolation::NeverFired, + token: token.to_string(), + agent: record.agent.clone(), + window, + detail: format!( + "accepted registration due at {} never fired", + due.to_rfc3339() + ), + }), + (Outcome::Indeterminate, 0) if !can_testify => report.unverifiable += 1, + (Outcome::Indeterminate, 0) => report.inconclusive += 1, + (Outcome::Rejected, 0) => {} + (_, _) => {} + } + } + + report.unknown_tokens = fires_by_token + .iter() + .filter(|(token, _)| !claimed.contains(*token)) + .map(|(_, fires)| fires.len() as u64) + .sum(); + + report.findings_omitted = findings.len().saturating_sub(MAX_FINDINGS) as u64; + findings.truncate(MAX_FINDINGS); + report.findings = findings; + report.delay = delay_stats(logs, fault, killed_targets, budget_ms); + report + } + + /// The three conditions that fail the scenario. + pub fn has_violations(&self) -> bool { + !self.findings.is_empty() + } + + /// The p99 of the cell that the SLO is about: actions due while the + /// executor was gone, on targets it owned. + pub fn fault_window_p99_ms(&self) -> Option { + self.delay + .iter() + .find(|d| { + d.group == TargetGroup::OnKilledExecutor && d.window == FireWindow::DuringFault + }) + .map(|d| d.delay.p99_ms) + } + + /// Lines an operator should see next to the read-back verdicts. + pub fn attention_lines(&self) -> Vec { + let mut lines = Vec::new(); + for finding in &self.findings { + lines.push(format!( + "{}: token {} on target {} ({}) — {}", + finding.violation, finding.token, finding.agent, finding.window, finding.detail + )); + } + if self.findings_omitted > 0 { + lines.push(format!( + "scheduled-fire findings are a sample: {} more were dropped", + self.findings_omitted + )); + } + if !self.targets_unreadable.is_empty() { + lines.push(format!( + "{} scheduled targets could not be read back, so {} registrations are \ + unverifiable rather than accounted for", + self.targets_unreadable.len(), + self.unverifiable + )); + } + if !self.targets_truncated.is_empty() { + lines.push(format!( + "{} scheduled targets filled their fire log and dropped entries — raise the \ + cadence or shorten the run before reading this as exactly-once evidence", + self.targets_truncated.len() + )); + } + if self.unknown_tokens > 0 { + lines.push(format!( + "{} fires carried a token no registration claims, which should be impossible \ + within one run nonce", + self.unknown_tokens + )); + } + if let Some(p99) = self.fault_window_p99_ms() + && p99 > self.lease_budget_ms + { + lines.push(format!( + "scheduled-fire p99 during the fault was {p99}ms against a {}ms lease budget", + self.lease_budget_ms + )); + } + lines + } +} + +/// Delay percentiles per (group, window) cell. +fn delay_stats( + logs: &[TargetFireLog], + fault: Option, + killed_targets: &BTreeSet, + budget_ms: u64, +) -> Vec { + let mut cells: BTreeMap<(TargetGroup, FireWindow), Vec> = BTreeMap::new(); + + for log in logs { + let group = if killed_targets.contains(&log.agent) { + TargetGroup::OnKilledExecutor + } else { + TargetGroup::Elsewhere + }; + for fire in &log.fires { + cells + .entry((group, FireWindow::of(fire.scheduled_at, fault))) + .or_default() + .push(fire.delay_ms()); + } + } + + cells + .into_iter() + .map(|((group, window), delays)| { + let min = delays.iter().copied().min().unwrap_or(0); + let over_budget = delays.iter().filter(|d| **d > budget_ms as i64).count() as u64; + FireDelayStats { + group, + window, + delay: LatencyStats::from_durations( + delays.iter().map(|d| (*d).max(0) as u64).collect(), + ), + min_delay_ms: min, + over_budget, + } + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::chaos::history::{AttemptRecord, Phase}; + use test_r::test; + + fn at(offset_secs: i64) -> DateTime { + DateTime::from_timestamp(1_800_000_000 + offset_secs, 0).unwrap() + } + + const LEAD: Duration = Duration::from_secs(10); + const BUDGET: Duration = Duration::from_secs(30); + + fn record( + token: &str, + agent: &str, + outcome: Outcome, + submitted_at: DateTime, + ) -> OperationRecord { + OperationRecord { + op_id: 0, + stream: Stream::Scheduled, + phase: Phase::Fault, + agent: agent.to_string(), + method: "schedule_fire_at".to_string(), + idempotency_key: token.to_string(), + submitted_at, + completed_at: Some(submitted_at), + attempts: 1, + outcome, + duration_ms: 12, + returned_value: None, + first_attempt_value: None, + error: None, + error_class: None, + attempt_log: vec![AttemptRecord { + attempt: 1, + started_at: submitted_at, + duration_ms: 12, + returned_value: None, + succeeded: outcome == Outcome::Confirmed, + error_class: None, + error: None, + }], + } + } + + fn fire(token: &str, scheduled: DateTime, delay_ms: i64) -> FireRecord { + FireRecord { + token: token.to_string(), + scheduled_at: scheduled, + observed_at: scheduled + TimeDelta::milliseconds(delay_ms), + } + } + + /// A log the agent answered in full. + fn log(agent: &str, fires: Vec) -> TargetFireLog { + TargetFireLog { + agent: agent.to_string(), + polls: Some(fires.len() as u64), + fires, + error: None, + } + } + + fn build(records: &[OperationRecord], logs: &[TargetFireLog]) -> ScheduleFireReport { + ScheduleFireReport::build(records, logs, LEAD, None, &BTreeSet::new(), BUDGET) + } + + /// The healthy shape: one registration, one fire. + #[test] + fn a_registration_paired_with_one_fire_is_not_a_finding() { + let r = record("t-0", "target-0", Outcome::Confirmed, at(0)); + let report = build( + std::slice::from_ref(&r), + &[log("target-0", vec![fire("t-0", at(10), 40)])], + ); + assert!(!report.has_violations(), "{:?}", report.findings); + assert_eq!(report.fired_once, 1); + assert_eq!(report.registrations_confirmed, 1); + } + + /// Accepted work has to happen. This is the loss half of the guarantee. + #[test] + fn a_confirmed_registration_that_never_fired_fails_the_run() { + let r = record("t-1", "target-0", Outcome::Confirmed, at(0)); + let report = build(std::slice::from_ref(&r), &[log("target-0", vec![])]); + assert!(report.has_violations()); + assert_eq!(report.findings[0].violation, FireViolation::NeverFired); + assert_eq!(report.findings[0].token, "t-1"); + assert!( + report.findings[0].detail.contains(&at(10).to_rfc3339()), + "the finding must say when the action was due: {}", + report.findings[0].detail + ); + } + + /// The duplicate half, and the reason the target records tokens rather than + /// a tally: a lease recovered while the original claim was still running + /// runs the action twice, and no count over a busy target would show it. + #[test] + fn a_token_that_fired_twice_fails_the_run() { + let r = record("t-2", "target-0", Outcome::Confirmed, at(0)); + let report = build( + std::slice::from_ref(&r), + &[log( + "target-0", + vec![fire("t-2", at(10), 40), fire("t-2", at(10), 31_000)], + )], + ); + assert!(report.has_violations()); + assert_eq!( + report.findings[0].violation, + FireViolation::FiredMoreThanOnce + ); + } + + /// A definite refusal means nothing was accepted, so an action that runs + /// anyway is the platform contradicting its own answer. + #[test] + fn a_refused_registration_that_fired_anyway_fails_the_run() { + let r = record("t-3", "target-0", Outcome::Rejected, at(0)); + let report = build( + std::slice::from_ref(&r), + &[log("target-0", vec![fire("t-3", at(10), 5)])], + ); + assert!(report.has_violations()); + assert_eq!( + report.findings[0].violation, + FireViolation::FiredDespiteRejection + ); + } + + /// The distinction the whole verdict rests on: the driver never learned + /// whether this registration landed, so an action that never fired is one + /// of two legitimate answers. + #[test] + fn an_indeterminate_registration_that_never_fired_is_counted_rather_than_failed() { + let r = record("t-4", "target-0", Outcome::Indeterminate, at(0)); + let report = build(std::slice::from_ref(&r), &[log("target-0", vec![])]); + assert!(!report.has_violations(), "{:?}", report.findings); + assert_eq!( + report.inconclusive, 1, + "but it must be counted, so a clean verdict over many of them reads as weaker" + ); + } + + /// ...and when it did fire, the doubt resolved in the platform's favour. + #[test] + fn an_indeterminate_registration_that_fired_resolves_the_doubt() { + let r = record("t-5", "target-0", Outcome::Indeterminate, at(0)); + let report = build( + std::slice::from_ref(&r), + &[log("target-0", vec![fire("t-5", at(10), 20)])], + ); + assert!(!report.has_violations()); + assert_eq!(report.fired_once, 1); + assert_eq!(report.indeterminate_that_fired, 1); + } + + /// A read that failed is not evidence about the platform. Turning it into + /// one would report a network problem as lost durable work. + #[test] + fn an_unreadable_target_leaves_its_registrations_unverifiable_rather_than_lost() { + let r = record("t-6", "target-0", Outcome::Confirmed, at(0)); + let unreadable = TargetFireLog { + agent: "target-0".to_string(), + polls: None, + fires: Vec::new(), + error: Some("timed out after 30s".to_string()), + }; + let report = build(std::slice::from_ref(&r), &[unreadable]); + assert!(!report.has_violations(), "{:?}", report.findings); + assert_eq!(report.unverifiable, 1); + assert_eq!(report.targets_unreadable, vec!["target-0".to_string()]); + } + + /// Same reasoning for a log that filled up: the fire may have happened and + /// been dropped. `polls` is what makes that detectable at all. + #[test] + fn a_truncated_fire_log_leaves_its_registrations_unverifiable_rather_than_lost() { + let r = record("t-7", "target-0", Outcome::Confirmed, at(0)); + let truncated = TargetFireLog { + agent: "target-0".to_string(), + // The agent fired 500 actions and kept 2 of them. + polls: Some(500), + fires: vec![fire("t-other", at(10), 5), fire("t-more", at(11), 5)], + error: None, + }; + let report = build(std::slice::from_ref(&r), &[truncated]); + assert!(!report.has_violations(), "{:?}", report.findings); + assert_eq!(report.unverifiable, 1); + assert_eq!(report.targets_truncated, vec!["target-0".to_string()]); + } + + /// The grouping that makes the percentile mean anything. Both cells are + /// reported; folding them together would let the untouched half hide what + /// the recovery cost. + #[test] + fn delay_is_split_by_whether_the_target_was_on_the_killed_executor() { + let fault = FaultWindow { + injected_at: at(100), + recovered_at: Some(at(200)), + }; + let killed = BTreeSet::from(["target-killed".to_string()]); + let logs = vec![ + log("target-killed", vec![fire("a", at(150), 28_000)]), + log("target-survivor", vec![fire("b", at(150), 40)]), + ]; + let report = ScheduleFireReport::build(&[], &logs, LEAD, Some(fault), &killed, BUDGET); + + let killed_cell = report + .delay + .iter() + .find(|d| d.group == TargetGroup::OnKilledExecutor) + .expect("the killed executor's targets need their own cell"); + assert_eq!(killed_cell.window, FireWindow::DuringFault); + assert_eq!(killed_cell.delay.p99_ms, 28_000); + + let survivor_cell = report + .delay + .iter() + .find(|d| d.group == TargetGroup::Elsewhere) + .expect("the control group needs its own cell"); + assert_eq!(survivor_cell.delay.p99_ms, 40); + + assert_eq!(report.fault_window_p99_ms(), Some(28_000)); + } + + /// Actions are placed by when they were *due*, not when they fired: an + /// action due during the outage that landed afterwards is exactly the + /// population the scenario is about. + #[test] + fn an_action_is_placed_in_the_window_its_due_time_fell_in() { + let fault = Some(FaultWindow { + injected_at: at(100), + recovered_at: Some(at(200)), + }); + assert_eq!(FireWindow::of(at(99), fault), FireWindow::BeforeFault); + assert_eq!(FireWindow::of(at(100), fault), FireWindow::DuringFault); + // Due mid-fault, fired long after it healed: still a fault-window action. + assert_eq!(FireWindow::of(at(199), fault), FireWindow::DuringFault); + assert_eq!(FireWindow::of(at(200), fault), FireWindow::AfterFault); + assert_eq!(FireWindow::of(at(150), None), FireWindow::Unknown); + } + + /// A fault that never cleared leaves everything after injection inside it, + /// rather than inventing an end. + #[test] + fn a_fault_that_never_cleared_has_no_after_window() { + let fault = Some(FaultWindow { + injected_at: at(100), + recovered_at: None, + }); + assert_eq!(FireWindow::of(at(10_000), fault), FireWindow::DuringFault); + } + + /// Two clocks measure this, so a small negative delay is skew. It stays + /// visible as a minimum instead of being folded into the percentiles. + #[test] + fn clock_skew_shows_as_a_negative_minimum_rather_than_flattering_the_percentiles() { + let logs = vec![log( + "target-0", + vec![fire("a", at(10), -35), fire("b", at(11), 60)], + )]; + let report = ScheduleFireReport::build(&[], &logs, LEAD, None, &BTreeSet::new(), BUDGET); + let cell = &report.delay[0]; + assert_eq!(cell.min_delay_ms, -35); + assert_eq!(cell.delay.p50_ms, 0, "the negative sample clamps to zero"); + assert_eq!(cell.delay.max_ms, 60); + } + + /// Fires over the lease budget are counted per cell, which is what turns a + /// percentile into SLO evidence rather than a number. + #[test] + fn fires_past_the_lease_budget_are_counted() { + let logs = vec![log( + "target-0", + vec![ + fire("a", at(10), 29_000), + fire("b", at(11), 31_000), + fire("c", at(12), 45_000), + ], + )]; + let report = ScheduleFireReport::build(&[], &logs, LEAD, None, &BTreeSet::new(), BUDGET); + assert_eq!(report.delay[0].over_budget, 2); + } + + /// An artifact nobody can open is not evidence. Truncation is stated. + #[test] + fn findings_beyond_the_cap_are_counted_rather_than_carried() { + let records: Vec = (0..MAX_FINDINGS + 25) + .map(|i| record(&format!("t-{i}"), "target-0", Outcome::Confirmed, at(0))) + .collect(); + let report = build(&records, &[log("target-0", vec![])]); + assert_eq!(report.findings.len(), MAX_FINDINGS); + assert_eq!(report.findings_omitted, 25); + assert!( + report + .attention_lines() + .iter() + .any(|line| line.contains("are a sample")), + "an operator must be told the list is partial" + ); + } + + /// Agent names carry the run nonce, so this cannot happen within a run. If + /// it ever does, the pairing is answering a different question than it + /// thinks it is, and the report says so. + #[test] + fn a_fire_whose_token_no_registration_claims_is_counted() { + let r = record("t-8", "target-0", Outcome::Confirmed, at(0)); + let report = build( + std::slice::from_ref(&r), + &[log( + "target-0", + vec![fire("t-8", at(10), 5), fire("stranger", at(10), 5)], + )], + ); + assert_eq!(report.unknown_tokens, 1); + assert!(!report.has_violations()); + assert!( + report + .attention_lines() + .iter() + .any(|line| line.contains("no registration claims")) + ); + } + + /// Only the scheduled stream is paired: a durable operation carries an + /// idempotency key too, and pairing it against a fire log would invent + /// findings out of an unrelated stream. + #[test] + fn operations_from_other_streams_are_not_paired() { + let mut durable = record("t-9", "target-0", Outcome::Confirmed, at(0)); + durable.stream = Stream::Durable; + let report = build(std::slice::from_ref(&durable), &[log("target-0", vec![])]); + assert_eq!(report.registrations_confirmed, 0); + assert!(!report.has_violations()); + } + + /// The p99 an operator is judged on is the fault-window cell for the + /// targets that were on the dead pod. With no such cell there is nothing to + /// judge, and the report says nothing rather than substituting another. + #[test] + fn the_reported_p99_is_absent_when_the_fault_window_produced_no_fires() { + let report = ScheduleFireReport::build(&[], &[], LEAD, None, &BTreeSet::new(), BUDGET); + assert_eq!(report.fault_window_p99_ms(), None); + } +} diff --git a/integration-tests/src/chaos/mod.rs b/integration-tests/src/chaos/mod.rs index 2404c937a8..6bccb8f6f7 100644 --- a/integration-tests/src/chaos/mod.rs +++ b/integration-tests/src/chaos/mod.rs @@ -33,6 +33,7 @@ //! conditions that fail a run outright. pub mod errors; +pub mod fires; pub mod history; pub mod ownership; pub mod pinned; @@ -40,6 +41,7 @@ pub mod prep; pub mod probe; pub mod result; pub mod scenarios; +pub mod scheduled; pub mod signal; pub mod summary; pub mod workload; @@ -63,6 +65,8 @@ pub enum ScenarioCode { S12, /// Rolling executor restarts under load. S13, + /// Executor pod kill while scheduled actions are between claim and fire. + S10, } impl ScenarioCode { @@ -73,16 +77,18 @@ impl ScenarioCode { ScenarioCode::S5 => "S5", ScenarioCode::S12 => "S12", ScenarioCode::S13 => "S13", + ScenarioCode::S10 => "S10", } } /// Every scenario this driver implements. The suite YAML is checked against /// this list, so a scenario cannot be enabled in YAML without code behind /// it, nor implemented without an operational switch in front of it. - pub const ALL: [ScenarioCode; 5] = [ + pub const ALL: [ScenarioCode; 6] = [ ScenarioCode::S1, ScenarioCode::S5, ScenarioCode::S8, + ScenarioCode::S10, ScenarioCode::S12, ScenarioCode::S13, ]; @@ -301,6 +307,51 @@ fn default_candidate_pool_multiplier() -> u32 { 8 } +/// Shape of the scheduled-registration workload (GOL-378). +/// +/// A third experiment shape next to [`WorkloadConfig`] and [`PinnedConfig`], not +/// a variation on either. The mixed workload asks what a stream of invocations +/// does when the platform is disturbed, and the pinned workload asks what +/// happens to specific invocations that were running on the pod that died. This +/// one asks about work the driver is not holding a connection to at all: an +/// action the platform promised to run later, whose executor died in between. +/// +/// The two numbers that decide whether the run measures anything are `leadSecs` +/// and `intervalMillis`. See [`crate::chaos::scheduled`] for why. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ScheduledConfig { + /// Target agents, each with its own emitter. Also the resolution of the + /// report: a finding localises to one target out of this many. + pub targets: u32, + /// Milliseconds between registrations on one target. The offered rate is + /// `targets / interval`. + pub interval_millis: u64, + /// How far ahead each action is registered. With the cadence above this + /// sets how many actions stand accepted but not yet run at any instant, + /// which is the population a kill has to land in the middle of. + pub lead_secs: u64, + /// What recovering a scheduled action is allowed to cost, which the + /// fire-delay percentiles are reported against. An SLO the run records + /// rather than a threshold it fails on: the floor is a shard reassignment, + /// or the executor's `lease_ttl` for the rarer case of an action that was + /// already claimed, and how much more than that is acceptable is a + /// judgement. + pub lease_budget_secs: u64, +} + +impl ScheduledConfig { + pub fn interval(&self) -> Duration { + Duration::from_millis(self.interval_millis) + } + pub fn lead(&self) -> Duration { + Duration::from_secs(self.lead_secs) + } + pub fn lease_budget(&self) -> Duration { + Duration::from_secs(self.lease_budget_secs) + } +} + /// One step of the executor scale schedule the workflow runs during the fault. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -372,6 +423,10 @@ pub struct ScenarioConfig { /// The pinned in-flight workload. Absent for scenarios that do not run one. #[serde(default)] pub pinned: Option, + /// The scheduled-registration workload. Absent for scenarios that do not + /// run one. + #[serde(default)] + pub scheduled: Option, /// Shard-ownership oracle settings. Absent for scenarios that do not sample /// executor assignments. #[serde(default)] @@ -425,6 +480,16 @@ impl ScenarioConfig { }) } + /// The scheduled-registration block. See [`Self::require_workload`]. + pub fn require_scheduled(&self) -> anyhow::Result<&ScheduledConfig> { + self.scheduled.as_ref().ok_or_else(|| { + anyhow::anyhow!( + "chaos scenario {} needs a `scheduled` block in the suite YAML", + self.code + ) + }) + } + /// The pinned workload block. See [`Self::require_workload`]. pub fn require_pinned(&self) -> anyhow::Result<&PinnedConfig> { self.pinned.as_ref().ok_or_else(|| { @@ -558,6 +623,7 @@ mod tests { rate_per_sec: 1, }), pinned: None, + scheduled: None, ownership: None, scale_during_fault: None, retry_policy: RetryPolicy::default(), @@ -656,6 +722,9 @@ mod tests { ScenarioCode::S13 => { entry.require_workload().unwrap(); } + ScenarioCode::S10 => { + entry.require_scheduled().unwrap(); + } } } } diff --git a/integration-tests/src/chaos/pinned.rs b/integration-tests/src/chaos/pinned.rs index 95f148f61a..4ae068c5ee 100644 --- a/integration-tests/src/chaos/pinned.rs +++ b/integration-tests/src/chaos/pinned.rs @@ -116,29 +116,35 @@ pub fn candidate_agent_name(key_prefix: &str, index: u32) -> String { format!("{key_prefix}-{}-{index:04}", Stream::PinnedHttp) } -/// The routing-table `AgentId` for one pinned agent. +/// The routing-table `AgentId` for one agent of `agent_type`. /// /// This has to match how the worker-service builds the id it routes on — /// the component id plus the *string form* of the parsed agent id — or the /// ownership calculation would be answering a different question from the one /// the platform answers. -fn routing_agent_id(ctx: &WorkloadContext, agent: &str) -> AgentId { - let parsed: ParsedAgentId = agent_id!(COUNTER_AGENT, agent.to_string()); +pub fn routing_agent_id(ctx: &WorkloadContext, agent_type: &str, agent: &str) -> AgentId { + let parsed: ParsedAgentId = agent_id!(agent_type, agent.to_string()); AgentId { component_id: ctx.counters.id, agent_id: parsed.to_string(), } } -/// Groups candidate agents by the executor that owns them. -fn owners( +/// Groups agents of `agent_type` by the executor that owns them. +/// +/// Takes the type rather than assuming `Counter` because S10 asks the same +/// question about its schedule targets, and ownership is per agent id: two +/// agent types with the same name are two different agents on two possibly +/// different executors. +pub fn owners_by_pod( ctx: &WorkloadContext, table: &RoutingTable, - candidates: &[String], + agent_type: &str, + agents: &[String], ) -> BTreeMap> { let mut by_pod: BTreeMap> = BTreeMap::new(); - for agent in candidates { - if let Some(pod) = table.lookup(&routing_agent_id(ctx, agent)) { + for agent in agents { + if let Some(pod) = table.lookup(&routing_agent_id(ctx, agent_type, agent)) { by_pod .entry(pod.to_string()) .or_default() @@ -148,6 +154,27 @@ fn owners( by_pod } +/// Groups candidate pinned agents by the executor that owns them. +fn owners( + ctx: &WorkloadContext, + table: &RoutingTable, + candidates: &[String], +) -> BTreeMap> { + owners_by_pod(ctx, table, COUNTER_AGENT, candidates) +} + +/// Host part of an executor address, which is what a Kubernetes `status.podIP` +/// field selector matches. +/// +/// No port at all is not a shape the shard-manager produces, but degrading to +/// the whole string beats panicking mid-run. +pub fn pod_ip_of(pod_address: &str) -> String { + pod_address + .rsplit_once(':') + .map(|(host, _)| host.to_string()) + .unwrap_or_else(|| pod_address.to_string()) +} + /// Chooses an executor and the agents it owns. /// /// Fails rather than falling back to an unpinned run: a scenario that quietly @@ -201,10 +228,7 @@ pub async fn select( } let agents: Vec = owned.into_iter().take(config.agents as usize).collect(); - let pod_ip = pod_address - .rsplit_once(':') - .map(|(host, _)| host.to_string()) - .unwrap_or_else(|| pod_address.clone()); + let pod_ip = pod_ip_of(&pod_address); info!( "S8: pinned {} agents to executor {pod_address} (scanned {pool} candidates across {} executors)", @@ -242,7 +266,7 @@ pub async fn verify_ownership( let mut drifted: Vec = Vec::new(); for agent in &selection.agents { let owner = table - .lookup(&routing_agent_id(ctx, agent)) + .lookup(&routing_agent_id(ctx, COUNTER_AGENT, agent)) .map(|pod| pod.to_string()); if owner.as_deref() != Some(selection.pod_address.as_str()) { drifted.push(format!( @@ -379,16 +403,10 @@ mod tests { /// hold for the `ip:port` form the routing table actually produces. #[test] fn a_pod_address_splits_into_an_ip_the_workflow_can_select_on() { - let split = |address: &str| { - address - .rsplit_once(':') - .map(|(host, _)| host.to_string()) - .unwrap_or_else(|| address.to_string()) - }; - assert_eq!(split("10.0.14.207:9000"), "10.0.14.207"); + assert_eq!(pod_ip_of("10.0.14.207:9000"), "10.0.14.207"); // No port at all is not a shape the shard-manager produces, but // degrading to the whole string beats panicking mid-run. - assert_eq!(split("10.0.14.207"), "10.0.14.207"); + assert_eq!(pod_ip_of("10.0.14.207"), "10.0.14.207"); } /// Candidate names carry the run prefix and the stream, and are zero-padded diff --git a/integration-tests/src/chaos/result.rs b/integration-tests/src/chaos/result.rs index 0937b0d0f2..99f7077611 100644 --- a/integration-tests/src/chaos/result.rs +++ b/integration-tests/src/chaos/result.rs @@ -25,8 +25,9 @@ //! same scenario ran anywhere else. use crate::chaos::pinned::PinnedSelection; +use crate::chaos::scheduled::ScheduledSelection; use crate::chaos::summary::{ChaosSummary, TerminationReason}; -use crate::chaos::{FaultConfig, PinnedConfig, RetryPolicy, WorkloadConfig}; +use crate::chaos::{FaultConfig, PinnedConfig, RetryPolicy, ScheduledConfig, WorkloadConfig}; use chrono::{DateTime, Utc}; use golem_test_framework::benchmark::RunMetadata; use serde::{Deserialize, Serialize}; @@ -130,6 +131,15 @@ pub struct ChaosResult { /// to own. Present only for scenarios that pin the target. #[serde(default, skip_serializing_if = "Option::is_none")] pub pinned_selection: Option, + /// The scheduled-registration workload the run was configured with, if any. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub scheduled: Option, + /// How the schedule targets divided around the executor the fault was aimed + /// at. Present only for S10, and load-bearing for reading its percentiles: + /// without it there is no way to tell the affected population from the + /// control group. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub scheduled_selection: Option, pub retry_policy: RetryPolicy, pub scope: RunScope, pub summary: ChaosSummary, @@ -197,6 +207,8 @@ mod tests { }), pinned: None, pinned_selection: None, + scheduled: None, + scheduled_selection: None, retry_policy: RetryPolicy::default(), scope: RunScope { environment_id: "env-1".to_string(), @@ -209,6 +221,92 @@ mod tests { } } + /// The S10 shape, whose report is read by a script in another repository. + /// + /// `ci-scripts/chaos-investigation-report.py` in golem-cloud renders these + /// fields by name. The two repositories cannot be changed atomically, so a + /// rename here would silently empty a section of the investigation report + /// rather than fail anything. Naming the keys in a test is what makes that + /// break loudly and locally. + #[test] + fn an_s10_result_carries_the_schedule_fire_fields_the_investigation_report_reads() { + use crate::chaos::fires::{ + FaultWindow, FireRecord, ScheduleFireReport, TargetFireLog, + }; + use crate::chaos::history::Stream; + + let now = Utc::now(); + let due = now + chrono::Duration::seconds(10); + let mut result = sample_result(TerminationReason::Completed); + result.scenario_code = "S10".to_string(); + result.scheduled = Some(crate::chaos::ScheduledConfig { + targets: 100, + interval_millis: 2000, + lead_secs: 10, + lease_budget_secs: 60, + }); + result.summary = ChaosSummary::build(&[], Vec::new(), Vec::new(), Some(now)) + .with_schedule_fires(ScheduleFireReport::build( + &[], + &[TargetFireLog { + agent: "chaos-s10-scheduled-target-0000".to_string(), + polls: Some(1), + fires: vec![FireRecord { + token: "chaos-s10-scheduled-target-0000-00000001".to_string(), + scheduled_at: due, + observed_at: due + chrono::Duration::seconds(4), + }], + error: None, + }], + std::time::Duration::from_secs(10), + Some(FaultWindow { + injected_at: now, + recovered_at: None, + }), + &std::collections::BTreeSet::from([ + "chaos-s10-scheduled-target-0000".to_string(), + ]), + std::time::Duration::from_secs(60), + )); + + let json = serde_json::to_value(&result).unwrap(); + let fires = &json["summary"]["scheduleFires"]; + for key in [ + "leaseBudgetMs", + "registrationsConfirmed", + "registrationsIndeterminate", + "firesRecorded", + "firedOnce", + "indeterminateThatFired", + "inconclusive", + "unverifiable", + "unknownTokens", + "targetsUnreadable", + "targetsTruncated", + "delay", + "findings", + "findingsOmitted", + ] { + assert!( + !fires[key].is_null(), + "summary.scheduleFires.{key} is what the investigation report reads" + ); + } + let cell = &fires["delay"][0]; + assert_eq!(cell["group"], "on-killed-executor"); + assert_eq!(cell["window"], "during-fault"); + assert_eq!(cell["delay"]["p99Ms"], 4000); + assert_eq!(cell["overBudget"], 0); + assert_eq!(cell["minDelayMs"], 4000); + assert_eq!(json["scheduled"]["leaseBudgetSecs"], 60); + + // And it still round-trips, so an archived S10 result stays readable. + let parsed: ChaosResult = serde_json::from_str(&json.to_string()).unwrap(); + assert_eq!(parsed.scenario_code, "S10"); + assert!(parsed.summary.schedule_fires.is_some()); + assert!(!Stream::Scheduled.to_string().is_empty()); + } + #[test] fn result_round_trips_through_json() { let result = sample_result(TerminationReason::Completed); @@ -400,6 +498,8 @@ mod sample_artifact { }), pinned: None, pinned_selection: None, + scheduled: None, + scheduled_selection: None, retry_policy: RetryPolicy::default(), scope: RunScope { environment_id: "0192f000-0000-7000-8000-000000000001".to_string(), @@ -418,4 +518,199 @@ mod sample_artifact { result.save(&path).unwrap(); println!("wrote sample result to {path}"); } + + /// The same, for S10, whose report has a section of its own. + /// + /// A separate artifact rather than a field added to the one above: the two + /// scenarios do not share a workload shape, and a sample that carried both + /// a mixed workload and a scheduled one would not resemble anything the + /// driver ever writes. + #[test] + fn write_sample_s10_result_artifact() { + use crate::chaos::fires::{FaultWindow, FireRecord, ScheduleFireReport, TargetFireLog}; + use crate::chaos::scheduled::ScheduledSelection; + + let Ok(path) = std::env::var("CHAOS_SAMPLE_RESULT_S10") else { + return; + }; + let at = |s: i64| Utc.timestamp_opt(1_800_000_000 + s, 0).unwrap(); + let killed = "chaos-s10-scheduled-target-0000"; + let survivor = "chaos-s10-scheduled-target-0001"; + + // Four registrations per target. On the killed executor one of them + // never fires, which is the finding the section exists to render. + let mut records = Vec::new(); + let mut killed_fires = Vec::new(); + let mut survivor_fires = Vec::new(); + for i in 0..8u64 { + let target = if i % 2 == 0 { killed } else { survivor }; + // Spread so some actions fall due before the kill and some during it, + // which is what gives the report both a control row and the row it is + // actually about. + let submitted = at(285 + i as i64 * 4); + let token = format!("{target}-{:08}", i / 2); + records.push(OperationRecord { + op_id: i, + stream: Stream::Scheduled, + phase: if submitted < at(300) { + Phase::Baseline + } else { + Phase::Fault + }, + agent: target.to_string(), + method: "schedule_fire_at".to_string(), + idempotency_key: token.clone(), + submitted_at: submitted, + completed_at: Some(submitted), + attempts: 1, + outcome: Outcome::Confirmed, + duration_ms: 8 + i, + returned_value: None, + first_attempt_value: None, + error: None, + error_class: None, + attempt_log: Vec::new(), + }); + + let due = submitted + chrono::Duration::seconds(10); + if target == killed { + // The last one is the action the kill swallowed. + if i < 6 { + killed_fires.push(FireRecord { + token, + scheduled_at: due, + // Late by a shard reassignment. + observed_at: due + chrono::Duration::milliseconds(41_500), + }); + } + } else { + survivor_fires.push(FireRecord { + token, + scheduled_at: due, + observed_at: due + chrono::Duration::milliseconds(120), + }); + } + } + + let logs = vec![ + TargetFireLog { + agent: killed.to_string(), + polls: Some(killed_fires.len() as u64), + fires: killed_fires, + error: None, + }, + TargetFireLog { + agent: survivor.to_string(), + polls: Some(survivor_fires.len() as u64), + fires: survivor_fires, + error: None, + }, + ]; + + let on_pod = std::collections::BTreeSet::from([killed.to_string()]); + let report = ScheduleFireReport::build( + &records, + &logs, + std::time::Duration::from_secs(10), + Some(FaultWindow { + injected_at: at(300), + recovered_at: Some(at(420)), + }), + &on_pod, + std::time::Duration::from_secs(60), + ); + + let readback: Vec = logs + .iter() + .map(|log| { + let scoped: Vec<&OperationRecord> = + records.iter().filter(|r| r.agent == log.agent).collect(); + AgentReadback::evaluate( + Stream::Scheduled, + &log.agent, + &scoped, + Ok(log.polls.unwrap_or(0)), + ) + }) + .collect(); + + let result = ChaosResult { + schema_version: RESULT_SCHEMA_VERSION, + scenario_code: "S10".to_string(), + scenario_name: "executor-crash-during-scheduled-fire".to_string(), + completed: false, + termination_reason: TerminationReason::ScheduledFireViolated { + findings: report.findings.len() as u64, + first: report + .findings + .first() + .map(|f| format!("{} on token {}", f.violation, f.token)) + .unwrap_or_default(), + }, + started_at: at(0), + ended_at: Some(at(900)), + phases: Phases { + baseline: Some({ + let mut w = PhaseWindow::started(at(0)); + w.end(at(300)); + w + }), + fault: Some({ + let mut w = PhaseWindow::started(at(300)); + w.end(at(420)); + w + }), + recovery: Some({ + let mut w = PhaseWindow::started(at(420)); + w.end(at(720)); + w + }), + }, + fault_injected_at: Some(at(300)), + fault_recovered_at: Some(at(420)), + fault_id: Some("chaos-s10-12345".to_string()), + fault_target_observed: Some("worker-executor-abc123".to_string()), + fault: FaultConfig { + kind: "pod-kill".to_string(), + target: "worker-executor".to_string(), + mode: "one".to_string(), + target_count: None, + duration_secs: 60, + }, + workload: None, + pinned: None, + pinned_selection: None, + scheduled: Some(crate::chaos::ScheduledConfig { + targets: 2, + interval_millis: 2000, + lead_secs: 10, + lease_budget_secs: 60, + }), + scheduled_selection: Some(ScheduledSelection { + pod_address: "10.0.1.1:9000".to_string(), + pod_ip: "10.0.1.1".to_string(), + on_pod: vec![killed.to_string()], + elsewhere: vec![survivor.to_string()], + targets_per_pod: [ + ("10.0.1.1:9000".to_string(), 1), + ("10.0.1.2:9000".to_string(), 1), + ] + .into_iter() + .collect(), + number_of_shards: 1024, + }), + retry_policy: RetryPolicy::default(), + scope: RunScope { + environment_id: "0192f000-0000-7000-8000-000000000001".to_string(), + component_ids: vec!["0192f000-0000-7000-8000-000000000002".to_string()], + agent_id_prefix: "chaos-s10".to_string(), + idempotency_key_prefix: "chaos-s10-".to_string(), + }, + summary: ChaosSummary::build(&records, readback, Vec::new(), Some(at(300))) + .with_schedule_fires(report), + run_metadata: None, + }; + result.save(&path).unwrap(); + println!("wrote sample S10 result to {path}"); + } } diff --git a/integration-tests/src/chaos/scenarios/mod.rs b/integration-tests/src/chaos/scenarios/mod.rs index 4ced61c0fe..8ff89b5af7 100644 --- a/integration-tests/src/chaos/scenarios/mod.rs +++ b/integration-tests/src/chaos/scenarios/mod.rs @@ -26,6 +26,7 @@ //! produce a plausible-looking report from a wasted maintenance window. pub mod s1; +pub mod s10; pub mod s12; pub mod s13; pub mod s5; @@ -36,6 +37,7 @@ use crate::chaos::history::{OperationHistory, OperationRecord, Stream}; use crate::chaos::ownership::OwnershipSample; use crate::chaos::pinned::PinnedSelection; use crate::chaos::result::{ChaosResult, Phases, RESULT_SCHEMA_VERSION, RunScope}; +use crate::chaos::scheduled::ScheduledSelection; use crate::chaos::signal::SignalError; use crate::chaos::summary::{ AgentReadback, ChaosSummary, ExactlyOnceReport, RoutingSnapshot, TerminationReason, @@ -76,6 +78,9 @@ pub struct ScenarioOutcome { pub termination_reason: TerminationReason, /// Present only for scenarios that pin the fault to one executor. pub pinned_selection: Option, + /// Present only for S10, which divides its targets around the executor the + /// fault was aimed at rather than driving only the ones it owns. + pub scheduled_selection: Option, } /// Assembles the archived result. @@ -99,6 +104,8 @@ pub fn build_result(config: &ScenarioConfig, outcome: ScenarioOutcome) -> ChaosR workload: config.workload.clone(), pinned: config.pinned.clone(), pinned_selection: outcome.pinned_selection, + scheduled: config.scheduled.clone(), + scheduled_selection: outcome.scheduled_selection, retry_policy: config.retry_policy.clone(), scope: outcome.scope, summary: outcome.summary, diff --git a/integration-tests/src/chaos/scenarios/s1.rs b/integration-tests/src/chaos/scenarios/s1.rs index 1b055901a3..da28910661 100644 --- a/integration-tests/src/chaos/scenarios/s1.rs +++ b/integration-tests/src/chaos/scenarios/s1.rs @@ -233,6 +233,7 @@ pub async fn run( summary, termination_reason: $reason, pinned_selection: None, + scheduled_selection: None, }, ); write_outputs(&result, &history, outputs)?; diff --git a/integration-tests/src/chaos/scenarios/s10.rs b/integration-tests/src/chaos/scenarios/s10.rs new file mode 100644 index 0000000000..d778bd9f99 --- /dev/null +++ b/integration-tests/src/chaos/scenarios/s10.rs @@ -0,0 +1,665 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! S10 — executor crash during scheduled-action fire (GOL-378). +//! +//! Every other scenario in this suite kills an executor while the driver is +//! holding a connection to it. S10 kills one while it is holding a *promise*: +//! work the platform accepted, acknowledged, and undertook to run later. Nobody +//! is waiting on the other end of a socket for it, which is exactly why it is +//! worth testing separately. A dropped invocation shows up as a failed request. +//! A dropped scheduled action shows up as nothing at all. +//! +//! ## The two windows, and which one a kill reliably lands in +//! +//! An executor's scheduler claims the actions that are due for the shards it +//! owns, leases each one, enqueues the invocation, and acknowledges. That +//! claim-to-acknowledge window is milliseconds wide: the action itself is a +//! near-no-op, and once the invocation is enqueued it is durable in the target +//! agent's oplog and covered by ordinary worker recovery rather than by the +//! lease. Killing an executor inside it is possible, not aimable, and its +//! signature is a fire delayed by roughly the lease TTL. The run reports when +//! that happens instead of claiming to have arranged it. +//! +//! The window a kill does land in, every time, is the wide one. At any instant +//! several hundred actions are registered and not yet due, and the shards they +//! belong to are owned by an executor that is about to stop existing. Nothing +//! claims them until the shards move, so the delay this scenario measures is +//! dominated by shard reassignment rather than by lease expiry. Both end in the +//! same question: does an accepted action still run, exactly once, once the +//! cluster has finished rearranging itself. +//! +//! ## Why the target agent records tokens rather than counting +//! +//! S12 already counts scheduled fires and compares the total against what the +//! driver registered. That is enough to notice that *something* went wrong +//! across a whole population, and useless for saying what. Here every +//! registration carries its own idempotency key into the scheduled action, and +//! the target agent records it when the action runs. Pairing tokens turns both +//! failures into statements about one named action: this registration, accepted +//! at this time, due at this time, never ran. See [`crate::chaos::fires`]. +//! +//! ## Why the kill is aimed, and why every target is still driven +//! +//! The driver picks the executor owning the largest share of targets and names +//! it in the readiness signal, the same way S8 does. Without that, `mode: one` +//! would pick a pod at random and the run could not say which actions were +//! supposed to be disturbed. +//! +//! Unlike S8, though, the targets it does *not* own keep running too. They are +//! the control group: on a two-executor cluster roughly half the population is +//! never touched, and reporting one percentile over both would let a lease +//! recovery that took its full TTL hide behind the half that was never +//! disturbed. +//! +//! ## What the run measures the kill against +//! +//! The driver cannot see a claim, and does not pretend to. Every target +//! re-registers on a fixed cadence at a fixed lead, so actions arrive into +//! every part of the cycle continuously and there is always a population +//! registered but not yet due. What the run then records is how large that +//! population actually was at the instant the pod died, and how much of it was +//! on the pod — measured from the history, not assumed from the cadence, +//! because a platform that had slowed down would have registered fewer than the +//! arithmetic says. A kill that caught none of it is reported as a warning: a +//! clean account of a mechanism that was never disturbed is not evidence. +//! +//! ## What fails the run +//! +//! Only the three token-level violations in [`ScheduleFireReport`]. Fire delay +//! is reported against the configured lease budget as SLO evidence, not +//! asserted: how much a lease recovery may cost is a judgement, and the number +//! that matters is in the result either way. + +use crate::chaos::fires::{FaultWindow, ScheduleFireReport}; +use crate::chaos::history::{OperationHistory, OperationRecord, Outcome, Phase, Stream}; +use crate::chaos::prep::ChaosPrepManifest; +use crate::chaos::result::{ChaosResult, PhaseWindow, Phases, RunScope}; +use crate::chaos::scenarios::{ + OutputPaths, ScenarioOutcome, WARMUP_SETTLE, build_result, readback_for, signal_termination, + snapshot_routing, wait_for_settled_routing, write_outputs, +}; +use crate::chaos::scheduled::{self, ScheduledSelection}; +use crate::chaos::signal::{BaselineReady, FaultSignals, FaultTarget}; +use crate::chaos::summary::{AgentReadback, ChaosSummary, TerminationReason}; +use crate::chaos::workload::{self, PhaseMarker, WorkloadContext}; +use crate::chaos::{ScenarioCode, ScenarioConfig, ScheduledConfig}; +use chrono::{DateTime, TimeDelta, Utc}; +use golem_test_framework::config::BenchmarkTestDependencies; +use golem_test_framework::dsl::TestDsl; +use std::collections::BTreeSet; +use std::time::Duration; +use tracing::{info, warn}; + +/// Extra quiet after the last registration's action is due, before the fire +/// logs are read. +/// +/// The rest of the settle is derived from the configuration rather than fixed: +/// the final registration falls due one `lead` after the workload stops, and if +/// its executor died holding the claim, the recovery costs up to one lease +/// budget on top. Reading before that elapsed would report actions as lost that +/// were merely late, which is the one mistake this scenario cannot afford. +const SETTLE_MARGIN: Duration = Duration::from_secs(30); + +/// How many targets to sample after the baseline to prove actions are firing at +/// all. +/// +/// A handful, because this is a smoke test rather than a measurement: if the +/// scheduling path is broken, every target is equally broken, and the point is +/// to fail before spending the fault window on a run that would report a clean +/// account of nothing. +const FIRE_PROOF_SAMPLE: usize = 5; + +pub async fn run( + config: &ScenarioConfig, + manifest: &ChaosPrepManifest, + deps: &BenchmarkTestDependencies, + signals: &FaultSignals, + outputs: &OutputPaths, +) -> anyhow::Result { + let started_at = Utc::now(); + let scheduled_config = config.require_scheduled()?; + let history = OperationHistory::new(ScenarioCode::S10.as_str()); + let key_prefix = crate::chaos::scenario_key_prefix(ScenarioCode::S10); + + let user = manifest.user_context(deps); + let counters = user + .get_latest_component_revision(&manifest.counters_component_id) + .await?; + let promise = user + .get_latest_component_revision(&manifest.promise_component_id) + .await?; + + let ctx = WorkloadContext { + user, + counters, + promise, + history: history.clone(), + retry: config.retry_policy.clone(), + phase: PhaseMarker::new(Phase::Baseline), + key_prefix: key_prefix.clone(), + }; + + let scope = RunScope { + environment_id: manifest.environment_id.0.to_string(), + component_ids: vec![manifest.counters_component_id.0.to_string()], + agent_id_prefix: key_prefix.clone(), + idempotency_key_prefix: format!("{key_prefix}-"), + }; + + let targets: Vec = (0..scheduled_config.targets) + .map(|index| ctx.schedule_target_name(index)) + .collect(); + + let mut phases = Phases::default(); + let mut routing_snapshots = Vec::new(); + let mut fault_injected_at = None; + let mut fault_recovered_at = None; + let mut fault_id = None; + let mut fault_target_observed = None; + let mut selection: Option = None; + let mut attention_extra: Vec = Vec::new(); + + macro_rules! finish { + ($reason:expr, $records:expr, $readback:expr, $fires:expr) => {{ + let mut summary = ChaosSummary::build( + $records, + $readback, + routing_snapshots.clone(), + fault_injected_at, + ); + summary.attention.extend(attention_extra.clone()); + if let Some(report) = $fires { + summary = summary.with_schedule_fires(report); + } + let result = build_result( + config, + ScenarioOutcome { + started_at, + phases: phases.clone(), + fault_injected_at, + fault_recovered_at, + fault_id: fault_id.clone(), + fault_target_observed: fault_target_observed.clone(), + scope: scope.clone(), + summary, + termination_reason: $reason, + pinned_selection: None, + scheduled_selection: selection.clone(), + }, + ); + write_outputs(&result, &history, outputs)?; + return Ok(result); + }}; + } + + // ── Warm-up ───────────────────────────────────────────────────────────── + routing_snapshots.push(snapshot_routing(deps, "before-warmup").await); + attention_extra.push(wait_for_settled_routing(deps, &mut routing_snapshots).await); + info!( + "S10: warming {} emitters and targets before the baseline", + targets.len() + ); + let warmed = scheduled::warm(&ctx, &targets).await; + info!("S10: warmed {warmed} agents, settling {WARMUP_SETTLE:?}"); + tokio::time::sleep(WARMUP_SETTLE).await; + + // ── Aim the fault ─────────────────────────────────────────────────────── + // Before the baseline, because a run that cannot be aimed should not spend + // a maintenance window proving it. + let chosen = match scheduled::select(&ctx, deps, &targets).await { + Ok(chosen) => chosen, + Err(e) => { + warn!("S10: could not aim the fault at an executor: {e:#}"); + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!("{e:#}"), + }, + &records, + Vec::new(), + None + ); + } + }; + selection = Some(chosen.clone()); + routing_snapshots.push(snapshot_routing(deps, "before-fault").await); + + // ── Baseline ──────────────────────────────────────────────────────────── + info!( + "S10: baseline phase, {} targets registering every {:?} at a {:?} lead, for {:?}", + targets.len(), + scheduled_config.interval(), + scheduled_config.lead(), + config.phases.baseline() + ); + phases.baseline = Some(PhaseWindow::started(Utc::now())); + let handle = scheduled::start(ctx.clone(), &targets, scheduled_config); + tokio::time::sleep(config.phases.baseline()).await; + if let Some(window) = phases.baseline.as_mut() { + window.end(Utc::now()); + } + + let baseline_operations = history.confirmed_in_phase(Phase::Baseline); + if baseline_operations == 0 { + warn!("S10: baseline registered nothing, aborting before injection"); + handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::PlatformUnreachable { + detail: "no scheduled registration succeeded during the baseline phase".to_string(), + }, + &records, + Vec::new(), + None + ); + } + + // Registering is not firing. A platform that accepted every registration and + // scheduled none of them would otherwise reach read-back and report a + // flawless account of a mechanism that never ran. + let sampled = sample_fire_count(&ctx, &targets).await; + if sampled == 0 { + warn!("S10: {baseline_operations} registrations accepted and no action has fired"); + handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::StreamNeverSucceeded { + stream: Stream::Scheduled.to_string(), + }, + &records, + Vec::new(), + None + ); + } + info!( + "S10: baseline complete ({baseline_operations} registrations, {sampled} fires across a \ + sample of {} targets)", + FIRE_PROOF_SAMPLE.min(targets.len()) + ); + + // ── Verify ownership, then signal ─────────────────────────────────────── + if let Err(e) = scheduled::verify_ownership(&ctx, deps, &chosen).await { + warn!("S10: target ownership no longer holds, refusing to inject: {e:#}"); + handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!("{e:#}"), + }, + &records, + Vec::new(), + None + ); + } + + info!( + "S10: signalling readiness with fault target {} ({} of {} targets on it)", + chosen.pod_address, + chosen.on_pod.len(), + targets.len() + ); + signals.write_baseline_ready(&BaselineReady { + scenario_code: ScenarioCode::S10.as_str().to_string(), + ready_at: Utc::now(), + baseline_operations, + fault_target: Some(FaultTarget { + pod_address: chosen.pod_address.clone(), + pod_ip: chosen.pod_ip.clone(), + owned_agents: chosen.on_pod.clone(), + }), + })?; + + // ── Fault ─────────────────────────────────────────────────────────────── + let injected = match signals.await_fault_injected(config.signal_timeout()).await { + Ok(injected) => injected, + Err(e) => { + warn!("S10: no fault-injected signal arrived: {e}"); + handle.stop().await; + let records = history.snapshot(); + finish!(signal_termination(&e), &records, Vec::new(), None); + } + }; + info!( + "S10: fault {} ({} on {}) reported active at {}", + injected.fault_id, injected.kind, injected.target, injected.injected_at + ); + fault_injected_at = Some(injected.injected_at); + fault_id = Some(injected.fault_id.clone()); + fault_target_observed = Some(injected.target.clone()); + ctx.phase.set(Phase::Fault); + phases.fault = Some(PhaseWindow::started(injected.injected_at)); + + // How much work was actually in the window the scenario is about. Measured + // from the history rather than assumed from the cadence, because a platform + // that had slowed down would have registered fewer than the arithmetic says. + let on_pod: BTreeSet = chosen.on_pod.iter().cloned().collect(); + let pending = pending_at_injection( + &history.snapshot(), + injected.injected_at, + scheduled_config.lead(), + &on_pod, + ); + attention_extra.push(pending.describe()); + info!("S10: {}", pending.describe()); + + let recovered = match signals.await_fault_recovered(config.signal_timeout()).await { + Ok(recovered) => recovered, + Err(e) => { + warn!("S10: no fault-recovered signal arrived: {e}"); + handle.stop().await; + let records = history.snapshot(); + finish!(signal_termination(&e), &records, Vec::new(), None); + } + }; + info!( + "S10: fault cleared at {} ({})", + recovered.recovered_at, recovered.termination_reason + ); + fault_recovered_at = Some(recovered.recovered_at); + if let Some(window) = phases.fault.as_mut() { + window.end(recovered.recovered_at); + } + + // ── Recovery ──────────────────────────────────────────────────────────── + info!( + "S10: recovery phase, registering for a further {:?}", + config.phases.recovery() + ); + ctx.phase.set(Phase::Recovery); + phases.recovery = Some(PhaseWindow::started(Utc::now())); + tokio::time::sleep(config.phases.recovery()).await; + let skipped = handle.skipped(); + handle.stop().await; + if let Some(window) = phases.recovery.as_mut() { + window.end(Utc::now()); + } + if skipped > 0 { + attention_extra.push(format!( + "S10 skipped {skipped} registration ticks because targets still had their budget \ + of {} in flight — the offered rate was clamped by the platform, so the phase \ + counts understate what the run intended to submit", + scheduled::MAX_IN_FLIGHT_PER_TARGET + )); + } + routing_snapshots.push(snapshot_routing(deps, "after-recovery").await); + + // ── Account ───────────────────────────────────────────────────────────── + let settle = settle_before_readback(scheduled_config); + info!("S10: letting the last actions fall due and fire, {settle:?} before read-back"); + tokio::time::sleep(settle).await; + + let records = history.snapshot(); + let logs = scheduled::read_logs(&ctx, &targets).await; + + let report = ScheduleFireReport::build( + &records, + &logs, + scheduled_config.lead(), + fault_injected_at.map(|injected_at| FaultWindow { + injected_at, + recovered_at: fault_recovered_at, + }), + &on_pod, + scheduled_config.lease_budget(), + ); + info!( + "S10: scheduled-fire account — {} registrations accepted, {} fired once, {} never \ + fired, {} inconclusive, {} unverifiable, {} findings", + report.registrations_confirmed, + report.fired_once, + report + .findings + .iter() + .filter(|f| f.violation == crate::chaos::fires::FireViolation::NeverFired) + .count(), + report.inconclusive, + report.unverifiable, + report.findings.len() + ); + if let Some(p99) = report.fault_window_p99_ms() { + info!( + "S10: fire delay p99 during the fault, on the killed executor's targets: {p99}ms \ + against a {}ms lease budget", + report.lease_budget_ms + ); + } + + // The count-based read-back as well, on the same read. It cannot localise + // anything the token pairing does not, but it is the view every other + // scenario reports and a disagreement between the two would itself be worth + // knowing about. + let readback = readback_from_polls(&records, &logs); + + let reason = if report.has_violations() { + TerminationReason::ScheduledFireViolated { + findings: report.findings.len() as u64, + first: report + .findings + .first() + .map(|f| format!("{} on token {}", f.violation, f.token)) + .unwrap_or_default(), + } + } else if report.fired_once == 0 { + TerminationReason::StreamNeverSucceeded { + stream: Stream::Scheduled.to_string(), + } + } else { + TerminationReason::Completed + }; + + finish!(reason, &records, readback, Some(report)); +} + +/// How long to wait after the workload stops before reading the fire logs. +fn settle_before_readback(config: &ScheduledConfig) -> Duration { + config.lead() + config.lease_budget() + SETTLE_MARGIN +} + +/// Actions that were registered but not yet due when the executor died. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PendingAtInjection { + pub total: u64, + pub on_killed_executor: u64, +} + +impl PendingAtInjection { + /// The line an operator needs in order to know whether the kill landed + /// anywhere near the mechanism under test. + pub fn describe(&self) -> String { + if self.total == 0 { + return "WARNING: no scheduled action was between registration and its due time \ + when the executor died, so this run says nothing about lease recovery" + .to_string(); + } + format!( + "S10 killed the executor with {} scheduled actions registered and not yet due, {} \ + of them on targets it owned", + self.total, self.on_killed_executor + ) + } +} + +/// Counts the actions in the claim window at the moment of injection. +/// +/// Registered before the kill, due after it, and not definitively refused. A +/// refused registration is not work the platform owes anything, so counting it +/// here would overstate what the kill was aimed at. +pub fn pending_at_injection( + records: &[OperationRecord], + injected_at: DateTime, + lead: Duration, + on_killed_executor: &BTreeSet, +) -> PendingAtInjection { + let lead = TimeDelta::from_std(lead).unwrap_or(TimeDelta::zero()); + let mut pending = PendingAtInjection { + total: 0, + on_killed_executor: 0, + }; + + for record in records + .iter() + .filter(|r| r.stream == Stream::Scheduled && r.outcome != Outcome::Rejected) + .filter(|r| r.submitted_at <= injected_at && r.submitted_at + lead >= injected_at) + { + pending.total += 1; + if on_killed_executor.contains(&record.agent) { + pending.on_killed_executor += 1; + } + } + pending +} + +/// Per-target read-back from `polls`, the view every other scenario reports. +fn readback_from_polls( + records: &[OperationRecord], + logs: &[crate::chaos::fires::TargetFireLog], +) -> Vec { + logs.iter() + .filter_map(|log| { + let scoped = records + .iter() + .filter(|r| r.stream == Stream::Scheduled && r.agent == log.agent); + let observed = match (log.polls, &log.error) { + (Some(polls), _) => Ok(polls), + (None, Some(error)) => Err(error.clone()), + (None, None) => Err(format!("target {} reported no poll count", log.agent)), + }; + readback_for(Stream::Scheduled, &log.agent, scoped, observed) + }) + .collect() +} + +/// Reads the fire count of a few targets, to prove actions are firing at all. +async fn sample_fire_count(ctx: &WorkloadContext, targets: &[String]) -> u64 { + let mut total = 0u64; + for target in targets.iter().take(FIRE_PROOF_SAMPLE) { + match workload::read_polls(ctx, target).await { + Ok(polls) => total += polls, + Err(e) => warn!("S10: could not sample fires on {target}: {e}"), + } + } + total +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::chaos::history::AttemptRecord; + use test_r::test; + + fn at(offset_secs: i64) -> DateTime { + DateTime::from_timestamp(1_800_000_000 + offset_secs, 0).unwrap() + } + + fn registration(agent: &str, submitted_at: DateTime, outcome: Outcome) -> OperationRecord { + OperationRecord { + op_id: 0, + stream: Stream::Scheduled, + phase: Phase::Baseline, + agent: agent.to_string(), + method: "schedule_fire_at".to_string(), + idempotency_key: format!("{agent}-key"), + submitted_at, + completed_at: Some(submitted_at), + attempts: 1, + outcome, + duration_ms: 10, + returned_value: None, + first_attempt_value: None, + error: None, + error_class: None, + attempt_log: vec![AttemptRecord { + attempt: 1, + started_at: submitted_at, + duration_ms: 10, + returned_value: None, + succeeded: outcome == Outcome::Confirmed, + error_class: None, + error: None, + }], + } + } + + const LEAD: Duration = Duration::from_secs(10); + + /// The population the scenario is about: registered before the kill, due + /// after it. + #[test] + fn only_actions_still_inside_their_lead_count_as_pending() { + let killed = BTreeSet::from(["target-0".to_string()]); + let records = vec![ + // Due at 105, after the kill at 100. + registration("target-0", at(95), Outcome::Confirmed), + // Due at 99, so it had already fired. + registration("target-0", at(89), Outcome::Confirmed), + // Registered after the kill. + registration("target-0", at(101), Outcome::Confirmed), + ]; + let pending = pending_at_injection(&records, at(100), LEAD, &killed); + assert_eq!(pending.total, 1); + assert_eq!(pending.on_killed_executor, 1); + } + + /// The split is what makes the percentile readable, so it has to be counted + /// here too rather than inferred later. + #[test] + fn pending_actions_are_split_by_the_executor_that_owned_them() { + let killed = BTreeSet::from(["target-0".to_string()]); + let records = vec![ + registration("target-0", at(95), Outcome::Confirmed), + registration("target-1", at(95), Outcome::Confirmed), + ]; + let pending = pending_at_injection(&records, at(100), LEAD, &killed); + assert_eq!(pending.total, 2); + assert_eq!(pending.on_killed_executor, 1); + } + + /// A refusal is not work the platform owes anything, so counting it would + /// overstate what the kill was aimed at. + #[test] + fn a_refused_registration_is_not_pending_work() { + let records = vec![registration("target-0", at(95), Outcome::Rejected)]; + let pending = pending_at_injection(&records, at(100), LEAD, &BTreeSet::new()); + assert_eq!(pending.total, 0); + } + + /// The loudest thing this scenario can say: the kill missed the mechanism + /// entirely, so nothing about lease recovery can be read from the run. + #[test] + fn a_kill_that_caught_no_pending_action_says_so_rather_than_reporting_a_clean_run() { + let pending = PendingAtInjection { + total: 0, + on_killed_executor: 0, + }; + assert!(pending.describe().starts_with("WARNING")); + } + + /// The last registration falls due one lead after the workload stops, and a + /// recovery costs up to a lease on top. Reading before that would report + /// late actions as lost. + #[test] + fn the_settle_covers_a_full_lead_plus_a_full_lease_recovery() { + let config = ScheduledConfig { + targets: 100, + interval_millis: 2000, + lead_secs: 10, + lease_budget_secs: 45, + }; + assert_eq!( + settle_before_readback(&config), + Duration::from_secs(10 + 45) + SETTLE_MARGIN + ); + } +} diff --git a/integration-tests/src/chaos/scenarios/s12.rs b/integration-tests/src/chaos/scenarios/s12.rs index d69779c6f4..de3adcbed5 100644 --- a/integration-tests/src/chaos/scenarios/s12.rs +++ b/integration-tests/src/chaos/scenarios/s12.rs @@ -132,6 +132,7 @@ pub async fn run( ), termination_reason: $reason, pinned_selection: None, + scheduled_selection: None, }, ); write_outputs(&result, &history, outputs)?; diff --git a/integration-tests/src/chaos/scenarios/s13.rs b/integration-tests/src/chaos/scenarios/s13.rs index 65d65c40d6..aa16419f95 100644 --- a/integration-tests/src/chaos/scenarios/s13.rs +++ b/integration-tests/src/chaos/scenarios/s13.rs @@ -202,6 +202,7 @@ pub async fn run( summary, termination_reason: $reason, pinned_selection: None, + scheduled_selection: None, }, ); write_outputs(&result, &history, outputs)?; diff --git a/integration-tests/src/chaos/scenarios/s5.rs b/integration-tests/src/chaos/scenarios/s5.rs index 08568cdca1..9dc7b5cc45 100644 --- a/integration-tests/src/chaos/scenarios/s5.rs +++ b/integration-tests/src/chaos/scenarios/s5.rs @@ -174,6 +174,7 @@ pub async fn run( summary, termination_reason: $reason, pinned_selection: None, + scheduled_selection: None, }, ); write_outputs(&result, &history, outputs)?; diff --git a/integration-tests/src/chaos/scenarios/s8.rs b/integration-tests/src/chaos/scenarios/s8.rs index b53fde75f1..5bc824e7bb 100644 --- a/integration-tests/src/chaos/scenarios/s8.rs +++ b/integration-tests/src/chaos/scenarios/s8.rs @@ -152,6 +152,7 @@ pub async fn run( summary, termination_reason: $reason, pinned_selection: selection.clone(), + scheduled_selection: None, }, ); write_outputs(&result, &history, outputs)?; diff --git a/integration-tests/src/chaos/scheduled.rs b/integration-tests/src/chaos/scheduled.rs new file mode 100644 index 0000000000..18aae84d49 --- /dev/null +++ b/integration-tests/src/chaos/scheduled.rs @@ -0,0 +1,546 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The scheduled-registration workload (GOL-378). +//! +//! The mixed workload's scheduled stream registers a poll and counts the fires. +//! This one registers a `fire` carrying the registration's own idempotency key, +//! which is what lets [`crate::chaos::fires`] pair an individual action against +//! the invocation that asked for it. +//! +//! ## Why the cadence, and why the lead +//! +//! `lead x targets / interval` is the population that matters: actions the +//! platform has accepted and not yet run. At 100 targets registering every two +//! seconds, ten seconds ahead, that is five hundred of them standing at any +//! instant, spread across both executors' shards. A kill lands in the middle of +//! that population by construction, which is what makes the cadence and the +//! lead the two numbers that decide whether the run measures anything. +//! +//! The lead also has to comfortably exceed the workflow's inject-and-verify +//! path — signal poll, `kubectl apply`, waiting for `AllInjected` — or every +//! action registered before the kill would already have fired by the time the +//! pod died. See [`crate::chaos::scenarios::s10`] for why the narrower +//! claim-to-acknowledge window is not something the driver tries to aim at. +//! +//! ## Why the target set is split rather than pinned +//! +//! [`crate::chaos::pinned`] drives *only* the agents its chosen executor owns, +//! because an S8 operation that was not on the dead pod says nothing. Here the +//! opposite is true: the actions on the surviving executor are the control +//! group. Every target is driven, the driver names the executor owning the +//! largest share, and the report splits the two so a lease recovery that took +//! its full TTL cannot hide behind the half of the population that was never +//! disturbed. + +use crate::chaos::ScheduledConfig; +use crate::chaos::fires::{FireRecord, TargetFireLog}; +use crate::chaos::history::Stream; +use crate::chaos::pinned::{owners_by_pod, pod_ip_of}; +use crate::chaos::workload::{ + self, SCHEDULE_COUNTER_AGENT, SCHEDULE_EMITTER_AGENT, WorkloadContext, +}; +use anyhow::Context; +use chrono::{DateTime, Utc}; +use golem_common::base_model::agent::ParsedAgentId; +use golem_common::{agent_id, data_value}; +use golem_test_framework::config::{BenchmarkTestDependencies, TestDependencies}; +use golem_test_framework::dsl::TestDsl; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use std::sync::Arc; +use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use tokio::sync::Semaphore; +use tokio::task::JoinSet; +use tracing::{info, warn}; + +/// Registrations one target may have in flight at once. +/// +/// Small on purpose. The cadence is what sets the rate; this only keeps a +/// stalled platform from accumulating tasks, and a target that has spent its +/// budget skips its tick and says so rather than queueing behind itself. +pub const MAX_IN_FLIGHT_PER_TARGET: usize = 8; + +/// How many targets are read back at once. Same reasoning as +/// [`crate::chaos::scenarios::read_back_agents`]: reads do not mutate, and +/// walking them one at a time behind a per-read ceiling outlasts the +/// maintenance window. +const READ_CONCURRENCY: usize = 16; + +/// The smallest share of targets one executor must own for the run to mean +/// anything, as a divisor of the target count. +/// +/// A two-executor cluster splits a hashed population roughly evenly, so a quarter +/// is a floor rather than an expectation. Below it the "affected" group is too +/// small for its percentile to say anything, and a run that reported one anyway +/// would be worse than one that refused. +const MIN_TARGET_SHARE_DIVISOR: usize = 4; + +/// The executor the fault will be aimed at, and how the targets divide around +/// it. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ScheduledSelection { + /// The executor endpoint as the shard-manager names it, e.g. + /// `10.0.14.207:9000`. + pub pod_address: String, + /// Host part of the address, which is what a Kubernetes `status.podIP` + /// field selector matches. + pub pod_ip: String, + /// Targets this executor owns. The population whose actions have to survive + /// a lease recovery. + pub on_pod: Vec, + /// Targets owned by any other executor: the run's own control group. + pub elsewhere: Vec, + /// How the targets spread across executors, so a run that refused to + /// proceed says whether the cluster was lopsided or the pool too small. + pub targets_per_pod: BTreeMap, + /// Shard count the routing table reported. Ownership is a hash modulo this, + /// so a selection cannot be re-derived later without it. + pub number_of_shards: usize, +} + +/// Chooses the executor to aim at: the one owning the largest share of targets. +/// +/// Fails rather than proceeding unaimed. Chaos Mesh's `mode: one` would pick a +/// pod at random, and a run that killed an executor owning six targets out of a +/// hundred would still produce a confident-looking report about lease recovery. +pub async fn select( + ctx: &WorkloadContext, + deps: &BenchmarkTestDependencies, + targets: &[String], +) -> anyhow::Result { + let table = deps + .shard_manager() + .get_routing_table() + .await + .context("reading the routing table to aim the scheduled fault")?; + + let by_pod = owners_by_pod(ctx, &table, SCHEDULE_COUNTER_AGENT, targets); + let targets_per_pod: BTreeMap = by_pod + .iter() + .map(|(pod, xs)| (pod.clone(), xs.len())) + .collect(); + + let (pod_address, on_pod) = by_pod + .iter() + .max_by_key(|(_, agents)| agents.len()) + .map(|(pod, agents)| (pod.clone(), agents.clone())) + .ok_or_else(|| { + anyhow::anyhow!( + "routing table assigned none of the {} schedule targets to any executor", + targets.len() + ) + })?; + + let floor = (targets.len() / MIN_TARGET_SHARE_DIVISOR).max(1); + if on_pod.len() < floor { + anyhow::bail!( + "the most-loaded executor owns only {} of {} schedule targets, below the {floor} \ + needed for its share to be worth measuring: {targets_per_pod:?}", + on_pod.len(), + targets.len() + ); + } + + let elsewhere: Vec = targets + .iter() + .filter(|t| !on_pod.contains(t)) + .cloned() + .collect(); + + info!( + "S10: aiming at executor {pod_address}, which owns {} of {} schedule targets ({} \ + elsewhere, across {} executors)", + on_pod.len(), + targets.len(), + elsewhere.len(), + targets_per_pod.len() + ); + + Ok(ScheduledSelection { + pod_ip: pod_ip_of(&pod_address), + pod_address, + on_pod, + elsewhere, + targets_per_pod, + number_of_shards: table.number_of_shards.value, + }) +} + +/// Re-checks, against a freshly read routing table, that the targets are still +/// divided the way the selection says. +/// +/// Called immediately before the readiness signal, for the same reason S8 does +/// it: a rebalance between selection and injection would leave the run reporting +/// a control group that was actually the affected one. +pub async fn verify_ownership( + ctx: &WorkloadContext, + deps: &BenchmarkTestDependencies, + selection: &ScheduledSelection, +) -> anyhow::Result<()> { + let table = deps + .shard_manager() + .get_routing_table() + .await + .context("re-reading the routing table to verify scheduled target ownership")?; + + let mut drifted = Vec::new(); + for agent in &selection.on_pod { + let owner = table + .lookup(&crate::chaos::pinned::routing_agent_id( + ctx, + SCHEDULE_COUNTER_AGENT, + agent, + )) + .map(|pod| pod.to_string()); + if owner.as_deref() != Some(selection.pod_address.as_str()) { + drifted.push(format!( + "{agent} now owned by {}", + owner.unwrap_or_else(|| "nobody".to_string()) + )); + } + } + + if !drifted.is_empty() { + anyhow::bail!( + "{} of {} schedule targets are no longer owned by {}: {}", + drifted.len(), + selection.on_pod.len(), + selection.pod_address, + drifted.join(", ") + ); + } + + info!( + "S10: verified all {} schedule targets are still owned by {}", + selection.on_pod.len(), + selection.pod_address + ); + Ok(()) +} + +/// A running registration workload. As elsewhere, dropping the handle does not +/// stop it: call [`ScheduledHandle::stop`] so in-flight registrations record +/// themselves instead of being cancelled mid-flight. +pub struct ScheduledHandle { + stop: Arc, + tasks: JoinSet<()>, + submitted: Arc, + skipped: Arc, +} + +impl ScheduledHandle { + pub fn submitted(&self) -> u64 { + self.submitted.load(Ordering::Relaxed) + } + + /// Ticks a target dropped because it already had its budget of + /// registrations in flight. Non-zero means the platform was slow enough to + /// clamp the cadence, which is context for reading everything else. + pub fn skipped(&self) -> u64 { + self.skipped.load(Ordering::Relaxed) + } + + pub async fn stop(mut self) { + self.stop.store(1, Ordering::Relaxed); + while self.tasks.join_next().await.is_some() {} + info!( + "Chaos scheduled workload stopped after {} registrations ({} ticks skipped)", + self.submitted(), + self.skipped() + ); + } +} + +/// Starts one registration loop per target. +pub fn start( + ctx: WorkloadContext, + targets: &[String], + config: &ScheduledConfig, +) -> ScheduledHandle { + let stop = Arc::new(AtomicU8::new(0)); + let submitted = Arc::new(AtomicU64::new(0)); + let skipped = Arc::new(AtomicU64::new(0)); + let mut tasks = JoinSet::new(); + let interval = config.interval(); + let lead = config.lead(); + let count = targets.len().max(1); + + info!( + "Chaos scheduled workload starting: {} targets, one registration every {:?} each \ + ({:.1}/s overall), {:?} ahead", + targets.len(), + interval, + targets.len() as f64 / interval.as_secs_f64(), + lead + ); + + for (index, target) in targets.iter().enumerate() { + let ctx = ctx.clone(); + let stop = stop.clone(); + let submitted = submitted.clone(); + let skipped = skipped.clone(); + let target = target.clone(); + // Spread the loops across one interval so the whole population does not + // register in the same instant, which would make the offered rate a + // sawtooth instead of the constant the phase stats assume. + let stagger = interval.mul_f64(index as f64 / count as f64); + + tasks.spawn(async move { + tokio::time::sleep(stagger).await; + let budget = Arc::new(Semaphore::new(MAX_IN_FLIGHT_PER_TARGET)); + let mut in_flight = JoinSet::new(); + let mut ticker = tokio::time::interval(interval); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + let mut seq = 0u64; + + while stop.load(Ordering::Relaxed) == 0 { + ticker.tick().await; + if stop.load(Ordering::Relaxed) != 0 { + break; + } + let Ok(permit) = budget.clone().try_acquire_owned() else { + skipped.fetch_add(1, Ordering::Relaxed); + continue; + }; + submitted.fetch_add(1, Ordering::Relaxed); + let ctx = ctx.clone(); + let target = target.clone(); + let this_seq = seq; + seq += 1; + in_flight.spawn(async move { + let _permit = permit; + register_one(&ctx, index as u32, &target, this_seq, lead).await; + }); + while in_flight.try_join_next().is_some() {} + } + + // Drain rather than cancel: a registration cancelled mid-flight is + // one the history cannot classify, and during a fault those are the + // interesting ones. + while in_flight.join_next().await.is_some() {} + }); + } + + ScheduledHandle { + stop, + tasks, + submitted, + skipped, + } +} + +/// Registers one action, `lead` from now, under a key the fire will carry back. +async fn register_one(ctx: &WorkloadContext, index: u32, target: &str, seq: u64, lead: Duration) { + let emitter = ctx.agent_name(Stream::Scheduled, index); + let key = ctx.idempotency_key(target, seq); + let parsed: ParsedAgentId = agent_id!(SCHEDULE_EMITTER_AGENT, emitter); + + // Computed once, outside the retry, so a retried registration asks for the + // same instant. A due time that moved with each attempt would make the fire + // log's `scheduledMillis` disagree with what the driver believes it asked + // for, and the delay measurement would be against the wrong baseline. + let fire_at = SystemTime::now() + lead; + let since_epoch = fire_at.duration_since(UNIX_EPOCH).unwrap_or_default(); + let (secs, nanos) = (since_epoch.as_secs(), since_epoch.subsec_nanos()); + + let ctx2 = ctx.clone(); + let target2 = target.to_string(); + // Recorded against the target rather than the emitter, because the target is + // where the fire lands and where the read-back looks. + workload::run_operation( + ctx, + Stream::Scheduled, + target.to_string(), + "schedule_fire_at", + key, + |k| { + let ctx = ctx2.clone(); + let parsed = parsed.clone(); + let target = target2.clone(); + async move { + let token = k.value.clone(); + ctx.user + .invoke_and_await_agent_with_key( + &ctx.counters, + &parsed, + &k, + "schedule_fire_at", + data_value!(target, secs, nanos, token), + ) + .await?; + Ok(None) + } + }, + ) + .await; +} + +/// Creates every emitter and target before the baseline starts. +/// +/// Returns how many agents were touched. Residency matters here for the same +/// reason it does in S8: an agent that has to cold-start on first use would put +/// start-up cost inside the baseline the recovery is measured against. +pub async fn warm(ctx: &WorkloadContext, targets: &[String]) -> usize { + let mut warmed = 0usize; + for (offset, chunk) in targets.chunks(READ_CONCURRENCY).enumerate() { + let mut batch = JoinSet::new(); + for (position, target) in chunk.iter().cloned().enumerate() { + let ctx = ctx.clone(); + let index = (offset * READ_CONCURRENCY + position) as u32; + batch.spawn(async move { + let emitter = ctx.agent_name(Stream::Scheduled, index); + let parsed: ParsedAgentId = agent_id!(SCHEDULE_EMITTER_AGENT, emitter.clone()); + if let Err(e) = ctx + .user + .invoke_and_await_agent(&ctx.counters, &parsed, "warm", data_value!()) + .await + { + warn!("S10: could not warm emitter {emitter}: {e:#}"); + } + // Reading the target creates it without mutating it, which is + // exactly the side effect wanted. + if let Err(e) = workload::read_polls(&ctx, &target).await { + warn!("S10: could not warm target {target}: {e}"); + } + }); + } + while batch.join_next().await.is_some() { + warmed += 1; + } + } + warmed +} + +/// Reads every target's fire log back. +/// +/// A failed read is carried as [`TargetFireLog::error`] rather than dropped: an +/// agent that could not be read says nothing either way about its actions, and +/// the account has to be able to tell that apart from an agent that lost them. +pub async fn read_logs(ctx: &WorkloadContext, targets: &[String]) -> Vec { + let total = targets.len(); + let mut logs = Vec::with_capacity(total); + + for chunk in targets.chunks(READ_CONCURRENCY) { + let mut batch = JoinSet::new(); + for target in chunk.iter().cloned() { + let ctx = ctx.clone(); + batch.spawn(async move { + let polls = workload::read_polls(&ctx, &target).await.ok(); + let fires = workload::read_fires(&ctx, &target).await; + match fires { + Ok(raw) => TargetFireLog { + agent: target, + polls, + fires: to_fire_records(&raw), + error: None, + }, + Err(e) => TargetFireLog { + agent: target, + polls, + fires: Vec::new(), + error: Some(e), + }, + } + }); + } + + let mut batch_results = Vec::new(); + while let Some(joined) = batch.join_next().await { + match joined { + Ok(log) => batch_results.push(log), + Err(e) => warn!("S10: a fire-log read task panicked: {e}"), + } + } + batch_results.sort_by(|a, b| a.agent.cmp(&b.agent)); + logs.extend(batch_results); + info!("S10: read fire logs for {} of {total} targets", logs.len()); + } + + logs +} + +/// Turns the agent's raw triples into timestamps. +/// +/// An entry whose millis cannot be a timestamp is dropped rather than clamped. +/// Dropping leaves the log shorter than the target's own `polls`, which is what +/// the account reads as "this target cannot testify" — a clamped nonsense +/// timestamp would instead have produced a confident wrong delay. +pub fn to_fire_records(raw: &[(String, u64, u64)]) -> Vec { + let mut out = Vec::with_capacity(raw.len()); + for (token, scheduled_millis, observed_millis) in raw { + match ( + millis_to_time(*scheduled_millis), + millis_to_time(*observed_millis), + ) { + (Some(scheduled_at), Some(observed_at)) => out.push(FireRecord { + token: token.clone(), + scheduled_at, + observed_at, + }), + _ => warn!( + "S10: dropping fire log entry {token} with unreadable timestamps \ + ({scheduled_millis}, {observed_millis})" + ), + } + } + out +} + +fn millis_to_time(millis: u64) -> Option> { + DateTime::from_timestamp_millis(i64::try_from(millis).ok()?) +} + +#[cfg(test)] +mod tests { + use super::*; + use test_r::test; + + /// The happy path of the conversion the whole account is built on. + #[test] + fn raw_triples_become_timestamped_fire_records() { + let raw = vec![("t-0".to_string(), 1_800_000_000_000, 1_800_000_000_450)]; + let records = to_fire_records(&raw); + assert_eq!(records.len(), 1); + assert_eq!(records[0].token, "t-0"); + assert_eq!(records[0].delay_ms(), 450); + } + + /// An unreadable entry has to shorten the log rather than become a + /// confident wrong delay: the shortfall against `polls` is what tells the + /// account this target cannot testify. + #[test] + fn an_entry_with_an_impossible_timestamp_is_dropped_rather_than_clamped() { + let raw = vec![ + ("good".to_string(), 1_800_000_000_000, 1_800_000_000_010), + ("bad".to_string(), u64::MAX, 1_800_000_000_010), + ]; + let records = to_fire_records(&raw); + assert_eq!(records.len(), 1); + assert_eq!(records[0].token, "good"); + + let log = TargetFireLog { + agent: "target-0".to_string(), + polls: Some(2), + fires: records, + error: None, + }; + assert!( + !log.is_complete(), + "a dropped entry must leave the log short of polls" + ); + } +} diff --git a/integration-tests/src/chaos/summary.rs b/integration-tests/src/chaos/summary.rs index 79416c03fd..952adaef2b 100644 --- a/integration-tests/src/chaos/summary.rs +++ b/integration-tests/src/chaos/summary.rs @@ -48,6 +48,7 @@ //! handful of suspect keys on it, which the workflow turns into ready-made trace //! queries. One global counter would have produced a haystack instead. +use crate::chaos::fires::ScheduleFireReport; use crate::chaos::history::{Outcome, Phase, Stream}; use crate::chaos::ownership::OwnershipSample; use crate::chaos::probe::KeyProbe; @@ -503,6 +504,12 @@ pub struct ChaosSummary { /// read as "checked, nothing found". #[serde(skip_serializing_if = "Option::is_none")] pub exactly_once: Option, + /// The scheduled-fire account, for scenarios that pair scheduled actions + /// against the registrations that made them. Absent for scenarios that do + /// not, rather than an empty report that would read as "checked, nothing + /// found". + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schedule_fires: Option, /// Shard-ownership samples, in the order they were taken. Empty for /// scenarios that do not sample executor assignments. /// @@ -621,6 +628,7 @@ impl ChaosSummary { .collect(), routing_snapshots, exactly_once: None, + schedule_fires: None, ownership: Vec::new(), attention, } @@ -640,6 +648,19 @@ impl ChaosSummary { self } + /// Attaches the scheduled-fire account and hoists everything it wants a + /// human to see into [`Self::attention`]. + /// + /// More than the findings, unlike [`Self::with_exactly_once`]: an + /// unreadable target or a truncated fire log weakens every verdict the + /// report makes, and that has to be visible next to the verdicts rather + /// than only in the numbers underneath them. + pub fn with_schedule_fires(mut self, report: ScheduleFireReport) -> Self { + self.attention.extend(report.attention_lines()); + self.schedule_fires = Some(report); + self + } + /// Attaches the shard-ownership samples and hoists their findings into /// [`Self::attention`]. /// @@ -688,6 +709,12 @@ pub enum TerminationReason { /// owners is an agent whose state can fork, and there is no instant at /// which that is legitimate. ShardOwnershipViolated { findings: u64, first: String }, + /// A scheduled action the platform accepted never fired, fired twice, or + /// fired after being refused. Asserted rather than reported: unlike a + /// count-based read-back, each of these is a statement about one named + /// action paired with one named registration, with no band of doubt around + /// it. See [`crate::chaos::fires`]. + ScheduledFireViolated { findings: u64, first: String }, /// An agent's durable state did not survive a component update. Asserted /// because an update is supposed to change what an agent runs and nothing /// about what it remembers — state that moved is the one outcome an update diff --git a/integration-tests/src/chaos/workload.rs b/integration-tests/src/chaos/workload.rs index d702805951..dfce97f949 100644 --- a/integration-tests/src/chaos/workload.rs +++ b/integration-tests/src/chaos/workload.rs @@ -65,8 +65,8 @@ use tracing::{debug, info, warn}; /// Agent type names exported by the counters component. const COUNTER_AGENT: &str = "Counter"; const EPHEMERAL_COUNTER_AGENT: &str = "EphemeralCounter"; -const SCHEDULE_EMITTER_AGENT: &str = "ScheduleEmitter"; -const SCHEDULE_COUNTER_AGENT: &str = "ScheduleCounter"; +pub(crate) const SCHEDULE_EMITTER_AGENT: &str = "ScheduleEmitter"; +pub(crate) const SCHEDULE_COUNTER_AGENT: &str = "ScheduleCounter"; const PROMISE_AGENT: &str = "PromiseAgent"; const QUOTA_COUNTER_AGENT: &str = "QuotaCounter"; @@ -113,6 +113,10 @@ const ATTEMPT_TIMEOUT: Duration = Duration::from_secs(120); /// a fault and a slow answer is still an answer. const READ_TIMEOUT: Duration = Duration::from_secs(30); +/// Ceiling on one fire-log read. Larger than [`READ_TIMEOUT`] because the +/// answer carries every fire the target recorded rather than one number. +const FIRE_READ_TIMEOUT: Duration = Duration::from_secs(60); + /// Payload written when completing a promise. const PROMISE_PAYLOAD: &[u8] = b"chaos"; @@ -671,15 +675,24 @@ pub(crate) async fn submit_one(ctx: &WorkloadContext, stream: Stream, index: u32 /// A timeout is reported as an unreadable agent rather than propagated: the /// read-back already models "could not be read" as a verdict of its own, and an /// agent that will not answer is exactly that. -async fn read_with_timeout(what: &str, agent: &str, read: F) -> Result +async fn read_with_timeout(what: &str, agent: &str, read: F) -> Result +where + F: std::future::Future>, +{ + read_within(what, agent, READ_TIMEOUT, read).await +} + +/// [`read_with_timeout`] with the ceiling named by the caller, for reads whose +/// answer is much bigger than a counter. +async fn read_within(what: &str, agent: &str, timeout: Duration, read: F) -> Result where - F: std::future::Future>, + F: std::future::Future>, { - match tokio::time::timeout(READ_TIMEOUT, read).await { + match tokio::time::timeout(timeout, read).await { Ok(result) => result, Err(_) => { - warn!("Chaos: reading {what} on {agent} timed out after {READ_TIMEOUT:?}"); - Err(format!("{what} timed out after {READ_TIMEOUT:?}")) + warn!("Chaos: reading {what} on {agent} timed out after {timeout:?}"); + Err(format!("{what} timed out after {timeout:?}")) } } } @@ -780,6 +793,34 @@ pub async fn read_quota_counter(ctx: &WorkloadContext, agent: &str) -> Result Result, String> { + read_within("fires", agent, FIRE_READ_TIMEOUT, async { + let parsed: ParsedAgentId = agent_id!(SCHEDULE_COUNTER_AGENT, agent.to_string()); + match ctx + .user + .invoke_and_await_agent(&ctx.counters, &parsed, "fires", data_value!()) + .await + { + Ok(value) => value + .into_return_value() + .and_then(|v| Vec::<(String, u64, u64)>::from_value(v).ok()) + .ok_or_else(|| "fires returned no readable value".to_string()), + Err(e) => Err(format!("{e:#}")), + } + }) + .await +} + /// Reads back how many scheduled polls actually fired on a target agent. pub async fn read_polls(ctx: &WorkloadContext, agent: &str) -> Result { read_with_timeout("polls", agent, async { diff --git a/test-components/agent-counters-v2/src/lib.rs b/test-components/agent-counters-v2/src/lib.rs index 83e5a4f656..6e448fb3b0 100644 --- a/test-components/agent-counters-v2/src/lib.rs +++ b/test-components/agent-counters-v2/src/lib.rs @@ -317,6 +317,27 @@ impl EphemeralSingletonCounter for EphemeralSingletonCounterImpl { } } +/// Ceiling on the fire log below. +/// +/// Far above what a chaos run produces — a few hundred fires per target — so it +/// only stops a misconfigured cadence from growing agent state without bound. +/// `polls` keeps counting past it, which is what makes truncation visible: a +/// reader that gets fewer entries than polls knows the log is short rather than +/// the fires missing. +const MAX_FIRE_LOG: usize = 10_000; + +/// Wall clock in milliseconds since the epoch. +/// +/// Read inside the agent rather than passed in, because the question S10 asks is +/// when the *platform* ran the action. The read is durable, so an agent that +/// replays reports the original fire time instead of the replay's. +fn now_millis() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|since| since.as_millis() as u64) + .unwrap_or(0) +} + /// Near-no-op target for schedule-density. The scheduled action under test is /// dispatching this method, not its guest-side work. #[agent_definition] @@ -328,20 +349,40 @@ trait ScheduleCounter { /// increment is far cheaper than anything the dispatch itself costs. fn poll(&mut self); - /// How many times `poll` has fired. Read after recovery to compare against - /// the number of actions the driver scheduled. + /// How many scheduled actions have fired, whether through `poll` or + /// `fire`. Read after recovery to compare against the number of actions the + /// driver scheduled. fn polls(&self) -> u32; + + /// `poll`, with enough recorded to identify the individual action (GOL-378). + /// + /// S10 kills the executor holding claimed but unacknowledged scheduled + /// actions, and a count cannot answer what it asks. Two questions need the + /// individual action: did *this* registration fire, and how far past its due + /// time did lease recovery put it. `token` is the registering invocation's + /// idempotency key, so a duplicate fire is a repeated token rather than an + /// arithmetic argument about totals. + fn fire(&mut self, token: String, scheduled_millis: u64); + + /// The fire log: one `(token, scheduled_millis, observed_millis)` per fire, + /// in the order the actions ran. + fn fires(&self) -> Vec<(String, u64, u64)>; } struct ScheduleCounterImpl { _id: String, polls: u32, + fires: Vec<(String, u64, u64)>, } #[agent_implementation] impl ScheduleCounter for ScheduleCounterImpl { fn new(id: String) -> Self { - Self { _id: id, polls: 0 } + Self { + _id: id, + polls: 0, + fires: Vec::new(), + } } fn poll(&mut self) { @@ -351,6 +392,17 @@ impl ScheduleCounter for ScheduleCounterImpl { fn polls(&self) -> u32 { self.polls } + + fn fire(&mut self, token: String, scheduled_millis: u64) { + self.polls += 1; + if self.fires.len() < MAX_FIRE_LOG { + self.fires.push((token, scheduled_millis, now_millis())); + } + } + + fn fires(&self) -> Vec<(String, u64, u64)> { + self.fires.clone() + } } /// Schedules no-op polls on durable targets. Keeping scheduling separate from @@ -366,6 +418,14 @@ trait ScheduleEmitter { nanoseconds: u32, context_spans: u32, ); + + /// Registers a `fire` carrying the token that identifies this registration. + /// + /// A second method rather than a payload on `schedule_poll_at`: the density + /// benchmark and the four scenarios already on the poll path measure + /// dispatch cost, and giving their scheduled action an argument and a + /// growing log would change what they measure. + fn schedule_fire_at(&self, target_name: String, seconds: u64, nanoseconds: u32, token: String); } struct ScheduleEmitterImpl { @@ -396,6 +456,19 @@ impl ScheduleEmitter for ScheduleEmitterImpl { nanoseconds, }); } + + fn schedule_fire_at(&self, target_name: String, seconds: u64, nanoseconds: u32, token: String) { + let mut target = ScheduleCounterClient::get(target_name); + let scheduled_millis = seconds * 1000 + (nanoseconds / 1_000_000) as u64; + target.schedule_fire( + token, + scheduled_millis, + Datetime { + seconds, + nanoseconds, + }, + ); + } } #[agent_definition(ephemeral)] diff --git a/test-components/agent-counters/src/lib.rs b/test-components/agent-counters/src/lib.rs index 75a4d0511c..8c38771bd1 100644 --- a/test-components/agent-counters/src/lib.rs +++ b/test-components/agent-counters/src/lib.rs @@ -317,6 +317,27 @@ impl EphemeralSingletonCounter for EphemeralSingletonCounterImpl { } } +/// Ceiling on the fire log below. +/// +/// Far above what a chaos run produces — a few hundred fires per target — so it +/// only stops a misconfigured cadence from growing agent state without bound. +/// `polls` keeps counting past it, which is what makes truncation visible: a +/// reader that gets fewer entries than polls knows the log is short rather than +/// the fires missing. +const MAX_FIRE_LOG: usize = 10_000; + +/// Wall clock in milliseconds since the epoch. +/// +/// Read inside the agent rather than passed in, because the question S10 asks is +/// when the *platform* ran the action. The read is durable, so an agent that +/// replays reports the original fire time instead of the replay's. +fn now_millis() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|since| since.as_millis() as u64) + .unwrap_or(0) +} + /// Near-no-op target for schedule-density. The scheduled action under test is /// dispatching this method, not its guest-side work. #[agent_definition] @@ -328,20 +349,40 @@ trait ScheduleCounter { /// increment is far cheaper than anything the dispatch itself costs. fn poll(&mut self); - /// How many times `poll` has fired. Read after recovery to compare against - /// the number of actions the driver scheduled. + /// How many scheduled actions have fired, whether through `poll` or + /// `fire`. Read after recovery to compare against the number of actions the + /// driver scheduled. fn polls(&self) -> u32; + + /// `poll`, with enough recorded to identify the individual action (GOL-378). + /// + /// S10 kills the executor holding claimed but unacknowledged scheduled + /// actions, and a count cannot answer what it asks. Two questions need the + /// individual action: did *this* registration fire, and how far past its due + /// time did lease recovery put it. `token` is the registering invocation's + /// idempotency key, so a duplicate fire is a repeated token rather than an + /// arithmetic argument about totals. + fn fire(&mut self, token: String, scheduled_millis: u64); + + /// The fire log: one `(token, scheduled_millis, observed_millis)` per fire, + /// in the order the actions ran. + fn fires(&self) -> Vec<(String, u64, u64)>; } struct ScheduleCounterImpl { _id: String, polls: u32, + fires: Vec<(String, u64, u64)>, } #[agent_implementation] impl ScheduleCounter for ScheduleCounterImpl { fn new(id: String) -> Self { - Self { _id: id, polls: 0 } + Self { + _id: id, + polls: 0, + fires: Vec::new(), + } } fn poll(&mut self) { @@ -351,6 +392,17 @@ impl ScheduleCounter for ScheduleCounterImpl { fn polls(&self) -> u32 { self.polls } + + fn fire(&mut self, token: String, scheduled_millis: u64) { + self.polls += 1; + if self.fires.len() < MAX_FIRE_LOG { + self.fires.push((token, scheduled_millis, now_millis())); + } + } + + fn fires(&self) -> Vec<(String, u64, u64)> { + self.fires.clone() + } } /// Schedules no-op polls on durable targets. Keeping scheduling separate from @@ -366,6 +418,14 @@ trait ScheduleEmitter { nanoseconds: u32, context_spans: u32, ); + + /// Registers a `fire` carrying the token that identifies this registration. + /// + /// A second method rather than a payload on `schedule_poll_at`: the density + /// benchmark and the four scenarios already on the poll path measure + /// dispatch cost, and giving their scheduled action an argument and a + /// growing log would change what they measure. + fn schedule_fire_at(&self, target_name: String, seconds: u64, nanoseconds: u32, token: String); } struct ScheduleEmitterImpl { @@ -396,6 +456,19 @@ impl ScheduleEmitter for ScheduleEmitterImpl { nanoseconds, }); } + + fn schedule_fire_at(&self, target_name: String, seconds: u64, nanoseconds: u32, token: String) { + let mut target = ScheduleCounterClient::get(target_name); + let scheduled_millis = seconds * 1000 + (nanoseconds / 1_000_000) as u64; + target.schedule_fire( + token, + scheduled_millis, + Datetime { + seconds, + nanoseconds, + }, + ); + } } #[agent_definition(ephemeral)] From 3d6d349915c9a42cd85fdb5d8138004b57fb8791 Mon Sep 17 00:00:00 2001 From: kmatasfp <33095685+kmatas@users.noreply.github.com> Date: Thu, 20 Aug 2026 13:22:24 -0700 Subject: [PATCH 02/40] Separate registration stalls from scheduler delay in the S10 account --- integration-tests/src/chaos/fires.rs | 191 ++++++++++++++----- integration-tests/src/chaos/history.rs | 76 ++++++++ integration-tests/src/chaos/result.rs | 27 ++- integration-tests/src/chaos/scenarios/s10.rs | 31 ++- integration-tests/src/chaos/scheduled.rs | 3 +- 5 files changed, 259 insertions(+), 69 deletions(-) diff --git a/integration-tests/src/chaos/fires.rs b/integration-tests/src/chaos/fires.rs index b18f38ea29..b3957bcbbd 100644 --- a/integration-tests/src/chaos/fires.rs +++ b/integration-tests/src/chaos/fires.rs @@ -63,8 +63,29 @@ //! executor stamps the fire — so a small negative delay is skew rather than an //! action that fired early. `minDelayMs` is reported per group so that skew is //! visible instead of silently folded into the percentiles. +//! +//! ### Actions that were late before the scheduler saw them +//! +//! The due time is minted *before* the registering invocation goes out, so a +//! registration that takes longer than the lead to complete describes an action +//! that was already overdue the moment it was accepted. The platform runs it +//! immediately and correctly, and the arithmetic still reports it as tens of +//! seconds late. +//! +//! This is not hypothetical. The first S10 run (2026-08-20) killed an executor +//! while 124 registrations were in flight to it; 26 of them stalled on the +//! client's 120s attempt timeout and then succeeded on retry in about 60ms. All +//! 26 fired correctly and instantly, and all 26 landed in the percentiles as +//! ~115s late — 13 in each group, which made the untouched control group look +//! exactly as damaged as the killed executor's targets. +//! +//! So they are separated. A fire whose registration completed after its due time +//! is counted as `overdueOnArrival` and kept out of the delay cells, because the +//! delay cells answer "how late was the scheduler" and this is the answer to +//! "how late was the client". Both are reported; neither is allowed to +//! impersonate the other. -use crate::chaos::history::{OperationRecord, Outcome, Stream}; +use crate::chaos::history::{FireRecord, OperationRecord, Outcome, Stream, TargetFireLog}; use crate::chaos::summary::LatencyStats; use chrono::{DateTime, TimeDelta, Utc}; use serde::{Deserialize, Serialize}; @@ -79,53 +100,6 @@ use std::time::Duration; /// number of findings. const MAX_FINDINGS: usize = 200; -/// One fire, as the target agent recorded it. -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct FireRecord { - /// The registering invocation's idempotency key. - pub token: String, - /// When the action was due, as the driver asked for it. - pub scheduled_at: DateTime, - /// When the platform ran it, as the executor's clock saw it. - pub observed_at: DateTime, -} - -impl FireRecord { - /// How far past its due time the action ran. Negative means clock skew - /// between the driver and the executor, not an action that fired early. - pub fn delay_ms(&self) -> i64 { - (self.observed_at - self.scheduled_at).num_milliseconds() - } -} - -/// Everything read back from one target agent. -#[derive(Debug, Clone)] -pub struct TargetFireLog { - pub agent: String, - /// `ScheduleCounter.polls`, which keeps counting past the log's cap and is - /// therefore what says whether the log below is complete. - pub polls: Option, - pub fires: Vec, - /// Why the agent could not be read, when it could not be. - pub error: Option, -} - -impl TargetFireLog { - /// Whether this log can testify about its own registrations. - /// - /// Two ways it cannot: the read failed outright, or the component's fire log - /// hit its cap and dropped entries. Both leave an absent fire ambiguous. - pub fn is_complete(&self) -> bool { - match (self.error.is_some(), self.polls) { - (true, _) => false, - (false, Some(polls)) => self.fires.len() as u64 >= polls, - // No `polls` read means no way to tell whether the log is whole. - (false, None) => false, - } - } -} - /// The fault window, as the workflow reported it. #[derive(Debug, Clone, Copy)] pub struct FaultWindow { @@ -287,6 +261,13 @@ pub struct ScheduleFireReport { /// Targets whose fire log hit the component's cap. pub targets_truncated: Vec, pub delay: Vec, + /// Fires whose registration only landed after the action was already due, so + /// the platform could not have run them on time whatever the scheduler did. + /// Held out of [`Self::delay`] — see the module docs. + pub overdue_on_arrival: u64, + /// How late those were, for the record. This is client-side registration + /// latency wearing a scheduler's clothes. + pub overdue_delay: LatencyStats, pub findings: Vec, /// Findings past [`MAX_FINDINGS`], which the report drops rather than /// carries. Non-zero means `findings` is a sample. @@ -347,16 +328,24 @@ impl ScheduleFireReport { targets_unreadable, targets_truncated, delay: Vec::new(), + overdue_on_arrival: 0, + overdue_delay: LatencyStats::default(), findings: Vec::new(), findings_omitted: 0, }; let mut claimed: BTreeSet<&str> = BTreeSet::new(); let mut findings: Vec = Vec::new(); + // When each registration actually landed, which is what decides whether + // its action still had a chance of being on time. + let mut registered_at: BTreeMap<&str, DateTime> = BTreeMap::new(); for record in records.iter().filter(|r| r.stream == Stream::Scheduled) { let token = record.idempotency_key.as_str(); claimed.insert(token); + if let Some(completed) = record.completed_at { + registered_at.insert(token, completed); + } let fires = fires_by_token.get(token).map(Vec::as_slice).unwrap_or(&[]); // The due time comes from the fire itself when there is one, because // that is what the platform was actually told. Without a fire the @@ -432,7 +421,11 @@ impl ScheduleFireReport { report.findings_omitted = findings.len().saturating_sub(MAX_FINDINGS) as u64; findings.truncate(MAX_FINDINGS); report.findings = findings; - report.delay = delay_stats(logs, fault, killed_targets, budget_ms); + let (delay, overdue) = delay_stats(logs, fault, killed_targets, budget_ms, ®istered_at); + report.delay = delay; + report.overdue_on_arrival = overdue.len() as u64; + report.overdue_delay = + LatencyStats::from_durations(overdue.iter().map(|d| (*d).max(0) as u64).collect()); report } @@ -489,6 +482,19 @@ impl ScheduleFireReport { self.unknown_tokens )); } + if self.overdue_on_arrival > 0 { + lines.push(format!( + "{} already overdue when the registration landed (worst {}ms late). That is \ + client-side registration latency, not scheduler delay, so they are excluded \ + from the delay percentiles and counted here instead", + if self.overdue_on_arrival == 1 { + "1 action was".to_string() + } else { + format!("{} actions were", self.overdue_on_arrival) + }, + self.overdue_delay.max_ms + )); + } if let Some(p99) = self.fault_window_p99_ms() && p99 > self.lease_budget_ms { @@ -501,14 +507,20 @@ impl ScheduleFireReport { } } -/// Delay percentiles per (group, window) cell. +/// Delay percentiles per (group, window) cell, and the fires held out of them. +/// +/// A fire is held out when its registration completed after the action was +/// already due: nothing the scheduler did could have made it on time, so +/// counting it as scheduler delay would blame the platform for the client. fn delay_stats( logs: &[TargetFireLog], fault: Option, killed_targets: &BTreeSet, budget_ms: u64, -) -> Vec { + registered_at: &BTreeMap<&str, DateTime>, +) -> (Vec, Vec) { let mut cells: BTreeMap<(TargetGroup, FireWindow), Vec> = BTreeMap::new(); + let mut overdue: Vec = Vec::new(); for log in logs { let group = if killed_targets.contains(&log.agent) { @@ -517,6 +529,13 @@ fn delay_stats( TargetGroup::Elsewhere }; for fire in &log.fires { + let late_on_arrival = registered_at + .get(fire.token.as_str()) + .is_some_and(|landed| *landed > fire.scheduled_at); + if late_on_arrival { + overdue.push(fire.delay_ms()); + continue; + } cells .entry((group, FireWindow::of(fire.scheduled_at, fault))) .or_default() @@ -524,7 +543,7 @@ fn delay_stats( } } - cells + let cells: Vec = cells .into_iter() .map(|((group, window), delays)| { let min = delays.iter().copied().min().unwrap_or(0); @@ -539,7 +558,8 @@ fn delay_stats( over_budget, } }) - .collect() + .collect(); + (cells, overdue) } #[cfg(test)] @@ -883,6 +903,71 @@ mod tests { assert!(!report.has_violations()); } + /// The correction the first S10 run forced. + /// + /// An executor kill stalls the invocations in flight to it. Those + /// registrations land late, describing actions that were already overdue, + /// and the platform then runs them immediately and correctly. Counting that + /// as scheduler delay put 13 fires over budget in the killed executor's + /// targets and 13 in the control group — which is the tell, since the + /// control group's shards never moved. + #[test] + fn a_fire_whose_registration_landed_late_is_held_out_of_the_delay_cells() { + // Registered at t=0 for t=10, but the invocation only returned at t=125 + // after a stalled attempt and a retry. The action fires at once. + let mut r = record("stalled", "target-0", Outcome::Confirmed, at(0)); + r.completed_at = Some(at(125)); + r.duration_ms = 125_000; + let prompt = record("prompt", "target-0", Outcome::Confirmed, at(20)); + + let report = build( + &[r, prompt], + &[log( + "target-0", + vec![ + fire("stalled", at(10), 115_200), + fire("prompt", at(30), 1_100), + ], + )], + ); + + assert_eq!(report.overdue_on_arrival, 1); + assert_eq!(report.overdue_delay.max_ms, 115_200); + assert_eq!( + report.delay.iter().map(|d| d.delay.count).sum::(), + 1, + "only the registration that landed before its due time belongs in the cells" + ); + assert_eq!(report.delay[0].delay.max_ms, 1_100); + assert_eq!( + report.delay[0].over_budget, 0, + "the stalled registration must not put the scheduler over budget" + ); + assert!( + report + .attention_lines() + .iter() + .any(|l| l.contains("client-side registration latency")), + "an operator has to be told why the count is held out" + ); + // Still exactly-once: both actions ran once. + assert!(!report.has_violations()); + assert_eq!(report.fired_once, 2); + } + + /// The boundary: landing exactly on the due time is not late. + #[test] + fn a_registration_that_landed_on_its_due_time_stays_in_the_cells() { + let mut r = record("t-0", "target-0", Outcome::Confirmed, at(0)); + r.completed_at = Some(at(10)); + let report = build( + std::slice::from_ref(&r), + &[log("target-0", vec![fire("t-0", at(10), 900)])], + ); + assert_eq!(report.overdue_on_arrival, 0); + assert_eq!(report.delay[0].delay.count, 1); + } + /// The p99 an operator is judged on is the fault-window cell for the /// targets that were on the dead pod. With no such cell there is nothing to /// judge, and the report says nothing rather than substituting another. diff --git a/integration-tests/src/chaos/history.rs b/integration-tests/src/chaos/history.rs index 9a00ab5185..6832f5a726 100644 --- a/integration-tests/src/chaos/history.rs +++ b/integration-tests/src/chaos/history.rs @@ -314,6 +314,65 @@ impl OperationRecord { } } +/// What a scheduled action recorded when it ran, and the log it was recorded +/// in (GOL-378). +/// +/// These live here rather than in [`crate::chaos::fires`] for the same reason +/// [`OperationRecord`] does: they are what was *observed*, and the analysis +/// that reduces them is a separate thing that a later ticket may want to redo +/// over an archived run. The first S10 run learned that the expensive way — its +/// delay percentiles turned out to need a correction that could not be applied +/// afterwards, because only the reduced numbers had been archived. + +/// One fire, as the target agent recorded it. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FireRecord { + /// The registering invocation's idempotency key. + pub token: String, + /// When the action was due, as the driver asked for it. + pub scheduled_at: DateTime, + /// When the platform ran it, as the executor's clock saw it. + pub observed_at: DateTime, +} + +impl FireRecord { + /// How far past its due time the action ran. Negative means clock skew + /// between the driver and the executor, not an action that fired early. + pub fn delay_ms(&self) -> i64 { + (self.observed_at - self.scheduled_at).num_milliseconds() + } +} + +/// Everything read back from one target agent. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TargetFireLog { + pub agent: String, + /// `ScheduleCounter.polls`, which keeps counting past the log's cap and is + /// therefore what says whether the log below is complete. + pub polls: Option, + pub fires: Vec, + /// Why the agent could not be read, when it could not be. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +impl TargetFireLog { + /// Whether this log can testify about its own registrations. + /// + /// Two ways it cannot: the read failed outright, or the component's fire log + /// hit its cap and dropped entries. Both leave an absent fire ambiguous. + pub fn is_complete(&self) -> bool { + match (self.error.is_some(), self.polls) { + (true, _) => false, + (false, Some(polls)) => self.fires.len() as u64 >= polls, + // No `polls` read means no way to tell whether the log is whole. + (false, None) => false, + } + } +} + /// The persisted history document. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -324,6 +383,11 @@ pub struct HistoryDocument { /// so a reader never mistakes a partial history for a short one. pub partial: bool, pub operations: Vec, + /// Per-target fire logs, for the scenarios that drive scheduled actions. + /// Empty for every other scenario rather than absent, so a reader never + /// wonders whether the section was dropped. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub scheduled_fires: Vec, } /// Append-only operation log, shared across the concurrent workload streams. @@ -335,6 +399,7 @@ pub struct OperationHistory { scenario_code: String, inner: Arc>>, next_id: Arc, + fire_logs: Arc>>, } impl OperationHistory { @@ -343,6 +408,7 @@ impl OperationHistory { scenario_code: scenario_code.into(), inner: Arc::new(Mutex::new(Vec::new())), next_id: Arc::new(std::sync::atomic::AtomicU64::new(0)), + fire_logs: Arc::new(Mutex::new(Vec::new())), } } @@ -383,12 +449,22 @@ impl OperationHistory { .count() as u64 } + /// Archives the fire logs read back at the end of a scheduled scenario. + /// + /// Called once, after read-back, before the history is written. Kept here + /// rather than in the result because the result is the reduced report and + /// this is the raw material it was reduced from. + pub fn record_fire_logs(&self, logs: Vec) { + *self.fire_logs.lock().unwrap() = logs; + } + pub fn document(&self, partial: bool) -> HistoryDocument { HistoryDocument { schema_version: HISTORY_SCHEMA_VERSION, scenario_code: self.scenario_code.clone(), partial, operations: self.snapshot(), + scheduled_fires: self.fire_logs.lock().unwrap().clone(), } } diff --git a/integration-tests/src/chaos/result.rs b/integration-tests/src/chaos/result.rs index 99f7077611..6c70ba2f17 100644 --- a/integration-tests/src/chaos/result.rs +++ b/integration-tests/src/chaos/result.rs @@ -230,10 +230,8 @@ mod tests { /// break loudly and locally. #[test] fn an_s10_result_carries_the_schedule_fire_fields_the_investigation_report_reads() { - use crate::chaos::fires::{ - FaultWindow, FireRecord, ScheduleFireReport, TargetFireLog, - }; - use crate::chaos::history::Stream; + use crate::chaos::fires::{FaultWindow, ScheduleFireReport}; + use crate::chaos::history::{FireRecord, Stream, TargetFireLog}; let now = Utc::now(); let due = now + chrono::Duration::seconds(10); @@ -263,9 +261,7 @@ mod tests { injected_at: now, recovered_at: None, }), - &std::collections::BTreeSet::from([ - "chaos-s10-scheduled-target-0000".to_string(), - ]), + &std::collections::BTreeSet::from(["chaos-s10-scheduled-target-0000".to_string()]), std::time::Duration::from_secs(60), )); @@ -527,7 +523,8 @@ mod sample_artifact { /// driver ever writes. #[test] fn write_sample_s10_result_artifact() { - use crate::chaos::fires::{FaultWindow, FireRecord, ScheduleFireReport, TargetFireLog}; + use crate::chaos::fires::{FaultWindow, ScheduleFireReport}; + use crate::chaos::history::{FireRecord, TargetFireLog}; use crate::chaos::scheduled::ScheduledSelection; let Ok(path) = std::env::var("CHAOS_SAMPLE_RESULT_S10") else { @@ -573,6 +570,15 @@ mod sample_artifact { }); let due = submitted + chrono::Duration::seconds(10); + // One registration stalls on the client's attempt timeout and only + // lands long after its action was due — the shape the first real run + // produced 26 times over. + if i == 1 { + let stalled = records.last_mut().unwrap(); + stalled.completed_at = Some(submitted + chrono::Duration::seconds(125)); + stalled.duration_ms = 125_000; + stalled.attempts = 2; + } if target == killed { // The last one is the action the kill swallowed. if i < 6 { @@ -584,10 +590,13 @@ mod sample_artifact { }); } } else { + // The stalled registration's action fires the moment it lands, + // which the raw arithmetic calls 115s late. + let late = if i == 1 { 115_200 } else { 120 }; survivor_fires.push(FireRecord { token, scheduled_at: due, - observed_at: due + chrono::Duration::milliseconds(120), + observed_at: due + chrono::Duration::milliseconds(late), }); } } diff --git a/integration-tests/src/chaos/scenarios/s10.rs b/integration-tests/src/chaos/scenarios/s10.rs index d778bd9f99..46ad072fa2 100644 --- a/integration-tests/src/chaos/scenarios/s10.rs +++ b/integration-tests/src/chaos/scenarios/s10.rs @@ -81,9 +81,29 @@ //! is reported against the configured lease budget as SLO evidence, not //! asserted: how much a lease recovery may cost is a judgement, and the number //! that matters is in the result either way. +//! +//! ## What the first run found (2026-08-20) +//! +//! All 30,626 accepted registrations fired exactly once. Nothing was lost, +//! nothing ran twice, no target failed to testify. Reassignment cost the +//! scheduler almost nothing: the p99 fire delay on the killed executor's targets +//! during the fault was 2,158ms against a baseline of 2,134ms, and both sit just +//! under the scheduler's own 2s refresh interval, which is what a due action +//! waits for when nothing is wrong. +//! +//! The delay percentiles needed a correction to say that, and the correction is +//! now part of the account — see [`crate::chaos::fires`]. The kill stalled 26 of +//! the 124 registrations that were in flight to that executor on the client's +//! 120s attempt timeout; each retried and succeeded in about 60ms. Those +//! actions fired immediately and correctly, and arrived in the percentiles as +//! ~115s late, evenly split between the two groups because emitters are spread +//! across both executors. The stall itself is the defect golemcloud/golem#3748 +//! addresses, reproduced here on the registration path. use crate::chaos::fires::{FaultWindow, ScheduleFireReport}; -use crate::chaos::history::{OperationHistory, OperationRecord, Outcome, Phase, Stream}; +use crate::chaos::history::{ + OperationHistory, OperationRecord, Outcome, Phase, Stream, TargetFireLog, +}; use crate::chaos::prep::ChaosPrepManifest; use crate::chaos::result::{ChaosResult, PhaseWindow, Phases, RunScope}; use crate::chaos::scenarios::{ @@ -402,6 +422,10 @@ pub async fn run( let records = history.snapshot(); let logs = scheduled::read_logs(&ctx, &targets).await; + // Archived alongside the operations, not just reduced into the report. The + // first S10 run needed a correction to its delay percentiles that could not + // be applied afterwards, because only the reduced numbers had been kept. + history.record_fire_logs(logs.clone()); let report = ScheduleFireReport::build( &records, @@ -522,10 +546,7 @@ pub fn pending_at_injection( } /// Per-target read-back from `polls`, the view every other scenario reports. -fn readback_from_polls( - records: &[OperationRecord], - logs: &[crate::chaos::fires::TargetFireLog], -) -> Vec { +fn readback_from_polls(records: &[OperationRecord], logs: &[TargetFireLog]) -> Vec { logs.iter() .filter_map(|log| { let scoped = records diff --git a/integration-tests/src/chaos/scheduled.rs b/integration-tests/src/chaos/scheduled.rs index 18aae84d49..ced3104d8f 100644 --- a/integration-tests/src/chaos/scheduled.rs +++ b/integration-tests/src/chaos/scheduled.rs @@ -45,8 +45,7 @@ //! disturbed. use crate::chaos::ScheduledConfig; -use crate::chaos::fires::{FireRecord, TargetFireLog}; -use crate::chaos::history::Stream; +use crate::chaos::history::{FireRecord, Stream, TargetFireLog}; use crate::chaos::pinned::{owners_by_pod, pod_ip_of}; use crate::chaos::workload::{ self, SCHEDULE_COUNTER_AGENT, SCHEDULE_EMITTER_AGENT, WorkloadContext, From c712f9367bd361dca363ac4e1c9b85c8fa114042 Mon Sep 17 00:00:00 2001 From: kmatasfp <33095685+kmatas@users.noreply.github.com> Date: Thu, 20 Aug 2026 13:38:19 -0700 Subject: [PATCH 03/40] Note which cut of the connect-timeout branch golem-dev runs --- integration-tests/src/chaos/scenarios/s10.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/integration-tests/src/chaos/scenarios/s10.rs b/integration-tests/src/chaos/scenarios/s10.rs index 46ad072fa2..abf986325d 100644 --- a/integration-tests/src/chaos/scenarios/s10.rs +++ b/integration-tests/src/chaos/scenarios/s10.rs @@ -97,8 +97,17 @@ //! 120s attempt timeout; each retried and succeeded in about 60ms. Those //! actions fired immediately and correctly, and arrived in the percentiles as //! ~115s late, evenly split between the two groups because emitters are spread -//! across both executors. The stall itself is the defect golemcloud/golem#3748 -//! addresses, reproduced here on the registration path. +//! across both executors. +//! +//! The stall is not a new finding, and it is not an unfixed one either. golem-dev +//! runs `v1.5.10-dev.1`, which is a tag on the branch of golemcloud/golem#3748 +//! rather than on `1.5.x`, so the cluster already carries the first half of that +//! work. It was cut on 2026-08-19, and ten further commits to the gRPC client +//! landed on that branch the next day. One of them, `edd668f4c`, is this exact +//! case: a pod killed with requests in flight reports most of them `Cancelled`, +//! the connection was never retired, and every later request queued onto a +//! channel that could never work again. Read a stall in an S10 run against which +//! cut of that branch the images were built before treating it as a regression. use crate::chaos::fires::{FaultWindow, ScheduleFireReport}; use crate::chaos::history::{ From 82890c8e81129114640d1584c337f327f6bd6f6a Mon Sep 17 00:00:00 2001 From: kmatasfp <33095685+kmatas@users.noreply.github.com> Date: Thu, 20 Aug 2026 15:05:55 -0700 Subject: [PATCH 04/40] Record the v1.5.10-dev.2 S10 rerun in the scenario docs --- integration-tests/src/chaos/scenarios/s10.rs | 46 ++++++++++++++++++-- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/integration-tests/src/chaos/scenarios/s10.rs b/integration-tests/src/chaos/scenarios/s10.rs index abf986325d..18ef49c536 100644 --- a/integration-tests/src/chaos/scenarios/s10.rs +++ b/integration-tests/src/chaos/scenarios/s10.rs @@ -100,14 +100,52 @@ //! across both executors. //! //! The stall is not a new finding, and it is not an unfixed one either. golem-dev -//! runs `v1.5.10-dev.1`, which is a tag on the branch of golemcloud/golem#3748 -//! rather than on `1.5.x`, so the cluster already carries the first half of that +//! ran `v1.5.10-dev.1`, which is a tag on the branch of golemcloud/golem#3748 +//! rather than on `1.5.x`, so the cluster already carried the first half of that //! work. It was cut on 2026-08-19, and ten further commits to the gRPC client //! landed on that branch the next day. One of them, `edd668f4c`, is this exact //! case: a pod killed with requests in flight reports most of them `Cancelled`, //! the connection was never retired, and every later request queued onto a -//! channel that could never work again. Read a stall in an S10 run against which -//! cut of that branch the images were built before treating it as a regression. +//! channel that could never work again. +//! +//! ## What the second run showed (2026-08-20, `v1.5.10-dev.2`) +//! +//! The scenario was re-run unchanged against a cluster rebuilt from the tip of +//! that branch, which is the ten client commits the first run's images predated. +//! Same driver, same 100 targets, same 2s interval and 10s lead; the only thing +//! that moved was the deployed image. It is the cleanest reading this harness +//! has produced, because the fault landed comparably — 353 actions pending at +//! the kill against the first run's 413 — and the undisturbed baseline windows +//! did not move at all (p99 2,132ms and 2,101ms, against 2,134ms and 2,142ms). +//! +//! Exactly-once held, as it had before: 30,730 of 30,730. The stall did not. +//! +//! | | dev.1 | dev.2 | +//! | -- | -- | -- | +//! | registrations over 20s | 32 | 0 | +//! | slowest registration | 125,188ms | 12,891ms | +//! | slow registrations *submitted after* the kill | 79 | 0 | +//! | fire delay past the 60s budget | 26 | 0 | +//! | worst fire delay, killed executor's targets | 117,135ms | 2,177ms | +//! | worst fire delay, control group | 117,095ms | 3,205ms | +//! +//! The last of those rows is the one that identifies the defect rather than +//! merely measuring it. Under dev.1 the damage was not confined to calls that +//! were riding the dying connection: 79 registrations submitted *after* the pod +//! was already gone also stalled, because the dead channel stayed in the cache +//! and kept accepting work. Under dev.2 that number is zero. Only the calls +//! actually in flight at the moment of the kill paid anything, which is the +//! shape a correctly retired connection produces. +//! +//! What remains is not a hang. The 62 registrations in flight at the kill took +//! 10.0-12.9s each, on a single attempt, and that is inside the band the rest of +//! the cluster pays for losing an executor. It does exceed the 10s lead, so those +//! actions were due before their registration landed and are reported as +//! `overdue_on_arrival` rather than as scheduler delay. Note that the count went +//! *up* against dev.1's 26 while the magnitude fell by an order of magnitude: +//! the classification counts anything slower than the lead, so shrinking a 125s +//! stall to 13s moves entries into it rather than out. Read the two numbers +//! together, never the count alone. use crate::chaos::fires::{FaultWindow, ScheduleFireReport}; use crate::chaos::history::{ From f20e445bba4190abe7e5c35cb62c7ca14a2a0451 Mon Sep 17 00:00:00 2001 From: kmatasfp <33095685+kmatas@users.noreply.github.com> Date: Thu, 20 Aug 2026 15:23:54 -0700 Subject: [PATCH 05/40] Separate scenario findings from run context in chaos summaries --- golem-test-framework/src/benchmark/results.rs | 14 ++ integration-tests/src/chaos/fires.rs | 41 +++-- integration-tests/src/chaos/history.rs | 4 +- integration-tests/src/chaos/result.rs | 58 +++++++ integration-tests/src/chaos/scenarios/mod.rs | 15 +- integration-tests/src/chaos/scenarios/s1.rs | 16 +- integration-tests/src/chaos/scenarios/s10.rs | 42 +++++- integration-tests/src/chaos/scenarios/s13.rs | 12 +- integration-tests/src/chaos/scenarios/s5.rs | 38 +++-- integration-tests/src/chaos/summary.rs | 141 +++++++++++++++++- 10 files changed, 331 insertions(+), 50 deletions(-) diff --git a/golem-test-framework/src/benchmark/results.rs b/golem-test-framework/src/benchmark/results.rs index 15785319ae..9ccd9685a7 100644 --- a/golem-test-framework/src/benchmark/results.rs +++ b/golem-test-framework/src/benchmark/results.rs @@ -525,6 +525,17 @@ pub struct RunMetadata { /// Container image tag of the deployed `worker-service`. #[serde(skip_serializing_if = "Option::is_none", default)] pub worker_service_image_tag: Option, + /// Image digest of the deployed `worker-executor`, when the manifest pins + /// one. A tag can be moved; the digest is what actually identifies the + /// build a run tested, which is the whole point of recording it. + #[serde(skip_serializing_if = "Option::is_none", default)] + pub worker_executor_image_digest: Option, + /// Image digest of the deployed `registry-service`. + #[serde(skip_serializing_if = "Option::is_none", default)] + pub registry_service_image_digest: Option, + /// Image digest of the deployed `worker-service`. + #[serde(skip_serializing_if = "Option::is_none", default)] + pub worker_service_image_digest: Option, /// Aurora ACU capacity for the main (`golem_dev`) cluster at run start. #[serde(skip_serializing_if = "Option::is_none", default)] pub aurora_acu_main: Option, @@ -575,6 +586,9 @@ impl RunMetadata { worker_executor_image_tag: env_str("GOLEM_BENCH_WORKER_EXECUTOR_IMAGE_TAG"), registry_service_image_tag: env_str("GOLEM_BENCH_REGISTRY_SERVICE_IMAGE_TAG"), worker_service_image_tag: env_str("GOLEM_BENCH_WORKER_SERVICE_IMAGE_TAG"), + worker_executor_image_digest: env_str("GOLEM_BENCH_WORKER_EXECUTOR_IMAGE_DIGEST"), + registry_service_image_digest: env_str("GOLEM_BENCH_REGISTRY_SERVICE_IMAGE_DIGEST"), + worker_service_image_digest: env_str("GOLEM_BENCH_WORKER_SERVICE_IMAGE_DIGEST"), aurora_acu_main: env_f64("GOLEM_BENCH_AURORA_ACU_MAIN"), aurora_acu_indexed: env_f64("GOLEM_BENCH_AURORA_ACU_INDEXED"), aurora_acu_keyvalue: env_f64("GOLEM_BENCH_AURORA_ACU_KEYVALUE"), diff --git a/integration-tests/src/chaos/fires.rs b/integration-tests/src/chaos/fires.rs index b3957bcbbd..2911aa7114 100644 --- a/integration-tests/src/chaos/fires.rs +++ b/integration-tests/src/chaos/fires.rs @@ -482,6 +482,31 @@ impl ScheduleFireReport { self.unknown_tokens )); } + if let Some(p99) = self.fault_window_p99_ms() + && p99 > self.lease_budget_ms + { + lines.push(format!( + "scheduled-fire p99 during the fault was {p99}ms against a {}ms lease budget", + self.lease_budget_ms + )); + } + lines + } + + /// Lines that explain the account without claiming anything is wrong. + /// + /// Overdue-on-arrival lives here rather than in [`Self::attention_lines`] + /// because it describes the client, not the platform: the due time is + /// minted before the registering invocation goes out, so a slow + /// registration describes an action that was already late when it arrived + /// and which the scheduler then ran correctly and at once. + /// + /// Read its count together with the worst case, never alone. The + /// classification catches anything slower than the lead, so a fix that + /// shortens a two-minute stall to twelve seconds moves entries *into* this + /// count while making the platform strictly better. + pub fn note_lines(&self) -> Vec { + let mut lines = Vec::new(); if self.overdue_on_arrival > 0 { lines.push(format!( "{} already overdue when the registration landed (worst {}ms late). That is \ @@ -495,14 +520,6 @@ impl ScheduleFireReport { self.overdue_delay.max_ms )); } - if let Some(p99) = self.fault_window_p99_ms() - && p99 > self.lease_budget_ms - { - lines.push(format!( - "scheduled-fire p99 during the fault was {p99}ms against a {}ms lease budget", - self.lease_budget_ms - )); - } lines } } @@ -945,11 +962,17 @@ mod tests { ); assert!( report - .attention_lines() + .note_lines() .iter() .any(|l| l.contains("client-side registration latency")), "an operator has to be told why the count is held out" ); + assert!( + report.attention_lines().is_empty(), + "holding a fire out of the cells is context, not a finding: CI raises an \ + annotation on attention, and a run where the platform did nothing wrong must \ + not raise one" + ); // Still exactly-once: both actions ran once. assert!(!report.has_violations()); assert_eq!(report.fired_once, 2); diff --git a/integration-tests/src/chaos/history.rs b/integration-tests/src/chaos/history.rs index 6832f5a726..2abaaf3bce 100644 --- a/integration-tests/src/chaos/history.rs +++ b/integration-tests/src/chaos/history.rs @@ -323,8 +323,8 @@ impl OperationRecord { /// over an archived run. The first S10 run learned that the expensive way — its /// delay percentiles turned out to need a correction that could not be applied /// afterwards, because only the reduced numbers had been archived. - -/// One fire, as the target agent recorded it. +/// +/// This one is a single fire, as the target agent recorded it. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct FireRecord { diff --git a/integration-tests/src/chaos/result.rs b/integration-tests/src/chaos/result.rs index 6c70ba2f17..2c58b1f544 100644 --- a/integration-tests/src/chaos/result.rs +++ b/integration-tests/src/chaos/result.rs @@ -282,6 +282,8 @@ mod tests { "delay", "findings", "findingsOmitted", + "overdueOnArrival", + "overdueDelay", ] { assert!( !fires[key].is_null(), @@ -303,6 +305,62 @@ mod tests { assert!(!Stream::Scheduled.to_string().is_empty()); } + /// The CI annotation branches on `summary.attention` being non-empty, so + /// the two lists have to stay two lists across the repo boundary. Folding + /// context back into `attention` would make the annotation fire on every + /// healthy run, which is how it came to mean nothing the first time. + #[test] + fn a_result_separates_findings_from_context_for_the_ci_annotation() { + use crate::chaos::summary::Note; + + let mut result = sample_result(TerminationReason::Completed); + result.summary.absorb([ + Note::context("routing at start: 1024/1024 shards (settled before measuring)"), + Note::attention("4 scheduled targets filled their fire log and dropped entries"), + ]); + + let json = serde_json::to_value(&result).unwrap(); + assert_eq!( + json["summary"]["attention"].as_array().unwrap().len(), + 1, + "summary.attention is what --attention-count counts" + ); + assert_eq!( + json["summary"]["notes"].as_array().unwrap().len(), + 1, + "summary.notes is what the report renders as run context" + ); + + let parsed: ChaosResult = serde_json::from_str(&json.to_string()).unwrap(); + assert_eq!(parsed.summary.attention.len(), 1); + assert_eq!(parsed.summary.notes.len(), 1); + } + + /// A tag can be moved; a digest identifies the build a run actually tested. + /// The workflow emits both, and the runbook tells a reader to match them + /// against the deployment manifests, so both key names are load-bearing. + #[test] + fn run_metadata_carries_the_image_digest_beside_the_tag() { + use golem_test_framework::benchmark::RunMetadata; + + let mut result = sample_result(TerminationReason::Completed); + result.run_metadata = Some(RunMetadata { + worker_executor_image_tag: Some("v1.5.10-dev.2".to_string()), + worker_executor_image_digest: Some("sha256:60eac87a".to_string()), + ..Default::default() + }); + + let json = serde_json::to_value(&result).unwrap(); + assert_eq!( + json["runMetadata"]["workerExecutorImageTag"], + "v1.5.10-dev.2" + ); + assert_eq!( + json["runMetadata"]["workerExecutorImageDigest"], + "sha256:60eac87a" + ); + } + #[test] fn result_round_trips_through_json() { let result = sample_result(TerminationReason::Completed); diff --git a/integration-tests/src/chaos/scenarios/mod.rs b/integration-tests/src/chaos/scenarios/mod.rs index 8ff89b5af7..43f71084d6 100644 --- a/integration-tests/src/chaos/scenarios/mod.rs +++ b/integration-tests/src/chaos/scenarios/mod.rs @@ -40,7 +40,7 @@ use crate::chaos::result::{ChaosResult, Phases, RESULT_SCHEMA_VERSION, RunScope} use crate::chaos::scheduled::ScheduledSelection; use crate::chaos::signal::SignalError; use crate::chaos::summary::{ - AgentReadback, ChaosSummary, ExactlyOnceReport, RoutingSnapshot, TerminationReason, + AgentReadback, ChaosSummary, ExactlyOnceReport, Note, RoutingSnapshot, TerminationReason, }; use crate::chaos::workload::{self, WorkloadContext}; use chrono::{DateTime, Utc}; @@ -208,11 +208,14 @@ const ROUTING_POLL_SECS: u64 = 3; /// Blocks until the routing table covers every shard, or the timeout lapses. /// /// Returns the line to record, so the result says which of the two happened -/// rather than leaving a reader to infer it from timings. +/// rather than leaving a reader to infer it from timings. A settled table is +/// context — it is what every healthy run reports. An unsettled one is a +/// finding, because the baseline then measures convergence rather than the +/// platform. pub async fn wait_for_settled_routing( deps: &BenchmarkTestDependencies, snapshots: &mut Vec, -) -> String { +) -> Note { let deadline = std::time::Instant::now() + std::time::Duration::from_secs(ROUTING_SETTLE_TIMEOUT_SECS); // Assigned on every path through the loop below before it is read. @@ -231,7 +234,7 @@ pub async fn wait_for_settled_routing( if assigned == total && executors > 0 { snapshots.push(snapshot_routing(deps, "settled-before-start").await); info!("Chaos: {last} — settled"); - return format!("{last} (settled before measuring)"); + return Note::context(format!("{last} (settled before measuring)")); } info!("Chaos: {last} — waiting for the table to cover every shard"); } @@ -244,11 +247,11 @@ pub async fn wait_for_settled_routing( if std::time::Instant::now() >= deadline { snapshots.push(snapshot_routing(deps, "unsettled-before-start").await); warn!("Chaos: {last} — proceeding anyway after {ROUTING_SETTLE_TIMEOUT_SECS}s"); - return format!( + return Note::attention(format!( "WARNING: measured against an unsettled cluster — {last}. \ Baseline numbers may reflect routing convergence rather than the \ platform." - ); + )); } tokio::time::sleep(std::time::Duration::from_secs(ROUTING_POLL_SECS)).await; } diff --git a/integration-tests/src/chaos/scenarios/s1.rs b/integration-tests/src/chaos/scenarios/s1.rs index da28910661..0d858db27c 100644 --- a/integration-tests/src/chaos/scenarios/s1.rs +++ b/integration-tests/src/chaos/scenarios/s1.rs @@ -76,7 +76,9 @@ use crate::chaos::scenarios::{ wait_for_settled_routing, warm_up, write_outputs, }; use crate::chaos::signal::{BaselineReady, FaultSignals, ScaleEvent}; -use crate::chaos::summary::{AgentReadback, ChaosSummary, ExactlyOnceReport, TerminationReason}; +use crate::chaos::summary::{ + AgentReadback, ChaosSummary, ExactlyOnceReport, Note, TerminationReason, +}; use crate::chaos::workload::{self, PhaseMarker, WorkloadContext}; use crate::chaos::{ScenarioCode, ScenarioConfig}; use chrono::Utc; @@ -162,7 +164,7 @@ pub async fn run( let mut fault_id = None; let mut fault_target_observed = None; let mut inconclusive: Option = None; - let mut attention_extra: Vec = Vec::new(); + let mut attention_extra: Vec = Vec::new(); // Sample the assignment continuously for the whole run, alongside the // labelled phase-boundary samples. See ASSIGNMENT_SAMPLE_INTERVAL. @@ -216,7 +218,7 @@ pub async fn run( if let Some(detail) = inconclusive.clone() { summary.attention.push(detail); } - summary.attention.extend(attention_extra.clone()); + summary.absorb(attention_extra.clone()); if let Some(report) = $exactly_once { summary = summary.with_exactly_once(report); } @@ -558,17 +560,17 @@ pub async fn run( holds {smallest} against a balanced {balanced}" ); if executors < expected { - attention_extra.push(format!( + attention_extra.push(Note::attention(format!( "executors were scaled back to {expected} during the fault, but only \ {executors} hold shards after settling — the cluster did not take the \ restored executor back" - )); + ))); } else if smallest * 2 < balanced { - attention_extra.push(format!( + attention_extra.push(Note::attention(format!( "after settling the least-loaded executor holds {smallest} shards against a \ balanced {balanced}: the cluster took the executor back but has not \ rebalanced onto it" - )); + ))); } } diff --git a/integration-tests/src/chaos/scenarios/s10.rs b/integration-tests/src/chaos/scenarios/s10.rs index 18ef49c536..2852f79ff5 100644 --- a/integration-tests/src/chaos/scenarios/s10.rs +++ b/integration-tests/src/chaos/scenarios/s10.rs @@ -159,7 +159,7 @@ use crate::chaos::scenarios::{ }; use crate::chaos::scheduled::{self, ScheduledSelection}; use crate::chaos::signal::{BaselineReady, FaultSignals, FaultTarget}; -use crate::chaos::summary::{AgentReadback, ChaosSummary, TerminationReason}; +use crate::chaos::summary::{AgentReadback, ChaosSummary, Note, TerminationReason}; use crate::chaos::workload::{self, PhaseMarker, WorkloadContext}; use crate::chaos::{ScenarioCode, ScenarioConfig, ScheduledConfig}; use chrono::{DateTime, TimeDelta, Utc}; @@ -236,7 +236,7 @@ pub async fn run( let mut fault_id = None; let mut fault_target_observed = None; let mut selection: Option = None; - let mut attention_extra: Vec = Vec::new(); + let mut attention_extra: Vec = Vec::new(); macro_rules! finish { ($reason:expr, $records:expr, $readback:expr, $fires:expr) => {{ @@ -246,7 +246,7 @@ pub async fn run( routing_snapshots.clone(), fault_injected_at, ); - summary.attention.extend(attention_extra.clone()); + summary.absorb(attention_extra.clone()); if let Some(report) = $fires { summary = summary.with_schedule_fires(report); } @@ -418,7 +418,7 @@ pub async fn run( scheduled_config.lead(), &on_pod, ); - attention_extra.push(pending.describe()); + attention_extra.push(pending.note()); info!("S10: {}", pending.describe()); let recovered = match signals.await_fault_recovered(config.signal_timeout()).await { @@ -453,12 +453,12 @@ pub async fn run( window.end(Utc::now()); } if skipped > 0 { - attention_extra.push(format!( + attention_extra.push(Note::attention(format!( "S10 skipped {skipped} registration ticks because targets still had their budget \ of {} in flight — the offered rate was clamped by the platform, so the phase \ counts understate what the run intended to submit", scheduled::MAX_IN_FLIGHT_PER_TARGET - )); + ))); } routing_snapshots.push(snapshot_routing(deps, "after-recovery").await); @@ -546,6 +546,19 @@ pub struct PendingAtInjection { } impl PendingAtInjection { + /// Whether the kill landed anywhere near the mechanism under test. + /// + /// A run that caught nothing pending proves nothing however clean the rest + /// of its numbers look, which is the one thing here a human has to act on. + pub fn needs_attention(&self) -> bool { + self.total == 0 + } + + /// The same line as [`Self::describe`], classified. + pub fn note(&self) -> Note { + Note::leveled(self.needs_attention(), self.describe()) + } + /// The line an operator needs in order to know whether the kill landed /// anywhere near the mechanism under test. pub fn describe(&self) -> String { @@ -625,6 +638,7 @@ async fn sample_fire_count(ctx: &WorkloadContext, targets: &[String]) -> u64 { mod tests { use super::*; use crate::chaos::history::AttemptRecord; + use crate::chaos::summary::NoteLevel; use test_r::test; fn at(offset_secs: i64) -> DateTime { @@ -712,6 +726,22 @@ mod tests { on_killed_executor: 0, }; assert!(pending.describe().starts_with("WARNING")); + assert!(pending.needs_attention()); + assert_eq!(pending.note().level, NoteLevel::Attention); + } + + /// The same sentence on a run that landed properly is context. It is the + /// first thing a reader wants and it is true of every healthy run, so + /// putting it in `attention` would fire CI's annotation every time. + #[test] + fn a_kill_that_landed_reports_its_count_as_context() { + let pending = PendingAtInjection { + total: 353, + on_killed_executor: 226, + }; + assert!(!pending.needs_attention()); + assert_eq!(pending.note().level, NoteLevel::Context); + assert!(pending.note().message.contains("353")); } /// The last registration falls due one lead after the workload stops, and a diff --git a/integration-tests/src/chaos/scenarios/s13.rs b/integration-tests/src/chaos/scenarios/s13.rs index aa16419f95..5e8f98ddda 100644 --- a/integration-tests/src/chaos/scenarios/s13.rs +++ b/integration-tests/src/chaos/scenarios/s13.rs @@ -69,7 +69,8 @@ use crate::chaos::scenarios::{ }; use crate::chaos::signal::{BaselineReady, FaultSignals, RestartEvent}; use crate::chaos::summary::{ - AgentReadback, ChaosSummary, ExactlyOnceReport, TerminationReason, stream_that_never_succeeded, + AgentReadback, ChaosSummary, ExactlyOnceReport, Note, TerminationReason, + stream_that_never_succeeded, }; use crate::chaos::workload::{self, PhaseMarker, WorkloadContext}; use crate::chaos::{ScenarioCode, ScenarioConfig}; @@ -138,7 +139,7 @@ pub async fn run( let mut fault_recovered_at = None; let mut fault_id = None; let mut fault_target_observed = None; - let mut attention_extra: Vec = Vec::new(); + let mut attention_extra: Vec = Vec::new(); // Sample the assignment continuously for the whole run. Five rebalances in // five minutes cannot be read from phase boundaries. @@ -185,7 +186,7 @@ pub async fn run( fault_injected_at, ); summary.ownership = samples; - summary.attention.extend(attention_extra.clone()); + summary.absorb(attention_extra.clone()); if let Some(report) = $exactly_once { summary = summary.with_exactly_once(report); } @@ -296,7 +297,10 @@ pub async fn run( } let restarts = signals.read_restart_events(); - attention_extra.push(describe_restarts(&restarts)); + attention_extra.push(Note::leveled( + restarts.is_empty(), + describe_restarts(&restarts), + )); for event in &restarts { info!( "S13: restart {} at {} ({})", diff --git a/integration-tests/src/chaos/scenarios/s5.rs b/integration-tests/src/chaos/scenarios/s5.rs index 9dc7b5cc45..afdce5e602 100644 --- a/integration-tests/src/chaos/scenarios/s5.rs +++ b/integration-tests/src/chaos/scenarios/s5.rs @@ -76,7 +76,8 @@ use crate::chaos::scenarios::{ }; use crate::chaos::signal::{BaselineReady, FaultSignals}; use crate::chaos::summary::{ - AgentReadback, ChaosSummary, ReadbackVerdict, TerminationReason, stream_that_never_succeeded, + AgentReadback, ChaosSummary, Note, ReadbackVerdict, TerminationReason, + stream_that_never_succeeded, }; use crate::chaos::workload::{self, PhaseMarker, WorkloadContext}; use crate::chaos::{ScenarioCode, ScenarioConfig}; @@ -150,7 +151,7 @@ pub async fn run( let mut fault_recovered_at = None; let mut fault_id = None; let mut fault_target_observed = None; - let mut attention_extra: Vec = Vec::new(); + let mut attention_extra: Vec = Vec::new(); macro_rules! finish { ($reason:expr, $records:expr, $readback:expr) => {{ @@ -160,7 +161,7 @@ pub async fn run( routing_snapshots.clone(), fault_injected_at, ); - summary.attention.extend(attention_extra.clone()); + summary.absorb(attention_extra.clone()); let result = build_result( config, ScenarioOutcome { @@ -255,10 +256,13 @@ pub async fn run( ); let requested = request_updates(&ctx, workload_config, target_revision).await; - attention_extra.push(format!( - "update to revision {target_revision} requested for {requested} of {} durable agents \ - at {update_started_at}", - workload_config.durable_agents + attention_extra.push(Note::leveled( + requested < workload_config.durable_agents as usize, + format!( + "update to revision {target_revision} requested for {requested} of {} durable \ + agents at {update_started_at}", + workload_config.durable_agents + ), )); // ── Signal: ready for the fault ───────────────────────────────────────── @@ -291,9 +295,9 @@ pub async fn run( "S5: fault {} ({} on {}) reported active at {}, {into_update}ms into the update", injected.fault_id, injected.kind, injected.target, injected.injected_at ); - attention_extra.push(format!( + attention_extra.push(Note::context(format!( "the executor kill landed {into_update}ms into the update" - )); + ))); fault_injected_at = Some(injected.injected_at); fault_id = Some(injected.fault_id.clone()); fault_target_observed = Some(injected.target.clone()); @@ -343,12 +347,16 @@ pub async fn run( .map(|(agent, _)| agent) .collect(); let unreadable = versions.values().filter(|v| v.is_none()).count(); - attention_extra.push(format!( - "after recovery {} of {} durable agents report component version {}; {} could not be read", - versions.len() - stale.len(), - versions.len(), - EXPECTED_VERSION_AFTER_UPDATE, - unreadable + attention_extra.push(Note::leveled( + !stale.is_empty() || unreadable > 0, + format!( + "after recovery {} of {} durable agents report component version {}; {} could not \ + be read", + versions.len() - stale.len(), + versions.len(), + EXPECTED_VERSION_AFTER_UPDATE, + unreadable + ), )); // ── Verdict ───────────────────────────────────────────────────────────── diff --git a/integration-tests/src/chaos/summary.rs b/integration-tests/src/chaos/summary.rs index 952adaef2b..b4f9c73de4 100644 --- a/integration-tests/src/chaos/summary.rs +++ b/integration-tests/src/chaos/summary.rs @@ -485,6 +485,62 @@ impl ExactlyOnceReport { } } +/// Whether a line a scenario reports is a finding or context. +/// +/// The distinction exists because CI branches on it. An annotation that fires +/// on every run — and one fires on every run if "routing settled before we +/// measured" counts as something needing review — trains its readers to ignore +/// it, which is worse than not having it. Context still reaches the report; it +/// just does not claim a human has to act. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NoteLevel { + /// Something a human should look at before trusting the run. + Attention, + /// Something a human needs in order to read the run, but which is not + /// itself a problem. + Context, +} + +/// One operator-facing line, and which of the two lists it belongs in. +/// +/// Scenarios build these as they go and hand the whole batch to +/// [`ChaosSummary::absorb`] at the end, so the classification lives next to the +/// condition that produced it rather than in whatever reads the result later. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Note { + pub level: NoteLevel, + pub message: String, +} + +impl Note { + /// A line that means a human should look before trusting the run. + pub fn attention(message: impl Into) -> Self { + Self { + level: NoteLevel::Attention, + message: message.into(), + } + } + + /// A line that a human needs in order to read the run, but which is not + /// itself a problem. + pub fn context(message: impl Into) -> Self { + Self { + level: NoteLevel::Context, + message: message.into(), + } + } + + /// Picks the level from a condition, for the common case where the same + /// sentence is a finding or context depending on the numbers in it. + pub fn leveled(needs_attention: bool, message: impl Into) -> Self { + if needs_attention { + Self::attention(message) + } else { + Self::context(message) + } + } +} + /// Everything the driver reports for a scenario. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -519,8 +575,19 @@ pub struct ChaosSummary { /// it interpretable. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub ownership: Vec, - /// The read-back verdicts that need a human, hoisted for scanning. + /// The verdicts and findings that need a human, hoisted for scanning. + /// + /// CI raises an annotation when this is non-empty, so nothing belongs here + /// that is true of a healthy run. Context goes in [`Self::notes`]. pub attention: Vec, + /// Lines a human needs in order to read the run, which are not themselves + /// problems: how the routing table looked before measuring, how much of the + /// mechanism under test the fault actually landed in, and so on. + /// + /// Kept out of [`Self::attention`] so that list keeps meaning "look at + /// this". Absent from older results, hence `default`. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub notes: Vec, } impl ChaosSummary { @@ -631,6 +698,17 @@ impl ChaosSummary { schedule_fires: None, ownership: Vec::new(), attention, + notes: Vec::new(), + } + } + + /// Files a batch of scenario notes into the two lists by their level. + pub fn absorb(&mut self, notes: impl IntoIterator) { + for note in notes { + match note.level { + NoteLevel::Attention => self.attention.push(note.message), + NoteLevel::Context => self.notes.push(note.message), + } } } @@ -657,6 +735,7 @@ impl ChaosSummary { /// than only in the numbers underneath them. pub fn with_schedule_fires(mut self, report: ScheduleFireReport) -> Self { self.attention.extend(report.attention_lines()); + self.notes.extend(report.note_lines()); self.schedule_fires = Some(report); self } @@ -1088,4 +1167,64 @@ mod tests { assert_eq!(stats.count, 0); assert_eq!(stats.max_ms, 0); } + + fn empty_summary() -> ChaosSummary { + ChaosSummary::build(&[], Vec::new(), Vec::new(), None) + } + + #[test] + fn notes_are_filed_by_level_rather_than_all_into_attention() { + let mut summary = empty_summary(); + summary.absorb([ + Note::context("routing at start: 1024/1024 shards (settled before measuring)"), + Note::attention("WARNING: measured against an unsettled cluster"), + ]); + + assert_eq!(summary.attention.len(), 1); + assert!(summary.attention[0].contains("unsettled")); + assert_eq!(summary.notes.len(), 1); + assert!(summary.notes[0].contains("settled before measuring")); + } + + /// The reason the split exists. CI raises an annotation when `attention` is + /// non-empty, so a clean run has to leave it empty — otherwise the + /// annotation fires every time and stops meaning anything. + #[test] + fn a_run_with_only_context_raises_nothing_for_ci() { + let mut summary = empty_summary(); + summary.absorb([ + Note::context("routing at start: 1024/1024 shards (settled before measuring)"), + Note::context("S10 killed the executor with 353 actions pending"), + ]); + + assert!(summary.attention.is_empty()); + assert_eq!(summary.notes.len(), 2); + } + + #[test] + fn leveled_picks_the_list_from_the_condition() { + assert_eq!(Note::leveled(true, "x").level, NoteLevel::Attention); + assert_eq!(Note::leveled(false, "x").level, NoteLevel::Context); + } + + /// Older results have no `notes` key at all, and must still deserialise. + #[test] + fn a_result_written_before_notes_existed_still_reads() { + let mut summary = empty_summary(); + summary.absorb([Note::context("context")]); + let mut json: serde_json::Value = serde_json::to_value(&summary).unwrap(); + assert!(json.get("notes").is_some(), "notes are serialised when set"); + + json.as_object_mut().unwrap().remove("notes"); + let back: ChaosSummary = serde_json::from_value(json).unwrap(); + assert!(back.notes.is_empty()); + } + + /// An empty `notes` is omitted rather than written as `[]`, matching how + /// every other optional block in this result behaves. + #[test] + fn an_empty_notes_list_is_not_serialised() { + let json = serde_json::to_value(empty_summary()).unwrap(); + assert!(json.get("notes").is_none()); + } } From b26a898b4c5aded04362b0dec0c7f54ee363eb81 Mon Sep 17 00:00:00 2001 From: kmatasfp <33095685+kmatas@users.noreply.github.com> Date: Thu, 20 Aug 2026 15:59:01 -0700 Subject: [PATCH 06/40] Drop per-run findings from the chaos scenario docs --- integration-tests/src/chaos/fires.rs | 13 ++-- integration-tests/src/chaos/scenarios/s10.rs | 79 +++++--------------- 2 files changed, 26 insertions(+), 66 deletions(-) diff --git a/integration-tests/src/chaos/fires.rs b/integration-tests/src/chaos/fires.rs index 2911aa7114..fbd81078db 100644 --- a/integration-tests/src/chaos/fires.rs +++ b/integration-tests/src/chaos/fires.rs @@ -72,12 +72,13 @@ //! immediately and correctly, and the arithmetic still reports it as tens of //! seconds late. //! -//! This is not hypothetical. The first S10 run (2026-08-20) killed an executor -//! while 124 registrations were in flight to it; 26 of them stalled on the -//! client's 120s attempt timeout and then succeeded on retry in about 60ms. All -//! 26 fired correctly and instantly, and all 26 landed in the percentiles as -//! ~115s late — 13 in each group, which made the untouched control group look -//! exactly as damaged as the killed executor's targets. +//! This is not hypothetical: killing an executor stalls the registrations that +//! were in flight to it, and a stall longer than the lead puts every one of +//! those actions into the delay cells as tens of seconds late even though each +//! fired the instant it was accepted. Worse, it lands in *both* groups, because +//! emitters are spread across executors independently of their targets — so an +//! untouched control group can be made to look exactly as damaged as the killed +//! executor's. //! //! So they are separated. A fire whose registration completed after its due time //! is counted as `overdueOnArrival` and kept out of the delay cells, because the diff --git a/integration-tests/src/chaos/scenarios/s10.rs b/integration-tests/src/chaos/scenarios/s10.rs index 2852f79ff5..5352ec0bd1 100644 --- a/integration-tests/src/chaos/scenarios/s10.rs +++ b/integration-tests/src/chaos/scenarios/s10.rs @@ -82,70 +82,28 @@ //! asserted: how much a lease recovery may cost is a judgement, and the number //! that matters is in the result either way. //! -//! ## What the first run found (2026-08-20) +//! ## Reading a fire delay: registration first, scheduler second //! -//! All 30,626 accepted registrations fired exactly once. Nothing was lost, -//! nothing ran twice, no target failed to testify. Reassignment cost the -//! scheduler almost nothing: the p99 fire delay on the killed executor's targets -//! during the fault was 2,158ms against a baseline of 2,134ms, and both sit just -//! under the scheduler's own 2s refresh interval, which is what a due action -//! waits for when nothing is wrong. +//! A large fire delay does not necessarily mean the scheduler was slow. The +//! registering invocation mints its action's due time *before* the call goes +//! out, so a registration that stalls in the client describes an action that was +//! already overdue when the platform first heard about it. Those fire instantly +//! and correctly, and would still arrive in the percentiles as minutes late. //! -//! The delay percentiles needed a correction to say that, and the correction is -//! now part of the account — see [`crate::chaos::fires`]. The kill stalled 26 of -//! the 124 registrations that were in flight to that executor on the client's -//! 120s attempt timeout; each retried and succeeded in about 60ms. Those -//! actions fired immediately and correctly, and arrived in the percentiles as -//! ~115s late, evenly split between the two groups because emitters are spread -//! across both executors. +//! [`crate::chaos::fires`] holds them out of the scheduler-delay cells and +//! reports them as `overdue_on_arrival` instead. Two things follow for anyone +//! reading a result: //! -//! The stall is not a new finding, and it is not an unfixed one either. golem-dev -//! ran `v1.5.10-dev.1`, which is a tag on the branch of golemcloud/golem#3748 -//! rather than on `1.5.x`, so the cluster already carried the first half of that -//! work. It was cut on 2026-08-19, and ten further commits to the gRPC client -//! landed on that branch the next day. One of them, `edd668f4c`, is this exact -//! case: a pod killed with requests in flight reports most of them `Cancelled`, -//! the connection was never retired, and every later request queued onto a -//! channel that could never work again. +//! * Check the registration latencies before concluding anything about the +//! scheduler. A client-side stall and a lease recovery look identical in a +//! delay percentile and have nothing to do with each other. +//! * Read the `overdue_on_arrival` count together with its worst case, never +//! alone. The classification catches any registration slower than the +//! configured lead, so a stall shrinking from minutes to seconds moves entries +//! *into* the bucket rather than out of it: the count can rise while the +//! platform gets strictly better. //! -//! ## What the second run showed (2026-08-20, `v1.5.10-dev.2`) -//! -//! The scenario was re-run unchanged against a cluster rebuilt from the tip of -//! that branch, which is the ten client commits the first run's images predated. -//! Same driver, same 100 targets, same 2s interval and 10s lead; the only thing -//! that moved was the deployed image. It is the cleanest reading this harness -//! has produced, because the fault landed comparably — 353 actions pending at -//! the kill against the first run's 413 — and the undisturbed baseline windows -//! did not move at all (p99 2,132ms and 2,101ms, against 2,134ms and 2,142ms). -//! -//! Exactly-once held, as it had before: 30,730 of 30,730. The stall did not. -//! -//! | | dev.1 | dev.2 | -//! | -- | -- | -- | -//! | registrations over 20s | 32 | 0 | -//! | slowest registration | 125,188ms | 12,891ms | -//! | slow registrations *submitted after* the kill | 79 | 0 | -//! | fire delay past the 60s budget | 26 | 0 | -//! | worst fire delay, killed executor's targets | 117,135ms | 2,177ms | -//! | worst fire delay, control group | 117,095ms | 3,205ms | -//! -//! The last of those rows is the one that identifies the defect rather than -//! merely measuring it. Under dev.1 the damage was not confined to calls that -//! were riding the dying connection: 79 registrations submitted *after* the pod -//! was already gone also stalled, because the dead channel stayed in the cache -//! and kept accepting work. Under dev.2 that number is zero. Only the calls -//! actually in flight at the moment of the kill paid anything, which is the -//! shape a correctly retired connection produces. -//! -//! What remains is not a hang. The 62 registrations in flight at the kill took -//! 10.0-12.9s each, on a single attempt, and that is inside the band the rest of -//! the cluster pays for losing an executor. It does exceed the 10s lead, so those -//! actions were due before their registration landed and are reported as -//! `overdue_on_arrival` rather than as scheduler delay. Note that the count went -//! *up* against dev.1's 26 while the magnitude fell by an order of magnitude: -//! the classification counts anything slower than the lead, so shrinking a 125s -//! stall to 13s moves entries into it rather than out. Read the two numbers -//! together, never the count alone. +//! Run-by-run findings live in the S10 runbook in golem-cloud, not here. use crate::chaos::fires::{FaultWindow, ScheduleFireReport}; use crate::chaos::history::{ @@ -264,6 +222,7 @@ pub async fn run( termination_reason: $reason, pinned_selection: None, scheduled_selection: selection.clone(), + promise_selection: None, }, ); write_outputs(&result, &history, outputs)?; From 3865ff98a7cc4092aaf9f781e0fca67a6ade3e7b Mon Sep 17 00:00:00 2001 From: kmatasfp <33095685+kmatas@users.noreply.github.com> Date: Thu, 20 Aug 2026 15:59:12 -0700 Subject: [PATCH 07/40] Add S11 chaos scenario driver and promise-wakeup account --- golem-test-framework/src/benchmark/config.rs | 2 + .../chaos_suites/cloud-chaos.yaml | 70 ++ integration-tests/src/benchmarks/all.rs | 4 + integration-tests/src/chaos/fires.rs | 54 +- integration-tests/src/chaos/history.rs | 87 +- integration-tests/src/chaos/mod.rs | 70 +- integration-tests/src/chaos/pinned.rs | 26 +- integration-tests/src/chaos/result.rs | 108 +- integration-tests/src/chaos/scenarios/mod.rs | 6 + integration-tests/src/chaos/scenarios/s1.rs | 1 + integration-tests/src/chaos/scenarios/s11.rs | 602 +++++++++++ integration-tests/src/chaos/scenarios/s12.rs | 1 + integration-tests/src/chaos/scenarios/s13.rs | 1 + integration-tests/src/chaos/scenarios/s5.rs | 1 + integration-tests/src/chaos/scenarios/s8.rs | 1 + integration-tests/src/chaos/scheduled.rs | 163 +-- integration-tests/src/chaos/split.rs | 366 +++++++ integration-tests/src/chaos/summary.rs | 37 +- integration-tests/src/chaos/waiters.rs | 549 ++++++++++ integration-tests/src/chaos/wakeups.rs | 963 ++++++++++++++++++ integration-tests/src/chaos/workload.rs | 12 + .../golem-it-promise-agent-rust/src/lib.rs | 111 ++ 22 files changed, 3038 insertions(+), 197 deletions(-) create mode 100644 integration-tests/src/chaos/scenarios/s11.rs create mode 100644 integration-tests/src/chaos/split.rs create mode 100644 integration-tests/src/chaos/waiters.rs create mode 100644 integration-tests/src/chaos/wakeups.rs diff --git a/golem-test-framework/src/benchmark/config.rs b/golem-test-framework/src/benchmark/config.rs index 653d281a5f..5706f48f26 100644 --- a/golem-test-framework/src/benchmark/config.rs +++ b/golem-test-framework/src/benchmark/config.rs @@ -244,6 +244,8 @@ pub enum ChaosScenarioArg { S13, /// Executor pod kill while scheduled actions are between claim and fire. S10, + /// Executor pod kill while agents are suspended on promises being completed. + S11, } /// Density subcommand action. diff --git a/integration-tests/chaos_suites/cloud-chaos.yaml b/integration-tests/chaos_suites/cloud-chaos.yaml index a3883ff25a..6dcdf4ca1a 100644 --- a/integration-tests/chaos_suites/cloud-chaos.yaml +++ b/integration-tests/chaos_suites/cloud-chaos.yaml @@ -393,6 +393,76 @@ scenarios: signalTimeoutSecs: 1800 + # S11 — executor pod kill while agents are suspended on promises (GOL-377). + # + # The only scenario whose agents are asleep when the fault lands. Each waiter + # creates a promise, parks inside an invocation awaiting it, and is resolved + # from outside a few seconds later. Killing the executor that owns them asks + # whether a completion the platform accepted still reaches the agent it was + # meant to wake, exactly once, after the shards move. + # + # Like S10 the driver names the pod and keeps driving the waiters on the other + # executor as a control group. Unlike every other scenario, its agents live in + # the promise component rather than the counters one. + - code: S11 + name: executor-crash-during-promise-completion + enabled: true + + fault: + kind: pod-kill + target: worker-executor + # The workflow narrows the selector to the pod the driver named; `one` + # stays as the belt-and-braces bound, exactly as in S8 and S10. + mode: one + durationSecs: 60 + + phases: + # Long enough for cold starts and route warm-up to settle, so the kill + # lands on a population that has been cycling steadily rather than one + # still arriving. + baselineSecs: 180 + # Covers the kill, the reschedule, and the shard reassignment that has to + # happen before anything can resume a waiter the dead executor owned. + faultSecs: 120 + # Rounds continue throughout. Long enough that waiters disturbed by the + # kill get several whole rounds afterwards, so a waiter that recovered + # late is distinguishable from one that never did. + recoverySecs: 300 + + promise: + # 200 waiters, each parked on at most one promise, so 200 agents are + # suspended at any instant. Also the resolution of the report: a lost + # wakeup localises to one waiter out of two hundred. + waiters: 200 + # 5s parked before the completion goes out, so ~40 completions a second + # across the pool. It has to comfortably exceed the workflow's + # inject-and-verify path — signal poll (5s) plus `kubectl apply` plus + # waiting for `AllInjected` — or every promise armed before the kill would + # already have been completed by the time the pod died. + dwellMillis: 5000 + # What resuming a suspended waiter may cost, and the number the wakeup + # delay percentiles are reported against. Derived rather than picked: a + # shard reassignment has to complete before anything owns the waiter, and + # then the worker has to be recovered and its oplog replayed to the point + # it was parked at. 60s covers both without being so generous that a + # regression would sit inside it. + # + # Recorded, not asserted. A p99 past it is an attention line and a number + # in the result, because how much a reassignment may cost is a judgement. + wakeupBudgetSecs: 60 + + retryPolicy: + # Identical to the others. One extra consequence here, spelled out in + # integration-tests/src/chaos/waiters.rs: a completion retry can repair a + # wakeup that was lost, but only for a completion that had *not* already + # been accepted — so the question this scenario asks, about completions the + # platform confirmed, is untouched by it. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + # S13 — rolling executor restarts under load (GOL-367). # # One executor killed every 60 seconds for five minutes while the mixed diff --git a/integration-tests/src/benchmarks/all.rs b/integration-tests/src/benchmarks/all.rs index ae6711b38c..0f61fc1bed 100644 --- a/integration-tests/src/benchmarks/all.rs +++ b/integration-tests/src/benchmarks/all.rs @@ -593,6 +593,7 @@ async fn run_chaos( ChaosScenarioArg::S12 => chaos::ScenarioCode::S12, ChaosScenarioArg::S13 => chaos::ScenarioCode::S13, ChaosScenarioArg::S10 => chaos::ScenarioCode::S10, + ChaosScenarioArg::S11 => chaos::ScenarioCode::S11, }; let config = suite .scenario(code, allow_disabled) @@ -627,6 +628,9 @@ async fn run_chaos( chaos::ScenarioCode::S10 => { chaos::scenarios::s10::run(&config, &manifest, &deps, &signals, &outputs).await } + chaos::ScenarioCode::S11 => { + chaos::scenarios::s11::run(&config, &manifest, &deps, &signals, &outputs).await + } }; deps.kill_all().await; diff --git a/integration-tests/src/chaos/fires.rs b/integration-tests/src/chaos/fires.rs index fbd81078db..4d2ee1b13b 100644 --- a/integration-tests/src/chaos/fires.rs +++ b/integration-tests/src/chaos/fires.rs @@ -101,54 +101,16 @@ use std::time::Duration; /// number of findings. const MAX_FINDINGS: usize = 200; -/// The fault window, as the workflow reported it. -#[derive(Debug, Clone, Copy)] -pub struct FaultWindow { - pub injected_at: DateTime, - /// Absent for a run that never saw the fault clear. - pub recovered_at: Option>, -} +/// The fault window, as the workflow reported it. See +/// [`crate::chaos::split::FaultWindow`]. +pub type FaultWindow = crate::chaos::split::FaultWindow; /// Which side of the fault an action was due on. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub enum FireWindow { - BeforeFault, - DuringFault, - AfterFault, - /// The run never learned when the fault was injected, so no action can be - /// placed relative to it. - Unknown, -} - -impl FireWindow { - pub fn as_str(self) -> &'static str { - match self { - FireWindow::BeforeFault => "before-fault", - FireWindow::DuringFault => "during-fault", - FireWindow::AfterFault => "after-fault", - FireWindow::Unknown => "unknown", - } - } - - fn of(due: DateTime, fault: Option) -> Self { - match fault { - None => FireWindow::Unknown, - Some(window) if due < window.injected_at => FireWindow::BeforeFault, - Some(FaultWindow { - recovered_at: Some(recovered), - .. - }) if due >= recovered => FireWindow::AfterFault, - Some(_) => FireWindow::DuringFault, - } - } -} - -impl std::fmt::Display for FireWindow { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(self.as_str()) - } -} +/// +/// An alias rather than a type of its own: every scenario that reports against a +/// fault window asks the same question, and `fireWindow` is the key an archived +/// S10 result already carries. +pub type FireWindow = crate::chaos::split::Window; /// Whether an action's target was on the executor the fault killed. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] diff --git a/integration-tests/src/chaos/history.rs b/integration-tests/src/chaos/history.rs index 2abaaf3bce..8b33f3fa0b 100644 --- a/integration-tests/src/chaos/history.rs +++ b/integration-tests/src/chaos/history.rs @@ -67,6 +67,12 @@ pub enum Stream { /// one known executor, so mixing them into the durable population would /// blur two different experiments. PinnedHttp, + /// `PromiseWaiter.arm` / `wait` / the external completion that resolves it + /// (GOL-377). Distinct from `Promise` even though both land on the promise + /// component: that stream creates and resolves a promise in one breath with + /// nobody suspended on it, and this one exists precisely to leave an agent + /// parked across a fault. + PromiseWait, } impl Stream { @@ -78,6 +84,7 @@ impl Stream { Stream::Promise => "promise", Stream::Quota => "quota", Stream::PinnedHttp => "pinned-http", + Stream::PromiseWait => "promise-wait", } } @@ -91,6 +98,11 @@ impl Stream { /// - `Promise` operations resolve a one-shot promise rather than advancing a /// counter, so there is no accumulated number to read. They are reported /// on created/completed counts and latency. + /// - `PromiseWait` agents *do* keep a durable count, but comparing totals + /// would be strictly weaker than what S11 already does with them: every + /// completion carries a token into the waiter's wakeup log, so the report + /// pairs individual completions against individual wakeups instead of + /// arguing about sums. See [`crate::chaos::wakeups`]. pub fn has_readback(self) -> bool { matches!( self, @@ -98,13 +110,14 @@ impl Stream { ) } - pub const ALL: [Stream; 6] = [ + pub const ALL: [Stream; 7] = [ Stream::Durable, Stream::Ephemeral, Stream::Scheduled, Stream::Promise, Stream::Quota, Stream::PinnedHttp, + Stream::PromiseWait, ]; } @@ -388,6 +401,71 @@ pub struct HistoryDocument { /// wonders whether the section was dropped. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub scheduled_fires: Vec, + /// Per-waiter wakeup logs, for the scenarios that park agents on promises. + /// Empty for every other scenario rather than absent, for the same reason as + /// `scheduled_fires`. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub promise_wakeups: Vec, +} + +/// One wakeup, as the waiter agent recorded it (GOL-377). +/// +/// The times are the *cluster's*, stamped inside the agent, which is what makes +/// this log the authority on whether a completion landed. The driver's own view +/// is in the operation record for the `wait` invocation, and during the fault +/// that view is frequently just a broken connection. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct WakeupRecord { + /// The round's idempotency key, carried in by `arm` and back out by the + /// wakeup. This is what pairs a completion to the wakeup it caused. + pub token: String, + /// When the waiter armed the promise. + pub armed_at: DateTime, + /// When the platform resumed the waiter. + pub woken_at: DateTime, +} + +impl WakeupRecord { + /// How long the waiter was parked, on one clock. + /// + /// Both ends are stamped by the executor, so this is free of the driver ↔ + /// cluster skew that the completion-to-wakeup delay carries. It is not the + /// delay itself — it also contains the round's deliberate dwell — but it is + /// what lets a reader tell a slow wakeup from a skewed clock. + pub fn parked_ms(&self) -> i64 { + (self.woken_at - self.armed_at).num_milliseconds() + } +} + +/// Everything read back from one waiter agent. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct WaiterWakeupLog { + pub agent: String, + /// `PromiseWaiter.wakes`, which keeps counting past the log's cap and is + /// therefore what says whether the log below is complete. + pub wakes: Option, + pub wakeups: Vec, + /// Why the agent could not be read, when it could not be. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +impl WaiterWakeupLog { + /// Whether this log can testify about its own completions. + /// + /// Same two ways it cannot as [`TargetFireLog::is_complete`], and the same + /// consequence: without a whole log an absent wakeup is ambiguous between a + /// lost completion and a dropped log entry, and S11 must not call the second + /// one a finding. + pub fn is_complete(&self) -> bool { + match (self.error.is_some(), self.wakes) { + (true, _) => false, + (false, Some(wakes)) => self.wakeups.len() as u64 >= wakes, + (false, None) => false, + } + } } /// Append-only operation log, shared across the concurrent workload streams. @@ -400,6 +478,7 @@ pub struct OperationHistory { inner: Arc>>, next_id: Arc, fire_logs: Arc>>, + wakeup_logs: Arc>>, } impl OperationHistory { @@ -409,6 +488,7 @@ impl OperationHistory { inner: Arc::new(Mutex::new(Vec::new())), next_id: Arc::new(std::sync::atomic::AtomicU64::new(0)), fire_logs: Arc::new(Mutex::new(Vec::new())), + wakeup_logs: Arc::new(Mutex::new(Vec::new())), } } @@ -458,6 +538,10 @@ impl OperationHistory { *self.fire_logs.lock().unwrap() = logs; } + pub fn record_wakeup_logs(&self, logs: Vec) { + *self.wakeup_logs.lock().unwrap() = logs; + } + pub fn document(&self, partial: bool) -> HistoryDocument { HistoryDocument { schema_version: HISTORY_SCHEMA_VERSION, @@ -465,6 +549,7 @@ impl OperationHistory { partial, operations: self.snapshot(), scheduled_fires: self.fire_logs.lock().unwrap().clone(), + promise_wakeups: self.wakeup_logs.lock().unwrap().clone(), } } diff --git a/integration-tests/src/chaos/mod.rs b/integration-tests/src/chaos/mod.rs index 6bccb8f6f7..9d4f9fcf77 100644 --- a/integration-tests/src/chaos/mod.rs +++ b/integration-tests/src/chaos/mod.rs @@ -43,7 +43,10 @@ pub mod result; pub mod scenarios; pub mod scheduled; pub mod signal; +pub mod split; pub mod summary; +pub mod waiters; +pub mod wakeups; pub mod workload; use anyhow::Context; @@ -67,6 +70,8 @@ pub enum ScenarioCode { S13, /// Executor pod kill while scheduled actions are between claim and fire. S10, + /// Executor pod kill while agents are suspended on promises being completed. + S11, } impl ScenarioCode { @@ -78,17 +83,19 @@ impl ScenarioCode { ScenarioCode::S12 => "S12", ScenarioCode::S13 => "S13", ScenarioCode::S10 => "S10", + ScenarioCode::S11 => "S11", } } /// Every scenario this driver implements. The suite YAML is checked against /// this list, so a scenario cannot be enabled in YAML without code behind /// it, nor implemented without an operational switch in front of it. - pub const ALL: [ScenarioCode; 6] = [ + pub const ALL: [ScenarioCode; 7] = [ ScenarioCode::S1, ScenarioCode::S5, ScenarioCode::S8, ScenarioCode::S10, + ScenarioCode::S11, ScenarioCode::S12, ScenarioCode::S13, ]; @@ -352,6 +359,50 @@ impl ScheduledConfig { } } +/// Shape of the suspended-waiter workload (GOL-377). +/// +/// The fourth experiment shape, and the only one whose agents are *asleep* when +/// the fault lands. [`ScheduledConfig`] leaves work with the platform and walks +/// away; this one leaves an agent parked mid-invocation on a promise, so the +/// thing that has to survive the kill is not a queued action but a suspended +/// worker and the completion on its way to it. +/// +/// The number that decides whether the run measures anything is `waiters`: each +/// one holds exactly one promise at a time, so the pool size *is* the population +/// standing parked at the instant the pod dies. `dwellMillis` decides how much +/// of that population is also mid-completion — see [`crate::chaos::waiters`]. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PromiseConfig { + /// Waiter agents, each parked on at most one promise at a time. Also the + /// resolution of the report: a finding localises to one waiter out of this + /// many. + pub waiters: u32, + /// How long a waiter stays parked before its promise is completed. + /// + /// Sets the completion rate — `waiters / dwell` — and, with it, how many + /// completions are genuinely in flight when the pod dies. It has to + /// comfortably exceed the workflow's inject-and-verify path (signal poll, + /// `kubectl apply`, waiting for `AllInjected`) or every promise armed before + /// the kill would already have been completed by the time it landed. + pub dwell_millis: u64, + /// What resuming a suspended waiter is allowed to cost, which the wakeup + /// delay percentiles are reported against. An SLO the run records rather + /// than a threshold it fails on: the floor is a shard reassignment plus the + /// worker recovery that replays the waiter's oplog, and how much more than + /// that is acceptable is a judgement. + pub wakeup_budget_secs: u64, +} + +impl PromiseConfig { + pub fn dwell(&self) -> Duration { + Duration::from_millis(self.dwell_millis) + } + pub fn wakeup_budget(&self) -> Duration { + Duration::from_secs(self.wakeup_budget_secs) + } +} + /// One step of the executor scale schedule the workflow runs during the fault. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -427,6 +478,9 @@ pub struct ScenarioConfig { /// run one. #[serde(default)] pub scheduled: Option, + /// The suspended-waiter workload. Absent for scenarios that do not run one. + #[serde(default)] + pub promise: Option, /// Shard-ownership oracle settings. Absent for scenarios that do not sample /// executor assignments. #[serde(default)] @@ -490,6 +544,16 @@ impl ScenarioConfig { }) } + /// The suspended-waiter block. See [`Self::require_workload`]. + pub fn require_promise(&self) -> anyhow::Result<&PromiseConfig> { + self.promise.as_ref().ok_or_else(|| { + anyhow::anyhow!( + "chaos scenario {} needs a `promise` block in the suite YAML", + self.code + ) + }) + } + /// The pinned workload block. See [`Self::require_workload`]. pub fn require_pinned(&self) -> anyhow::Result<&PinnedConfig> { self.pinned.as_ref().ok_or_else(|| { @@ -624,6 +688,7 @@ mod tests { }), pinned: None, scheduled: None, + promise: None, ownership: None, scale_during_fault: None, retry_policy: RetryPolicy::default(), @@ -725,6 +790,9 @@ mod tests { ScenarioCode::S10 => { entry.require_scheduled().unwrap(); } + ScenarioCode::S11 => { + entry.require_promise().unwrap(); + } } } } diff --git a/integration-tests/src/chaos/pinned.rs b/integration-tests/src/chaos/pinned.rs index 4ae068c5ee..010aa80a17 100644 --- a/integration-tests/src/chaos/pinned.rs +++ b/integration-tests/src/chaos/pinned.rs @@ -63,6 +63,7 @@ use crate::chaos::history::Stream; use crate::chaos::workload::{self, WorkloadContext}; use anyhow::Context; use golem_common::base_model::agent::ParsedAgentId; +use golem_common::model::component::ComponentDto; use golem_common::model::{AgentId, RoutingTable}; use golem_common::{agent_id, data_value}; use golem_test_framework::config::{BenchmarkTestDependencies, TestDependencies}; @@ -123,9 +124,20 @@ pub fn candidate_agent_name(key_prefix: &str, index: u32) -> String { /// ownership calculation would be answering a different question from the one /// the platform answers. pub fn routing_agent_id(ctx: &WorkloadContext, agent_type: &str, agent: &str) -> AgentId { + routing_agent_id_in(&ctx.counters, agent_type, agent) +} + +/// [`routing_agent_id`] against a named component rather than the counters one. +/// +/// Ownership is per agent id and an agent id contains its component, so an agent +/// type that lives in a different component — S11's waiters are in the promise +/// component — hashes to a different shard than the same name would under +/// `counters`. Looking one up against the wrong component would silently aim a +/// kill at whichever executor happened to own a name nothing uses. +pub fn routing_agent_id_in(component: &ComponentDto, agent_type: &str, agent: &str) -> AgentId { let parsed: ParsedAgentId = agent_id!(agent_type, agent.to_string()); AgentId { - component_id: ctx.counters.id, + component_id: component.id, agent_id: parsed.to_string(), } } @@ -141,10 +153,20 @@ pub fn owners_by_pod( table: &RoutingTable, agent_type: &str, agents: &[String], +) -> BTreeMap> { + owners_by_pod_in(&ctx.counters, table, agent_type, agents) +} + +/// [`owners_by_pod`] against a named component. See [`routing_agent_id_in`]. +pub fn owners_by_pod_in( + component: &ComponentDto, + table: &RoutingTable, + agent_type: &str, + agents: &[String], ) -> BTreeMap> { let mut by_pod: BTreeMap> = BTreeMap::new(); for agent in agents { - if let Some(pod) = table.lookup(&routing_agent_id(ctx, agent_type, agent)) { + if let Some(pod) = table.lookup(&routing_agent_id_in(component, agent_type, agent)) { by_pod .entry(pod.to_string()) .or_default() diff --git a/integration-tests/src/chaos/result.rs b/integration-tests/src/chaos/result.rs index 2c58b1f544..cc61bfc9fa 100644 --- a/integration-tests/src/chaos/result.rs +++ b/integration-tests/src/chaos/result.rs @@ -26,8 +26,11 @@ use crate::chaos::pinned::PinnedSelection; use crate::chaos::scheduled::ScheduledSelection; +use crate::chaos::split::PodSplit; use crate::chaos::summary::{ChaosSummary, TerminationReason}; -use crate::chaos::{FaultConfig, PinnedConfig, RetryPolicy, ScheduledConfig, WorkloadConfig}; +use crate::chaos::{ + FaultConfig, PinnedConfig, PromiseConfig, RetryPolicy, ScheduledConfig, WorkloadConfig, +}; use chrono::{DateTime, Utc}; use golem_test_framework::benchmark::RunMetadata; use serde::{Deserialize, Serialize}; @@ -140,6 +143,15 @@ pub struct ChaosResult { /// control group. #[serde(default, skip_serializing_if = "Option::is_none")] pub scheduled_selection: Option, + /// The suspended-waiter workload the run was configured with, if any. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub promise: Option, + /// How the promise waiters divided around the executor the fault was aimed + /// at. Present only for S11, and load-bearing for the same reason as + /// `scheduledSelection`: without it there is no way to tell the affected + /// population from the control group. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub promise_selection: Option, pub retry_policy: RetryPolicy, pub scope: RunScope, pub summary: ChaosSummary, @@ -209,6 +221,8 @@ mod tests { pinned_selection: None, scheduled: None, scheduled_selection: None, + promise: None, + promise_selection: None, retry_policy: RetryPolicy::default(), scope: RunScope { environment_id: "env-1".to_string(), @@ -221,6 +235,94 @@ mod tests { } } + /// The S11 shape. Same contract as the S10 test below, for the same reason: + /// `ci-scripts/chaos-investigation-report.py` in golem-cloud reads these by + /// name, and the two repositories cannot be changed atomically. + #[test] + fn an_s11_result_carries_the_promise_wakeup_fields_the_investigation_report_reads() { + use crate::chaos::history::{WaiterWakeupLog, WakeupRecord}; + use crate::chaos::split::{FaultWindow, PodSplit}; + use crate::chaos::wakeups::WakeupReport; + + let now = Utc::now(); + let waiter = "chaos-s11-promise-waiter-0000".to_string(); + let split = PodSplit { + pod_address: "10.0.1.1:9000".to_string(), + pod_ip: "10.0.1.1".to_string(), + on_pod: vec![waiter.clone()], + elsewhere: Vec::new(), + targets_per_pod: std::collections::BTreeMap::new(), + number_of_shards: 1024, + }; + + let mut result = sample_result(TerminationReason::Completed); + result.scenario_code = "S11".to_string(); + result.promise = Some(crate::chaos::PromiseConfig { + waiters: 200, + dwell_millis: 5000, + wakeup_budget_secs: 60, + }); + result.promise_selection = Some(split.clone()); + result.summary = ChaosSummary::build(&[], Vec::new(), Vec::new(), Some(now)) + .with_promise_wakeups(WakeupReport::build( + &[], + &[WaiterWakeupLog { + agent: waiter.clone(), + wakes: Some(1), + wakeups: vec![WakeupRecord { + token: format!("{waiter}-00000001"), + armed_at: now, + woken_at: now + chrono::Duration::seconds(6), + }], + error: None, + }], + &split, + Some(FaultWindow { + injected_at: now, + recovered_at: None, + }), + std::time::Duration::from_secs(5), + std::time::Duration::from_secs(60), + 0, + )); + + let json = serde_json::to_value(&result).unwrap(); + let wakeups = &json["summary"]["promiseWakeups"]; + for key in [ + "wakeupBudgetMs", + "dwellMs", + "completionsConfirmed", + "completionsIndeterminate", + "completionsRejected", + "wakeupsRecorded", + "wokeOnce", + "indeterminateThatWoke", + "inconclusive", + "unverifiable", + "unknownTokens", + "waitersUnreadable", + "waitersTruncated", + "waitersStoodDown", + "waitersWedged", + "delay", + "findings", + "findingsOmitted", + ] { + assert!( + !wakeups[key].is_null(), + "summary.promiseWakeups.{key} is what the investigation report reads" + ); + } + assert_eq!(json["promise"]["wakeupBudgetSecs"], 60); + assert_eq!(json["promiseSelection"]["podIp"], "10.0.1.1"); + + // And it still round-trips, so an archived S11 result stays readable. + let parsed: ChaosResult = serde_json::from_str(&json.to_string()).unwrap(); + assert_eq!(parsed.scenario_code, "S11"); + assert!(parsed.summary.promise_wakeups.is_some()); + assert!(parsed.promise_selection.is_some()); + } + /// The S10 shape, whose report is read by a script in another repository. /// /// `ci-scripts/chaos-investigation-report.py` in golem-cloud renders these @@ -554,6 +656,8 @@ mod sample_artifact { pinned_selection: None, scheduled: None, scheduled_selection: None, + promise: None, + promise_selection: None, retry_policy: RetryPolicy::default(), scope: RunScope { environment_id: "0192f000-0000-7000-8000-000000000001".to_string(), @@ -766,6 +870,8 @@ mod sample_artifact { .collect(), number_of_shards: 1024, }), + promise: None, + promise_selection: None, retry_policy: RetryPolicy::default(), scope: RunScope { environment_id: "0192f000-0000-7000-8000-000000000001".to_string(), diff --git a/integration-tests/src/chaos/scenarios/mod.rs b/integration-tests/src/chaos/scenarios/mod.rs index 43f71084d6..4bdad51533 100644 --- a/integration-tests/src/chaos/scenarios/mod.rs +++ b/integration-tests/src/chaos/scenarios/mod.rs @@ -27,6 +27,7 @@ pub mod s1; pub mod s10; +pub mod s11; pub mod s12; pub mod s13; pub mod s5; @@ -81,6 +82,9 @@ pub struct ScenarioOutcome { /// Present only for S10, which divides its targets around the executor the /// fault was aimed at rather than driving only the ones it owns. pub scheduled_selection: Option, + /// Present only for S11, which divides its waiters around the executor the + /// fault was aimed at the same way S10 divides its targets. + pub promise_selection: Option, } /// Assembles the archived result. @@ -106,6 +110,8 @@ pub fn build_result(config: &ScenarioConfig, outcome: ScenarioOutcome) -> ChaosR pinned_selection: outcome.pinned_selection, scheduled: config.scheduled.clone(), scheduled_selection: outcome.scheduled_selection, + promise: config.promise.clone(), + promise_selection: outcome.promise_selection, retry_policy: config.retry_policy.clone(), scope: outcome.scope, summary: outcome.summary, diff --git a/integration-tests/src/chaos/scenarios/s1.rs b/integration-tests/src/chaos/scenarios/s1.rs index 0d858db27c..798f52826b 100644 --- a/integration-tests/src/chaos/scenarios/s1.rs +++ b/integration-tests/src/chaos/scenarios/s1.rs @@ -236,6 +236,7 @@ pub async fn run( termination_reason: $reason, pinned_selection: None, scheduled_selection: None, + promise_selection: None, }, ); write_outputs(&result, &history, outputs)?; diff --git a/integration-tests/src/chaos/scenarios/s11.rs b/integration-tests/src/chaos/scenarios/s11.rs new file mode 100644 index 0000000000..f5fd04f99f --- /dev/null +++ b/integration-tests/src/chaos/scenarios/s11.rs @@ -0,0 +1,602 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! S11 — executor crash during promise completion (GOL-377). +//! +//! S8 kills an executor with invocations running on it. S10 kills one holding +//! work it promised to do later. S11 kills one holding agents that are *asleep*: +//! each waiter is suspended inside an invocation, parked on a promise, occupying +//! no thread and running no code, and something outside is about to resolve that +//! promise and expect the agent to carry on. +//! +//! ## Why this is its own scenario +//! +//! A suspended waiter is the one durable-execution state with no in-memory +//! representation to lose and no queue entry to drain. Nothing about it looks +//! like work in flight. Resuming it depends on a chain that a pod kill can break +//! in the middle: the completion is written to storage under the promise's key, +//! then the worker that owns the promise is activated so it notices. Those are +//! two steps, and only the first of them is durable. +//! +//! So the failure this scenario exists to catch is specific and quiet. A +//! completion is accepted — the caller gets a success — the write lands, and the +//! activation goes to an executor that is already dying. The promise is resolved +//! forever after, and the agent waiting on it is never told. Nothing errors, +//! nothing retries, and no count anywhere goes down. +//! +//! ## What the run measures the kill against +//! +//! Each of `waiters` agents holds exactly one promise at a time, so the number of +//! agents standing suspended when the pod dies is a known constant rather than a +//! sample. The driver records how many of them were actually parked across the +//! injection, and on which executor, measured from the history rather than +//! assumed from the cadence — a platform that had slowed down would have armed +//! fewer than the arithmetic says. A kill that caught nothing parked is reported +//! as a warning: a clean account of a mechanism that was never disturbed is not +//! evidence. +//! +//! Like S8 and S10 the kill is aimed, and like S10 every waiter keeps running. +//! The ones on other executors are the control group, which is what stops a +//! recovery that took its whole budget from hiding behind the half of the +//! population that was never touched. +//! +//! ## Two independent answers, and why both are kept +//! +//! Every round is observed twice. The driver holds the `wait` invocation open, so +//! it sees the wakeup as a returning call; the waiter writes the wakeup into its +//! own durable log, which the run reads afterwards. +//! +//! The two disagree exactly when it matters. Killing the executor takes the +//! driver's connection with it, so for the rounds this scenario is about, the +//! client's view is a broken pipe and nothing more. The agent's log is what +//! answers, and [`crate::chaos::wakeups`] is built around it. +//! +//! ## What fails the run +//! +//! Only the three token-level violations in [`WakeupReport`]. Wakeup delay is +//! reported against the configured budget as SLO evidence, not asserted: how much +//! a shard reassignment plus a worker recovery may cost is a judgement, and the +//! number is in the result either way. +//! +//! ## The waiter that answers nothing +//! +//! An unreadable agent normally means the run cannot say. For a suspended waiter +//! it can mean the opposite, because the read queues behind the invocation the +//! waiter is parked in. A waiter that stopped producing rounds during the run and +//! then answered no read is a worker still parked on a promise resolved minutes +//! ago — the defect itself, observed from two directions. The report separates +//! that from an agent that merely timed out. See [`crate::chaos::wakeups`]. + +use crate::chaos::history::{OperationHistory, OperationRecord, Outcome, Phase, Stream}; +use crate::chaos::prep::ChaosPrepManifest; +use crate::chaos::result::{ChaosResult, PhaseWindow, Phases, RunScope}; +use crate::chaos::scenarios::{ + OutputPaths, ScenarioOutcome, WARMUP_SETTLE, build_result, signal_termination, + snapshot_routing, wait_for_settled_routing, write_outputs, +}; +use crate::chaos::signal::{BaselineReady, FaultSignals, FaultTarget}; +use crate::chaos::split::{self, FaultWindow, PodSplit}; +use crate::chaos::summary::{ChaosSummary, Note, TerminationReason}; +use crate::chaos::waiters; +use crate::chaos::wakeups::WakeupReport; +use crate::chaos::workload::{PhaseMarker, WorkloadContext}; +use crate::chaos::{PromiseConfig, ScenarioCode, ScenarioConfig}; +use chrono::{DateTime, Utc}; +use golem_test_framework::config::BenchmarkTestDependencies; +use golem_test_framework::dsl::TestDsl; +use std::collections::BTreeSet; +use std::time::Duration; +use tracing::{info, warn}; + +/// Extra quiet after the workload stops, before the wakeup logs are read. +/// +/// The rest of the settle is derived from the configuration: the last round's +/// completion goes out one dwell after the workload stops accepting new rounds, +/// and if its waiter's executor died holding it, the resume costs up to one +/// wakeup budget on top. Reading before that elapsed would report wakeups as +/// lost that were merely late, which is the one mistake this scenario cannot +/// afford. +const SETTLE_MARGIN: Duration = Duration::from_secs(30); + +/// How many waiters to sample after the baseline to prove wakeups happen at all. +/// +/// A handful, because this is a smoke test rather than a measurement: if the +/// completion path is broken, every waiter is equally broken, and the point is to +/// fail before spending the fault window on a run that would report a clean +/// account of nothing. +const WAKE_PROOF_SAMPLE: usize = 5; + +pub async fn run( + config: &ScenarioConfig, + manifest: &ChaosPrepManifest, + deps: &BenchmarkTestDependencies, + signals: &FaultSignals, + outputs: &OutputPaths, +) -> anyhow::Result { + let started_at = Utc::now(); + let promise_config = config.require_promise()?; + let history = OperationHistory::new(ScenarioCode::S11.as_str()); + let key_prefix = crate::chaos::scenario_key_prefix(ScenarioCode::S11); + + let user = manifest.user_context(deps); + let counters = user + .get_latest_component_revision(&manifest.counters_component_id) + .await?; + let promise = user + .get_latest_component_revision(&manifest.promise_component_id) + .await?; + + let ctx = WorkloadContext { + user, + counters, + promise, + history: history.clone(), + retry: config.retry_policy.clone(), + phase: PhaseMarker::new(Phase::Baseline), + key_prefix: key_prefix.clone(), + }; + + let scope = RunScope { + environment_id: manifest.environment_id.0.to_string(), + // The promise component, not the counters one: S11's agents live there, + // and a reader narrowing traces by component needs the one that was + // actually driven. + component_ids: vec![manifest.promise_component_id.0.to_string()], + agent_id_prefix: key_prefix.clone(), + idempotency_key_prefix: format!("{key_prefix}-"), + }; + + let waiter_names: Vec = (0..promise_config.waiters) + .map(|index| ctx.waiter_name(index)) + .collect(); + + let mut phases = Phases::default(); + let mut routing_snapshots = Vec::new(); + let mut fault_injected_at = None; + let mut fault_recovered_at = None; + let mut fault_id = None; + let mut fault_target_observed = None; + let mut selection: Option = None; + let mut attention_extra: Vec = Vec::new(); + + macro_rules! finish { + ($reason:expr, $records:expr, $wakeups:expr) => {{ + let mut summary = ChaosSummary::build( + $records, + Vec::new(), + routing_snapshots.clone(), + fault_injected_at, + ); + summary.absorb(attention_extra.clone()); + if let Some(report) = $wakeups { + summary = summary.with_promise_wakeups(report); + } + let result = build_result( + config, + ScenarioOutcome { + started_at, + phases: phases.clone(), + fault_injected_at, + fault_recovered_at, + fault_id: fault_id.clone(), + fault_target_observed: fault_target_observed.clone(), + scope: scope.clone(), + summary, + termination_reason: $reason, + pinned_selection: None, + scheduled_selection: None, + promise_selection: selection.clone(), + }, + ); + write_outputs(&result, &history, outputs)?; + return Ok(result); + }}; + } + + // ── Warm-up ───────────────────────────────────────────────────────────── + routing_snapshots.push(snapshot_routing(deps, "before-warmup").await); + attention_extra.push(wait_for_settled_routing(deps, &mut routing_snapshots).await); + info!( + "S11: warming {} waiters before the baseline", + waiter_names.len() + ); + let warmed = waiters::warm(&ctx, &waiter_names).await; + info!("S11: warmed {warmed} waiters, settling {WARMUP_SETTLE:?}"); + tokio::time::sleep(WARMUP_SETTLE).await; + + // ── Aim the fault ─────────────────────────────────────────────────────── + // Before the baseline, because a run that cannot be aimed should not spend a + // maintenance window proving it. + let chosen = match split::select(split::waiter_subject(&ctx), deps, &waiter_names).await { + Ok(chosen) => chosen, + Err(e) => { + warn!("S11: could not aim the fault at an executor: {e:#}"); + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!("{e:#}"), + }, + &records, + None + ); + } + }; + selection = Some(chosen.clone()); + routing_snapshots.push(snapshot_routing(deps, "before-fault").await); + + // ── Baseline ──────────────────────────────────────────────────────────── + info!( + "S11: baseline phase, {} waiters parking for {:?} a round, for {:?}", + waiter_names.len(), + promise_config.dwell(), + config.phases.baseline() + ); + phases.baseline = Some(PhaseWindow::started(Utc::now())); + let handle = waiters::start(ctx.clone(), &waiter_names, promise_config); + tokio::time::sleep(config.phases.baseline()).await; + if let Some(window) = phases.baseline.as_mut() { + window.end(Utc::now()); + } + + let baseline_operations = history.confirmed_in_phase(Phase::Baseline); + if baseline_operations == 0 { + warn!("S11: baseline completed no promise round, aborting before injection"); + handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::PlatformUnreachable { + detail: "no promise round succeeded during the baseline phase".to_string(), + }, + &records, + None + ); + } + + // Completing is not waking. A platform that accepted every completion and + // resumed nobody would otherwise reach read-back and report a flawless + // account of a mechanism that never ran. + let sampled = sample_wakes(&ctx, &waiter_names).await; + if sampled == 0 { + warn!("S11: {baseline_operations} operations accepted and no waiter has woken"); + handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::StreamNeverSucceeded { + stream: Stream::PromiseWait.to_string(), + }, + &records, + None + ); + } + info!( + "S11: baseline complete ({baseline_operations} operations, {sampled} wakeups across a \ + sample of {} waiters)", + WAKE_PROOF_SAMPLE.min(waiter_names.len()) + ); + + // ── Verify ownership, then signal ─────────────────────────────────────── + if let Err(e) = split::verify_ownership(split::waiter_subject(&ctx), deps, &chosen).await { + warn!("S11: waiter ownership no longer holds, refusing to inject: {e:#}"); + handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!("{e:#}"), + }, + &records, + None + ); + } + + info!( + "S11: signalling readiness with fault target {} ({} of {} waiters on it)", + chosen.pod_address, + chosen.on_pod.len(), + waiter_names.len() + ); + signals.write_baseline_ready(&BaselineReady { + scenario_code: ScenarioCode::S11.as_str().to_string(), + ready_at: Utc::now(), + baseline_operations, + fault_target: Some(FaultTarget { + pod_address: chosen.pod_address.clone(), + pod_ip: chosen.pod_ip.clone(), + owned_agents: chosen.on_pod.clone(), + }), + })?; + + // ── Fault ─────────────────────────────────────────────────────────────── + let injected = match signals.await_fault_injected(config.signal_timeout()).await { + Ok(injected) => injected, + Err(e) => { + warn!("S11: no fault-injected signal arrived: {e}"); + handle.stop().await; + let records = history.snapshot(); + finish!(signal_termination(&e), &records, None); + } + }; + info!( + "S11: fault {} ({} on {}) reported active at {}", + injected.fault_id, injected.kind, injected.target, injected.injected_at + ); + fault_injected_at = Some(injected.injected_at); + fault_id = Some(injected.fault_id.clone()); + fault_target_observed = Some(injected.target.clone()); + ctx.phase.set(Phase::Fault); + phases.fault = Some(PhaseWindow::started(injected.injected_at)); + + let on_pod: BTreeSet = chosen.on_pod.iter().cloned().collect(); + let parked = parked_at_injection(&history.snapshot(), injected.injected_at, &on_pod); + attention_extra.push(parked.note()); + info!("S11: {}", parked.describe()); + + let recovered = match signals.await_fault_recovered(config.signal_timeout()).await { + Ok(recovered) => recovered, + Err(e) => { + warn!("S11: no fault-recovered signal arrived: {e}"); + handle.stop().await; + let records = history.snapshot(); + finish!(signal_termination(&e), &records, None); + } + }; + info!( + "S11: fault cleared at {} ({})", + recovered.recovered_at, recovered.termination_reason + ); + fault_recovered_at = Some(recovered.recovered_at); + if let Some(window) = phases.fault.as_mut() { + window.end(recovered.recovered_at); + } + + // ── Recovery ──────────────────────────────────────────────────────────── + info!( + "S11: recovery phase, running rounds for a further {:?}", + config.phases.recovery() + ); + ctx.phase.set(Phase::Recovery); + phases.recovery = Some(PhaseWindow::started(Utc::now())); + tokio::time::sleep(config.phases.recovery()).await; + let stood_down = handle.stalled(); + let rounds = handle.rounds(); + handle.stop().await; + if let Some(window) = phases.recovery.as_mut() { + window.end(Utc::now()); + } + routing_snapshots.push(snapshot_routing(deps, "after-recovery").await); + + // ── Account ───────────────────────────────────────────────────────────── + let settle = settle_before_readback(promise_config); + info!("S11: letting the last completions land, {settle:?} before read-back"); + tokio::time::sleep(settle).await; + + let records = history.snapshot(); + let logs = waiters::read_logs(&ctx, &waiter_names).await; + // Archived alongside the operations, not just reduced into the report: the + // reduced numbers cannot be recomputed later, and a correction to how a + // delay is classified has to be applicable to a run that has already + // happened. + history.record_wakeup_logs(logs.clone()); + + let report = WakeupReport::build( + &records, + &logs, + &chosen, + fault_injected_at.map(|injected_at| FaultWindow { + injected_at, + recovered_at: fault_recovered_at, + }), + promise_config.dwell(), + promise_config.wakeup_budget(), + stood_down, + ); + info!( + "S11: promise-wakeup account — {rounds} rounds started, {} completions accepted, {} \ + woke once, {} never woke, {} inconclusive, {} unverifiable, {} findings", + report.completions_confirmed, + report.woke_once, + report + .findings + .iter() + .filter(|f| f.violation == crate::chaos::wakeups::WakeupViolation::NeverWoke) + .count(), + report.inconclusive, + report.unverifiable, + report.findings.len() + ); + if let Some(p99) = report.fault_window_p99_ms() { + info!( + "S11: wakeup delay p99 during the fault, on the killed executor's waiters: {p99}ms \ + against a {}ms budget", + report.wakeup_budget_ms + ); + } + + let reason = if report.has_violations() { + TerminationReason::PromiseWakeupViolated { + findings: report.violations(), + first: report + .findings + .first() + .map(|f| format!("{} on token {}", f.violation, f.token)) + .unwrap_or_default(), + } + } else if report.woke_once == 0 { + TerminationReason::StreamNeverSucceeded { + stream: Stream::PromiseWait.to_string(), + } + } else { + TerminationReason::Completed + }; + + finish!(reason, &records, Some(report)); +} + +/// How long to wait after the workload stops before reading the wakeup logs. +fn settle_before_readback(config: &PromiseConfig) -> Duration { + config.dwell() + config.wakeup_budget() + SETTLE_MARGIN +} + +/// Waiters that were suspended on a promise when the executor died. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ParkedAtInjection { + pub total: u64, + pub on_killed_executor: u64, +} + +impl ParkedAtInjection { + /// Whether the kill landed anywhere near the mechanism under test. + /// + /// A run that caught nothing parked proves nothing however clean the rest of + /// its numbers look, which is the one thing here a human has to act on. + pub fn needs_attention(&self) -> bool { + self.total == 0 + } + + /// The same line as [`Self::describe`], classified. + pub fn note(&self) -> Note { + Note::leveled(self.needs_attention(), self.describe()) + } + + pub fn describe(&self) -> String { + if self.total == 0 { + return "WARNING: no waiter was suspended on a promise when the executor died, so \ + this run says nothing about promise-completion recovery" + .to_string(); + } + format!( + "S11 killed the executor with {} waiters suspended on promises, {} of them on \ + waiters it owned", + self.total, self.on_killed_executor + ) + } +} + +/// Counts the waiters parked across the moment of injection. +/// +/// A `wait` invocation that had been submitted before the kill and had not +/// returned by then. Not derived from the round arithmetic, because a platform +/// that had slowed down would have armed fewer rounds than the cadence says, and +/// the whole point of this number is to say whether the kill landed in anything. +pub fn parked_at_injection( + records: &[OperationRecord], + injected_at: DateTime, + on_killed_executor: &BTreeSet, +) -> ParkedAtInjection { + let mut parked = ParkedAtInjection { + total: 0, + on_killed_executor: 0, + }; + + for record in records + .iter() + .filter(|r| r.stream == Stream::PromiseWait && r.method == "wait") + .filter(|r| r.outcome != Outcome::Rejected) + .filter(|r| r.submitted_at <= injected_at) + .filter(|r| r.completed_at.is_none_or(|done| done > injected_at)) + { + parked.total += 1; + if on_killed_executor.contains(&record.agent) { + parked.on_killed_executor += 1; + } + } + + parked +} + +/// Total wakeups across a small sample of waiters. +async fn sample_wakes(ctx: &WorkloadContext, waiters_list: &[String]) -> u64 { + let sample: Vec = waiters_list + .iter() + .take(WAKE_PROOF_SAMPLE) + .cloned() + .collect(); + waiters::read_logs(ctx, &sample) + .await + .iter() + .filter_map(|log| log.wakes) + .sum() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::chaos::history::{Outcome, Phase, Stream}; + use test_r::test; + + fn at(secs: i64) -> DateTime { + DateTime::from_timestamp(secs, 0).unwrap() + } + + fn wait_record(agent: &str, submitted: i64, completed: Option) -> OperationRecord { + OperationRecord { + op_id: 1, + stream: Stream::PromiseWait, + phase: Phase::Baseline, + agent: agent.to_string(), + method: "wait".to_string(), + idempotency_key: format!("{agent}-wait"), + submitted_at: at(submitted), + completed_at: completed.map(at), + attempts: 1, + outcome: Outcome::Confirmed, + duration_ms: 0, + returned_value: None, + first_attempt_value: None, + error: None, + error_class: None, + attempt_log: Vec::new(), + } + } + + #[test] + fn a_waiter_still_parked_at_the_kill_is_counted() { + let records = vec![wait_record("w-1", 90, Some(150))]; + let on_pod: BTreeSet = ["w-1".to_string()].into_iter().collect(); + assert_eq!( + parked_at_injection(&records, at(100), &on_pod), + ParkedAtInjection { + total: 1, + on_killed_executor: 1, + } + ); + } + + /// A round that had already woken before the kill was not disturbed by it, + /// and counting it would overstate what the fault landed in. + #[test] + fn a_waiter_that_woke_before_the_kill_is_not_counted() { + let records = vec![wait_record("w-1", 80, Some(90))]; + let on_pod = BTreeSet::new(); + assert_eq!(parked_at_injection(&records, at(100), &on_pod).total, 0); + } + + /// A `wait` that never returned at all is the most interesting case there + /// is, so it must not fall out of the count for want of a completion time. + #[test] + fn a_wait_that_never_returned_is_still_counted_as_parked() { + let records = vec![wait_record("w-1", 90, None)]; + let on_pod = BTreeSet::new(); + assert_eq!(parked_at_injection(&records, at(100), &on_pod).total, 1); + } + + #[test] + fn a_kill_that_caught_nothing_parked_needs_attention() { + let parked = ParkedAtInjection { + total: 0, + on_killed_executor: 0, + }; + assert!(parked.needs_attention()); + assert!(parked.describe().contains("says nothing")); + } +} diff --git a/integration-tests/src/chaos/scenarios/s12.rs b/integration-tests/src/chaos/scenarios/s12.rs index de3adcbed5..e2865f5357 100644 --- a/integration-tests/src/chaos/scenarios/s12.rs +++ b/integration-tests/src/chaos/scenarios/s12.rs @@ -133,6 +133,7 @@ pub async fn run( termination_reason: $reason, pinned_selection: None, scheduled_selection: None, + promise_selection: None, }, ); write_outputs(&result, &history, outputs)?; diff --git a/integration-tests/src/chaos/scenarios/s13.rs b/integration-tests/src/chaos/scenarios/s13.rs index 5e8f98ddda..ab796a5753 100644 --- a/integration-tests/src/chaos/scenarios/s13.rs +++ b/integration-tests/src/chaos/scenarios/s13.rs @@ -204,6 +204,7 @@ pub async fn run( termination_reason: $reason, pinned_selection: None, scheduled_selection: None, + promise_selection: None, }, ); write_outputs(&result, &history, outputs)?; diff --git a/integration-tests/src/chaos/scenarios/s5.rs b/integration-tests/src/chaos/scenarios/s5.rs index afdce5e602..f07775afc1 100644 --- a/integration-tests/src/chaos/scenarios/s5.rs +++ b/integration-tests/src/chaos/scenarios/s5.rs @@ -176,6 +176,7 @@ pub async fn run( termination_reason: $reason, pinned_selection: None, scheduled_selection: None, + promise_selection: None, }, ); write_outputs(&result, &history, outputs)?; diff --git a/integration-tests/src/chaos/scenarios/s8.rs b/integration-tests/src/chaos/scenarios/s8.rs index 5bc824e7bb..10636310bd 100644 --- a/integration-tests/src/chaos/scenarios/s8.rs +++ b/integration-tests/src/chaos/scenarios/s8.rs @@ -153,6 +153,7 @@ pub async fn run( termination_reason: $reason, pinned_selection: selection.clone(), scheduled_selection: None, + promise_selection: None, }, ); write_outputs(&result, &history, outputs)?; diff --git a/integration-tests/src/chaos/scheduled.rs b/integration-tests/src/chaos/scheduled.rs index ced3104d8f..d7707f9743 100644 --- a/integration-tests/src/chaos/scheduled.rs +++ b/integration-tests/src/chaos/scheduled.rs @@ -46,18 +46,12 @@ use crate::chaos::ScheduledConfig; use crate::chaos::history::{FireRecord, Stream, TargetFireLog}; -use crate::chaos::pinned::{owners_by_pod, pod_ip_of}; -use crate::chaos::workload::{ - self, SCHEDULE_COUNTER_AGENT, SCHEDULE_EMITTER_AGENT, WorkloadContext, -}; -use anyhow::Context; +use crate::chaos::workload::{self, SCHEDULE_EMITTER_AGENT, WorkloadContext}; use chrono::{DateTime, Utc}; use golem_common::base_model::agent::ParsedAgentId; use golem_common::{agent_id, data_value}; -use golem_test_framework::config::{BenchmarkTestDependencies, TestDependencies}; +use golem_test_framework::config::BenchmarkTestDependencies; use golem_test_framework::dsl::TestDsl; -use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; use std::sync::Arc; use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; @@ -78,157 +72,36 @@ pub const MAX_IN_FLIGHT_PER_TARGET: usize = 8; /// maintenance window. const READ_CONCURRENCY: usize = 16; -/// The smallest share of targets one executor must own for the run to mean -/// anything, as a divisor of the target count. +/// The executor S10 aims at, and how its targets divide around it. /// -/// A two-executor cluster splits a hashed population roughly evenly, so a quarter -/// is a floor rather than an expectation. Below it the "affected" group is too -/// small for its percentile to say anything, and a run that reported one anyway -/// would be worse than one that refused. -const MIN_TARGET_SHARE_DIVISOR: usize = 4; - -/// The executor the fault will be aimed at, and how the targets divide around -/// it. -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ScheduledSelection { - /// The executor endpoint as the shard-manager names it, e.g. - /// `10.0.14.207:9000`. - pub pod_address: String, - /// Host part of the address, which is what a Kubernetes `status.podIP` - /// field selector matches. - pub pod_ip: String, - /// Targets this executor owns. The population whose actions have to survive - /// a lease recovery. - pub on_pod: Vec, - /// Targets owned by any other executor: the run's own control group. - pub elsewhere: Vec, - /// How the targets spread across executors, so a run that refused to - /// proceed says whether the cluster was lopsided or the pool too small. - pub targets_per_pod: BTreeMap, - /// Shard count the routing table reported. Ownership is a hash modulo this, - /// so a selection cannot be re-derived later without it. - pub number_of_shards: usize, -} +/// An alias rather than a type of its own: S11 splits its waiters exactly the +/// same way, the logic lives in [`crate::chaos::split`], and the name is kept +/// here because `scheduledSelection` is the key the archived result and the +/// golem-cloud report both read. +pub type ScheduledSelection = crate::chaos::split::PodSplit; -/// Chooses the executor to aim at: the one owning the largest share of targets. -/// -/// Fails rather than proceeding unaimed. Chaos Mesh's `mode: one` would pick a -/// pod at random, and a run that killed an executor owning six targets out of a -/// hundred would still produce a confident-looking report about lease recovery. +/// Chooses the executor to aim at. See [`crate::chaos::split::select`]. pub async fn select( ctx: &WorkloadContext, deps: &BenchmarkTestDependencies, targets: &[String], ) -> anyhow::Result { - let table = deps - .shard_manager() - .get_routing_table() - .await - .context("reading the routing table to aim the scheduled fault")?; - - let by_pod = owners_by_pod(ctx, &table, SCHEDULE_COUNTER_AGENT, targets); - let targets_per_pod: BTreeMap = by_pod - .iter() - .map(|(pod, xs)| (pod.clone(), xs.len())) - .collect(); - - let (pod_address, on_pod) = by_pod - .iter() - .max_by_key(|(_, agents)| agents.len()) - .map(|(pod, agents)| (pod.clone(), agents.clone())) - .ok_or_else(|| { - anyhow::anyhow!( - "routing table assigned none of the {} schedule targets to any executor", - targets.len() - ) - })?; - - let floor = (targets.len() / MIN_TARGET_SHARE_DIVISOR).max(1); - if on_pod.len() < floor { - anyhow::bail!( - "the most-loaded executor owns only {} of {} schedule targets, below the {floor} \ - needed for its share to be worth measuring: {targets_per_pod:?}", - on_pod.len(), - targets.len() - ); - } - - let elsewhere: Vec = targets - .iter() - .filter(|t| !on_pod.contains(t)) - .cloned() - .collect(); - - info!( - "S10: aiming at executor {pod_address}, which owns {} of {} schedule targets ({} \ - elsewhere, across {} executors)", - on_pod.len(), - targets.len(), - elsewhere.len(), - targets_per_pod.len() - ); - - Ok(ScheduledSelection { - pod_ip: pod_ip_of(&pod_address), - pod_address, - on_pod, - elsewhere, - targets_per_pod, - number_of_shards: table.number_of_shards.value, - }) + crate::chaos::split::select(crate::chaos::split::schedule_subject(ctx), deps, targets).await } -/// Re-checks, against a freshly read routing table, that the targets are still -/// divided the way the selection says. -/// -/// Called immediately before the readiness signal, for the same reason S8 does -/// it: a rebalance between selection and injection would leave the run reporting -/// a control group that was actually the affected one. +/// Re-checks the division immediately before injection. See +/// [`crate::chaos::split::verify_ownership`]. pub async fn verify_ownership( ctx: &WorkloadContext, deps: &BenchmarkTestDependencies, selection: &ScheduledSelection, ) -> anyhow::Result<()> { - let table = deps - .shard_manager() - .get_routing_table() - .await - .context("re-reading the routing table to verify scheduled target ownership")?; - - let mut drifted = Vec::new(); - for agent in &selection.on_pod { - let owner = table - .lookup(&crate::chaos::pinned::routing_agent_id( - ctx, - SCHEDULE_COUNTER_AGENT, - agent, - )) - .map(|pod| pod.to_string()); - if owner.as_deref() != Some(selection.pod_address.as_str()) { - drifted.push(format!( - "{agent} now owned by {}", - owner.unwrap_or_else(|| "nobody".to_string()) - )); - } - } - - if !drifted.is_empty() { - anyhow::bail!( - "{} of {} schedule targets are no longer owned by {}: {}", - drifted.len(), - selection.on_pod.len(), - selection.pod_address, - drifted.join(", ") - ); - } - - info!( - "S10: verified all {} schedule targets are still owned by {}", - selection.on_pod.len(), - selection.pod_address - ); - Ok(()) + crate::chaos::split::verify_ownership( + crate::chaos::split::schedule_subject(ctx), + deps, + selection, + ) + .await } /// A running registration workload. As elsewhere, dropping the handle does not diff --git a/integration-tests/src/chaos/split.rs b/integration-tests/src/chaos/split.rs new file mode 100644 index 0000000000..4ebe6f565c --- /dev/null +++ b/integration-tests/src/chaos/split.rs @@ -0,0 +1,366 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Aiming a kill at one executor while still driving the agents it does not own. +//! +//! [`crate::chaos::pinned`] answers a different question. There, an operation +//! that was not on the dead pod says nothing, so the driver keeps only the +//! agents its chosen executor owns and discards the rest. Here every agent is +//! driven and the ones elsewhere are the run's own control group: on a +//! two-executor cluster roughly half the population is never touched, and +//! reporting one percentile across both would let a recovery that took its full +//! budget hide behind the half that was never disturbed. +//! +//! Both scenarios that work this way — S10's schedule targets and S11's promise +//! waiters — need exactly the same three things, which is why they live here +//! rather than being written twice: pick the executor owning the largest share, +//! refuse to proceed if that share is too small to mean anything, and re-check +//! the division immediately before the fault is injected. + +use crate::chaos::pinned::{owners_by_pod_in, pod_ip_of, routing_agent_id_in}; +use crate::chaos::workload::WorkloadContext; +use anyhow::Context; +use chrono::{DateTime, Utc}; +use golem_common::model::component::ComponentDto; +use golem_test_framework::config::{BenchmarkTestDependencies, TestDependencies}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use tracing::info; + +/// The smallest share of agents one executor must own for the run to mean +/// anything, as a divisor of the population. +/// +/// A two-executor cluster splits a hashed population roughly evenly, so a +/// quarter is a floor rather than an expectation. Below it the "affected" group +/// is too small for its percentile to say anything, and a run that reported one +/// anyway would be worse than one that refused. +const MIN_SHARE_DIVISOR: usize = 4; + +/// What the split is about, for the messages a reader eventually sees. +/// +/// Carried rather than hard-coded because the failure modes here are reported to +/// an operator mid-maintenance-window, and "the most-loaded executor owns only 6 +/// of 100 agents" is a worse thing to read at 3am than the same sentence naming +/// promise waiters. +#[derive(Debug, Clone, Copy)] +pub struct Subject<'a> { + /// Scenario code, used only to prefix log lines. + pub scenario: &'a str, + /// The component the agents live in. Ownership is per agent id and an agent + /// id contains its component, so this is not cosmetic. + pub component: &'a ComponentDto, + /// Agent type, e.g. `ScheduleCounter`. + pub agent_type: &'a str, + /// Plural noun for messages, e.g. `schedule targets`. + pub noun: &'a str, +} + +/// The fault window, as the workflow reported it. +#[derive(Debug, Clone, Copy)] +pub struct FaultWindow { + pub injected_at: DateTime, + /// Absent for a run that never saw the fault clear. + pub recovered_at: Option>, +} + +/// Which side of the fault an event fell on. +/// +/// Shared rather than written per scenario because the classification is the +/// same question every time — an event before the kill, while the executor was +/// gone, or after it came back — and because the three names end up in archived +/// results that a reader compares across scenarios. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum Window { + BeforeFault, + DuringFault, + AfterFault, + /// The run never learned when the fault was injected, so nothing can be + /// placed relative to it. + Unknown, +} + +impl Window { + pub fn as_str(self) -> &'static str { + match self { + Window::BeforeFault => "before-fault", + Window::DuringFault => "during-fault", + Window::AfterFault => "after-fault", + Window::Unknown => "unknown", + } + } + + /// Where `at` falls relative to the fault. + pub fn of(at: DateTime, fault: Option) -> Self { + match fault { + None => Window::Unknown, + Some(window) if at < window.injected_at => Window::BeforeFault, + Some(FaultWindow { + recovered_at: Some(recovered), + .. + }) if at >= recovered => Window::AfterFault, + Some(_) => Window::DuringFault, + } + } +} + +impl std::fmt::Display for Window { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// The executor the fault will be aimed at, and how the agents divide around it. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PodSplit { + /// The executor endpoint as the shard-manager names it, e.g. + /// `10.0.14.207:9000`. + pub pod_address: String, + /// Host part of the address, which is what a Kubernetes `status.podIP` + /// field selector matches. + pub pod_ip: String, + /// Agents this executor owns: the population that has to survive recovery. + pub on_pod: Vec, + /// Agents owned by any other executor: the run's own control group. + pub elsewhere: Vec, + /// How the agents spread across executors, so a run that refused to proceed + /// says whether the cluster was lopsided or the pool too small. + pub targets_per_pod: BTreeMap, + /// Shard count the routing table reported. Ownership is a hash modulo this, + /// so a selection cannot be re-derived later without it. + pub number_of_shards: usize, +} + +impl PodSplit { + /// Which group an agent belongs to, or `None` for one the selection never + /// saw. + pub fn group_of(&self, agent: &str) -> Option { + if self.on_pod.iter().any(|a| a == agent) { + Some(Group::OnPod) + } else if self.elsewhere.iter().any(|a| a == agent) { + Some(Group::Elsewhere) + } else { + None + } + } +} + +/// Which side of the kill an agent was on. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Group { + /// Owned by the executor the fault was aimed at. + OnPod, + /// Owned by some other executor: the control group. + Elsewhere, +} + +impl Group { + pub fn as_str(self) -> &'static str { + match self { + Group::OnPod => "on-pod", + Group::Elsewhere => "elsewhere", + } + } +} + +/// Chooses the executor to aim at: the one owning the largest share of agents. +/// +/// Fails rather than proceeding unaimed. Chaos Mesh's `mode: one` would pick a +/// pod at random, and a run that killed an executor owning six agents out of a +/// hundred would still produce a confident-looking report about recovery. +pub async fn select( + subject: Subject<'_>, + deps: &BenchmarkTestDependencies, + agents: &[String], +) -> anyhow::Result { + let table = deps + .shard_manager() + .get_routing_table() + .await + .with_context(|| { + format!( + "reading the routing table to aim the {} fault", + subject.noun + ) + })?; + + let by_pod = owners_by_pod_in(subject.component, &table, subject.agent_type, agents); + let targets_per_pod: BTreeMap = by_pod + .iter() + .map(|(pod, xs)| (pod.clone(), xs.len())) + .collect(); + + let (pod_address, on_pod) = by_pod + .iter() + .max_by_key(|(_, agents)| agents.len()) + .map(|(pod, agents)| (pod.clone(), agents.clone())) + .ok_or_else(|| { + anyhow::anyhow!( + "routing table assigned none of the {} {} to any executor", + agents.len(), + subject.noun + ) + })?; + + let floor = (agents.len() / MIN_SHARE_DIVISOR).max(1); + if on_pod.len() < floor { + anyhow::bail!( + "the most-loaded executor owns only {} of {} {}, below the {floor} needed for its \ + share to be worth measuring: {targets_per_pod:?}", + on_pod.len(), + agents.len(), + subject.noun + ); + } + + let elsewhere: Vec = agents + .iter() + .filter(|t| !on_pod.contains(t)) + .cloned() + .collect(); + + info!( + "{}: aiming at executor {pod_address}, which owns {} of {} {} ({} elsewhere, across {} \ + executors)", + subject.scenario, + on_pod.len(), + agents.len(), + subject.noun, + elsewhere.len(), + targets_per_pod.len() + ); + + Ok(PodSplit { + pod_ip: pod_ip_of(&pod_address), + pod_address, + on_pod, + elsewhere, + targets_per_pod, + number_of_shards: table.number_of_shards.value, + }) +} + +/// Re-checks, against a freshly read routing table, that the agents are still +/// divided the way the selection says. +/// +/// Called immediately before the readiness signal, for the same reason +/// [`crate::chaos::pinned`] does it: a rebalance between selection and injection +/// would leave the run reporting a control group that was actually the affected +/// one. +pub async fn verify_ownership( + subject: Subject<'_>, + deps: &BenchmarkTestDependencies, + split: &PodSplit, +) -> anyhow::Result<()> { + let table = deps + .shard_manager() + .get_routing_table() + .await + .with_context(|| { + format!( + "re-reading the routing table to verify {} ownership", + subject.noun + ) + })?; + + let mut drifted = Vec::new(); + for agent in &split.on_pod { + let owner = table + .lookup(&routing_agent_id_in( + subject.component, + subject.agent_type, + agent, + )) + .map(|pod| pod.to_string()); + if owner.as_deref() != Some(split.pod_address.as_str()) { + drifted.push(format!( + "{agent} now owned by {}", + owner.unwrap_or_else(|| "nobody".to_string()) + )); + } + } + + if !drifted.is_empty() { + anyhow::bail!( + "{} of {} {} are no longer owned by {}: {}", + drifted.len(), + split.on_pod.len(), + subject.noun, + split.pod_address, + drifted.join(", ") + ); + } + + info!( + "{}: verified all {} {} are still owned by {}", + subject.scenario, + split.on_pod.len(), + subject.noun, + split.pod_address + ); + Ok(()) +} + +/// The counters component's schedule targets, as S10 aims at them. +pub fn schedule_subject<'a>(ctx: &'a WorkloadContext) -> Subject<'a> { + Subject { + scenario: "S10", + component: &ctx.counters, + agent_type: crate::chaos::workload::SCHEDULE_COUNTER_AGENT, + noun: "schedule targets", + } +} + +/// The promise component's waiters, as S11 aims at them. +pub fn waiter_subject<'a>(ctx: &'a WorkloadContext) -> Subject<'a> { + Subject { + scenario: "S11", + component: &ctx.promise, + agent_type: crate::chaos::waiters::PROMISE_WAITER_AGENT, + noun: "promise waiters", + } +} + +#[cfg(test)] +mod tests { + use super::*; + use test_r::test; + + fn split() -> PodSplit { + PodSplit { + pod_address: "10.0.1.1:9000".to_string(), + pod_ip: "10.0.1.1".to_string(), + on_pod: vec!["a".to_string(), "b".to_string()], + elsewhere: vec!["c".to_string()], + targets_per_pod: BTreeMap::new(), + number_of_shards: 1024, + } + } + + #[test] + fn a_split_places_each_agent_in_exactly_one_group() { + let split = split(); + assert_eq!(split.group_of("a"), Some(Group::OnPod)); + assert_eq!(split.group_of("c"), Some(Group::Elsewhere)); + } + + /// An agent the selection never saw is not silently counted as a control: + /// the caller has to decide what an unknown agent means, because in every + /// scenario here it means the population drifted. + #[test] + fn an_agent_outside_the_selection_belongs_to_no_group() { + assert_eq!(split().group_of("z"), None); + } +} diff --git a/integration-tests/src/chaos/summary.rs b/integration-tests/src/chaos/summary.rs index b4f9c73de4..3a47aaf0c5 100644 --- a/integration-tests/src/chaos/summary.rs +++ b/integration-tests/src/chaos/summary.rs @@ -52,6 +52,7 @@ use crate::chaos::fires::ScheduleFireReport; use crate::chaos::history::{Outcome, Phase, Stream}; use crate::chaos::ownership::OwnershipSample; use crate::chaos::probe::KeyProbe; +use crate::chaos::wakeups::WakeupReport; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::time::Duration; @@ -566,6 +567,11 @@ pub struct ChaosSummary { /// found". #[serde(default, skip_serializing_if = "Option::is_none")] pub schedule_fires: Option, + /// The promise-wakeup account, for scenarios that pair completions against + /// the waiters they were supposed to resume. Absent for scenarios that do + /// not, for the same reason as `scheduleFires`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub promise_wakeups: Option, /// Shard-ownership samples, in the order they were taken. Empty for /// scenarios that do not sample executor assignments. /// @@ -696,6 +702,7 @@ impl ChaosSummary { routing_snapshots, exactly_once: None, schedule_fires: None, + promise_wakeups: None, ownership: Vec::new(), attention, notes: Vec::new(), @@ -740,6 +747,21 @@ impl ChaosSummary { self } + /// Attaches the promise-wakeup account and hoists everything it wants a + /// human to see into [`Self::attention`]. + /// + /// Same split as [`Self::with_schedule_fires`], and one extra reason for it: + /// a waiter that could not be read is normally the weakest outcome there is, + /// but for a suspended waiter it can be the strongest evidence in the run. + /// The report decides which, and only what it classifies as attention lands + /// there. + pub fn with_promise_wakeups(mut self, report: WakeupReport) -> Self { + self.attention.extend(report.attention_lines()); + self.notes.extend(report.note_lines()); + self.promise_wakeups = Some(report); + self + } + /// Attaches the shard-ownership samples and hoists their findings into /// [`Self::attention`]. /// @@ -794,6 +816,12 @@ pub enum TerminationReason { /// action paired with one named registration, with no band of doubt around /// it. See [`crate::chaos::fires`]. ScheduledFireViolated { findings: u64, first: String }, + /// A promise completion the platform accepted never woke its waiter, woke it + /// twice, or woke it after being refused. Asserted for the same reason as + /// [`Self::ScheduledFireViolated`]: each is a statement about one named + /// completion paired with one named waiter, with no band of doubt around it. + /// See [`crate::chaos::wakeups`]. + PromiseWakeupViolated { findings: u64, first: String }, /// An agent's durable state did not survive a component update. Asserted /// because an update is supposed to change what an agent runs and nothing /// about what it remembers — state that moved is the one outcome an update @@ -1066,12 +1094,19 @@ mod tests { /// A reader must never have to wonder whether a stream was skipped or just /// had nothing to say. + /// + /// The waiter stream is here for a different reason from the other two, and + /// the distinction is worth keeping straight: those two have no durable + /// count to read, while this one has a count that is *weaker* than what the + /// scenario already does with it. Its absence from the count-based read-back + /// means the token pairing in `promiseWakeups` is the account, not that + /// nothing was checked. #[test] fn streams_without_readback_are_named_rather_than_omitted() { let summary = ChaosSummary::build(&[], Vec::new(), Vec::new(), None); assert_eq!( summary.streams_without_readback, - vec![Stream::Ephemeral, Stream::Promise] + vec![Stream::Ephemeral, Stream::Promise, Stream::PromiseWait] ); } diff --git a/integration-tests/src/chaos/waiters.rs b/integration-tests/src/chaos/waiters.rs new file mode 100644 index 0000000000..68a569811b --- /dev/null +++ b/integration-tests/src/chaos/waiters.rs @@ -0,0 +1,549 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The suspended-waiter workload (GOL-377). +//! +//! Every other stream in this suite keeps an agent *busy*. This one keeps a pool +//! of them asleep. Each waiter repeats one round: +//! +//! 1. `arm(token)` creates a promise and returns it. Fast, and the only part the +//! driver needs an answer from. +//! 2. `wait(token, promise)` parks the agent on that promise. The invocation +//! stays open — on the platform as a suspended worker, in the driver as a +//! task holding a connection. +//! 3. After `dwellMillis`, the driver completes the promise from the outside. +//! 4. The wakeup ends the round, and the waiter starts the next one. +//! +//! ## Why one promise per waiter +//! +//! An agent runs its invocations one at a time, so a waiter parked in `wait` +//! cannot be armed again until it wakes. That is not a limitation to work +//! around; it is what makes the pool size mean something exact. With `waiters` +//! agents and one promise each, the number of agents standing suspended at the +//! instant the pod dies is a known constant rather than a sample — the same +//! property [`crate::chaos::pinned`] gets from one in-flight operation per +//! agent, for the same reason. +//! +//! It also makes a lost wakeup visible while the run is still going. A waiter +//! whose completion never arrives can never start another round, so it simply +//! stops producing. See [`WaiterHandle::stalled`]. +//! +//! ## Why the dwell +//! +//! `dwellMillis` decides how the population divides at the moment of the kill. +//! Every waiter is either parked-and-waiting (the dwell) or being-woken (the +//! completion round trip), and the dwell is far the longer of the two — so the +//! kill lands mostly in the first, which is the state this scenario is about, +//! and occasionally in the second, which is the narrower race S8 already covers +//! for ordinary invocations. +//! +//! The dwell also has to comfortably exceed the workflow's inject-and-verify +//! path — signal poll, `kubectl apply`, waiting for `AllInjected` — or every +//! promise armed before the kill would already have been completed by the time +//! the pod died, and the run would measure nothing. +//! +//! ## Why completions are retried like everything else +//! +//! The suite's retry policy — one same-key retry, transport failures only — +//! applies here too, and a completion is the one operation where that could +//! plausibly be accused of hiding the defect: `complete` writes with +//! `set_if_not_exists` and re-triggers the wakeup, so a retry can repair a +//! completion whose wakeup was lost. +//! +//! It cannot hide what this scenario asks, because the question is scoped to +//! completions the platform *confirmed*. A retry only happens when the previous +//! attempt did not return success, so a confirmed completion is always a single +//! accepted call. "The platform said yes and the waiter never woke" is exactly +//! as detectable with the retry as without it, and the retry keeps the workload +//! behaving like a client anyone would actually write. + +use crate::chaos::PromiseConfig; +use crate::chaos::history::{Stream, WaiterWakeupLog, WakeupRecord}; +use crate::chaos::workload::{self, WorkloadContext}; +use chrono::{DateTime, TimeZone, Utc}; +use golem_common::base_model::agent::ParsedAgentId; +use golem_common::model::PromiseId; +use golem_common::{agent_id, data_value}; +use golem_test_framework::dsl::TestDsl; +use golem_wasm::FromValue; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::time::Duration; +use tokio::task::JoinSet; +use tracing::{info, warn}; + +/// Agent type exported by the promise component for this workload. +pub const PROMISE_WAITER_AGENT: &str = "PromiseWaiter"; + +/// Payload a completion carries. Nothing reads it; the token is what identifies +/// a round, and it travels through the agent rather than through the promise. +const COMPLETION_PAYLOAD: &[u8] = b"chaos-s11"; + +/// How long a waiter's loop waits for its own wakeup before concluding the +/// waiter is parked for good and standing down. +/// +/// Deliberately a multiple of the wakeup budget rather than a fixed number: the +/// budget is already the run's statement about what a recovery may cost, and +/// anything that treats "slow" as "lost" would turn a bad p99 into a false +/// finding. Standing down is not a verdict either — the read-back still asks the +/// waiter what happened, and a waiter that woke late says so. +const STALL_MULTIPLE: u32 = 4; + +/// Floor under [`STALL_MULTIPLE`], for a configuration with a very short budget. +const MIN_STALL_TIMEOUT: Duration = Duration::from_secs(180); + +/// How many waiters are read back at once. Same reasoning as +/// [`crate::chaos::scenarios::read_back_agents`]: reads do not mutate, and +/// walking a few hundred of them one at a time behind a per-read ceiling +/// outlasts the maintenance window. +const READ_CONCURRENCY: usize = 16; + +/// Ceiling on one wakeup-log read. Generous, because it happens on a cluster +/// that has just been through a fault, and because the answer carries every +/// wakeup the waiter recorded rather than one number. +const READ_TIMEOUT: Duration = Duration::from_secs(60); + +/// A running waiter workload. +/// +/// As elsewhere, dropping the handle does not stop it: call +/// [`WaiterHandle::stop`] so rounds in flight record themselves instead of being +/// cancelled mid-completion. +pub struct WaiterHandle { + tasks: JoinSet<()>, + running: Arc, + rounds: Arc, + stalled: Arc, +} + +impl WaiterHandle { + /// Rounds started across all waiters. + pub fn rounds(&self) -> u64 { + self.rounds.load(Ordering::Relaxed) + } + + /// Waiters that stood down because a wakeup never arrived. + /// + /// The live half of this scenario's oracle. A waiter counted here was parked + /// on a promise the driver had completed, and stayed parked long enough that + /// no recovery budget explains it. The read-back then decides whether it + /// eventually woke. + pub fn stalled(&self) -> u64 { + self.stalled.load(Ordering::Relaxed) + } + + pub async fn stop(mut self) { + self.running.store(false, Ordering::Relaxed); + while self.tasks.join_next().await.is_some() {} + } +} + +/// Warms every waiter so the baseline measures resident agents rather than cold +/// starts. Returns how many answered. +pub async fn warm(ctx: &WorkloadContext, waiters: &[String]) -> usize { + let mut warmed = 0; + let mut set = JoinSet::new(); + for waiter in waiters { + let ctx = ctx.clone(); + let waiter = waiter.clone(); + set.spawn(async move { + let parsed: ParsedAgentId = agent_id!(PROMISE_WAITER_AGENT, waiter.clone()); + ctx.user + .invoke_and_await_agent(&ctx.promise, &parsed, "wakes", data_value!()) + .await + .is_ok() + }); + } + while let Some(result) = set.join_next().await { + if matches!(result, Ok(true)) { + warmed += 1; + } + } + warmed +} + +/// Starts one loop per waiter. +pub fn start(ctx: WorkloadContext, waiters: &[String], config: &PromiseConfig) -> WaiterHandle { + let running = Arc::new(AtomicBool::new(true)); + let rounds = Arc::new(AtomicU64::new(0)); + let stalled = Arc::new(AtomicU64::new(0)); + let mut tasks = JoinSet::new(); + + let stall_timeout = (config.wakeup_budget() * STALL_MULTIPLE).max(MIN_STALL_TIMEOUT); + info!( + "S11: starting {} waiters, {:?} dwell, standing a waiter down after {stall_timeout:?} \ + without a wakeup", + waiters.len(), + config.dwell() + ); + + for waiter in waiters { + let ctx = ctx.clone(); + let waiter = waiter.clone(); + let running = running.clone(); + let rounds = rounds.clone(); + let stalled = stalled.clone(); + let dwell = config.dwell(); + tasks.spawn(async move { + run_waiter(ctx, waiter, dwell, stall_timeout, running, rounds, stalled).await; + }); + } + + WaiterHandle { + tasks, + running, + rounds, + stalled, + } +} + +/// One waiter's rounds, until the workload stops or the waiter stands down. +async fn run_waiter( + ctx: WorkloadContext, + waiter: String, + dwell: Duration, + stall_timeout: Duration, + running: Arc, + rounds: Arc, + stalled: Arc, +) { + let parsed: ParsedAgentId = agent_id!(PROMISE_WAITER_AGENT, waiter.clone()); + let mut round = 0u64; + + while running.load(Ordering::Relaxed) { + let token = ctx.idempotency_key(&waiter, round); + round += 1; + rounds.fetch_add(1, Ordering::Relaxed); + + let Some(promise_id) = arm(&ctx, &waiter, &parsed, &token).await else { + // Nothing to complete and nobody parked. The next round tries again + // after a dwell, which keeps a wholly unreachable platform from + // spinning. + tokio::time::sleep(dwell).await; + continue; + }; + + // Park the waiter. Held open deliberately: the driver's own view of the + // wakeup is one of the two independent answers this scenario collects, + // and the fault is expected to take it away. + let wait_task = { + let ctx = ctx.clone(); + let waiter = waiter.clone(); + let parsed = parsed.clone(); + let token = token.clone(); + let promise_id = promise_id.clone(); + tokio::spawn(async move { wait(&ctx, &waiter, &parsed, &token, &promise_id).await }) + }; + + tokio::time::sleep(dwell).await; + complete(&ctx, &waiter, &token, &promise_id).await; + + match tokio::time::timeout(stall_timeout, wait_task).await { + Ok(_) => {} + Err(_) => { + warn!( + "S11: waiter {waiter} has not woken {stall_timeout:?} after its completion \ + on token {token}; standing it down" + ); + stalled.fetch_add(1, Ordering::Relaxed); + return; + } + } + } +} + +/// Creates the round's promise, recording the invocation. +/// +/// The promise id comes back through a cell rather than a return value because +/// [`workload::run_operation`] owns the retry rule and the failure +/// classification for every stream in the suite, and it reports outcomes rather +/// than payloads. Duplicating it here to get one value back would put the two +/// load-bearing rules of every chaos scenario in two places. +async fn arm( + ctx: &WorkloadContext, + waiter: &str, + parsed: &ParsedAgentId, + token: &str, +) -> Option { + let cell: Arc>> = Arc::new(std::sync::Mutex::new(None)); + let sink = cell.clone(); + let ctx2 = ctx.clone(); + let parsed2 = parsed.clone(); + + workload::run_operation( + ctx, + Stream::PromiseWait, + waiter.to_string(), + "arm", + format!("{token}-arm"), + |key| { + let ctx = ctx2.clone(); + let parsed = parsed2.clone(); + let sink = sink.clone(); + let token = token.to_string(); + async move { + let created = ctx + .user + .invoke_and_await_agent_with_key( + &ctx.promise, + &parsed, + &key, + "arm", + data_value!(token), + ) + .await?; + let value = created + .into_return_value_and_type() + .ok_or_else(|| anyhow::anyhow!("arm returned no promise id"))?; + let promise_id = PromiseId::from_value(value.value) + .map_err(|e| anyhow::anyhow!("invalid promise id: {e}"))?; + *sink.lock().unwrap() = Some(promise_id); + Ok(None) + } + }, + ) + .await; + + cell.lock().unwrap().clone() +} + +/// Parks the waiter until its promise resolves. +async fn wait( + ctx: &WorkloadContext, + waiter: &str, + parsed: &ParsedAgentId, + token: &str, + promise_id: &PromiseId, +) { + let ctx2 = ctx.clone(); + let parsed2 = parsed.clone(); + workload::run_operation( + ctx, + Stream::PromiseWait, + waiter.to_string(), + "wait", + format!("{token}-wait"), + |key| { + let ctx = ctx2.clone(); + let parsed = parsed2.clone(); + let token = token.to_string(); + let promise_id = promise_id.clone(); + async move { + ctx.user + .invoke_and_await_agent_with_key( + &ctx.promise, + &parsed, + &key, + "wait", + data_value!(token, promise_id), + ) + .await?; + Ok(None) + } + }, + ) + .await; +} + +/// Completes the round's promise from outside the agent. +/// +/// Recorded under the round's token rather than under a key of its own, which is +/// what joins this record to the wakeup the waiter logs. The completion API is +/// keyed by promise id and takes no idempotency key of its own — a retry is the +/// same completion because it names the same promise, and `set_if_not_exists` on +/// the platform side is what makes that true. +async fn complete(ctx: &WorkloadContext, waiter: &str, token: &str, promise_id: &PromiseId) { + let ctx2 = ctx.clone(); + workload::run_operation( + ctx, + Stream::PromiseWait, + waiter.to_string(), + "complete", + token.to_string(), + |_key| { + let ctx = ctx2.clone(); + let promise_id = promise_id.clone(); + async move { + ctx.user + .complete_promise(&promise_id, COMPLETION_PAYLOAD.to_vec()) + .await?; + Ok(None) + } + }, + ) + .await; +} + +/// Reads every waiter's wakeup log. +pub async fn read_logs(ctx: &WorkloadContext, waiters: &[String]) -> Vec { + let mut logs = Vec::with_capacity(waiters.len()); + for chunk in waiters.chunks(READ_CONCURRENCY) { + let mut set = JoinSet::new(); + for waiter in chunk { + let ctx = ctx.clone(); + let waiter = waiter.clone(); + set.spawn(async move { read_log(&ctx, &waiter).await }); + } + while let Some(result) = set.join_next().await { + match result { + Ok(log) => logs.push(log), + Err(e) => warn!("S11: a wakeup-log read task failed: {e}"), + } + } + } + logs.sort_by(|a, b| a.agent.cmp(&b.agent)); + logs +} + +async fn read_log(ctx: &WorkloadContext, waiter: &str) -> WaiterWakeupLog { + let parsed: ParsedAgentId = agent_id!(PROMISE_WAITER_AGENT, waiter.to_string()); + + let wakes = match read_within(waiter, "wakes", async { + ctx.user + .invoke_and_await_agent(&ctx.promise, &parsed, "wakes", data_value!()) + .await + .map_err(|e| format!("{e:#}")) + }) + .await + { + Ok(value) => value + .into_return_value_and_type() + .and_then(|v| u32::from_value(v.value).ok()) + .map(|v| v as u64), + Err(e) => { + return WaiterWakeupLog { + agent: waiter.to_string(), + wakes: None, + wakeups: Vec::new(), + error: Some(e), + }; + } + }; + + match read_within(waiter, "wakeups", async { + ctx.user + .invoke_and_await_agent(&ctx.promise, &parsed, "wakeups", data_value!()) + .await + .map_err(|e| format!("{e:#}")) + }) + .await + { + Ok(value) => { + let wakeups = value + .into_return_value_and_type() + .map(|v| parse_wakeups(v.value)) + .unwrap_or_default(); + WaiterWakeupLog { + agent: waiter.to_string(), + wakes, + wakeups, + error: None, + } + } + Err(e) => WaiterWakeupLog { + agent: waiter.to_string(), + wakes, + wakeups: Vec::new(), + error: Some(e), + }, + } +} + +/// A read-back invocation under [`READ_TIMEOUT`]. +/// +/// A timeout is a verdict, not an error to propagate. It is also the loudest +/// thing this scenario can observe: a waiter that will not answer `wakeups` is +/// usually a waiter still parked on a promise that was completed long ago, and +/// [`crate::chaos::wakeups`] treats that case differently from an ordinary +/// unreadable agent. +async fn read_within(waiter: &str, what: &str, read: F) -> Result +where + F: std::future::Future>, +{ + match tokio::time::timeout(READ_TIMEOUT, read).await { + Ok(result) => result, + Err(_) => { + warn!("S11: reading {what} on waiter {waiter} timed out after {READ_TIMEOUT:?}"); + Err(format!("{what} timed out after {READ_TIMEOUT:?}")) + } + } +} + +/// Turns the agent's `(token, armed_millis, woken_millis)` triples into records. +fn parse_wakeups(value: golem_wasm::Value) -> Vec { + let raw: Vec<(String, u64, u64)> = match Vec::<(String, u64, u64)>::from_value(value) { + Ok(raw) => raw, + Err(e) => { + warn!("S11: could not read a wakeup log: {e}"); + return Vec::new(); + } + }; + raw.into_iter() + .map(|(token, armed_millis, woken_millis)| WakeupRecord { + token, + armed_at: from_millis(armed_millis), + woken_at: from_millis(woken_millis), + }) + .collect() +} + +/// Epoch milliseconds as the agent stamped them. +/// +/// A zero means the agent had no armed time for the token, which the component +/// only produces if its own arm log rolled over. It is kept as the epoch rather +/// than dropped so the resulting nonsense interval is visible instead of the +/// wakeup silently going missing. +fn from_millis(millis: u64) -> DateTime { + Utc.timestamp_millis_opt(millis as i64) + .single() + .unwrap_or_else(|| Utc.timestamp_nanos(0)) +} + +#[cfg(test)] +mod tests { + use super::*; + use test_r::test; + + #[test] + fn a_stall_timeout_scales_with_the_wakeup_budget() { + let config = PromiseConfig { + waiters: 10, + dwell_millis: 5000, + wakeup_budget_secs: 120, + }; + assert_eq!( + (config.wakeup_budget() * STALL_MULTIPLE).max(MIN_STALL_TIMEOUT), + Duration::from_secs(480) + ); + } + + /// A budget short enough that four of it would call an ordinary recovery a + /// stall still gets the floor. + #[test] + fn a_short_wakeup_budget_still_gets_the_floor() { + let config = PromiseConfig { + waiters: 10, + dwell_millis: 5000, + wakeup_budget_secs: 10, + }; + assert_eq!( + (config.wakeup_budget() * STALL_MULTIPLE).max(MIN_STALL_TIMEOUT), + MIN_STALL_TIMEOUT + ); + } + + #[test] + fn an_agent_timestamp_of_zero_stays_visible_as_the_epoch() { + assert_eq!(from_millis(0).timestamp_millis(), 0); + } +} diff --git a/integration-tests/src/chaos/wakeups.rs b/integration-tests/src/chaos/wakeups.rs new file mode 100644 index 0000000000..212e4fea25 --- /dev/null +++ b/integration-tests/src/chaos/wakeups.rs @@ -0,0 +1,963 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Pairing completions against wakeups (GOL-377). +//! +//! S11's question is not "how many promises resolved". It is, for each +//! completion the platform *accepted*, whether the agent suspended on that +//! promise was resumed — once, and only once. Counting cannot answer it: a lost +//! wakeup and a duplicate wakeup cancel out in a total, and neither can be +//! localised to an agent afterwards. +//! +//! So every round carries a token. The driver mints it, `arm` records it against +//! the promise it created, and the waiter writes it into its wakeup log when it +//! is resumed. Pairing the two turns each failure into a statement about one +//! named round: this completion, accepted at this time, against this waiter, +//! never woke it. +//! +//! ## What counts as a finding, and what only counts as doubt +//! +//! A completion the platform confirmed is a promise the platform made. If the +//! waiter never woke and its log is whole, that is a finding, full stop. +//! +//! A completion that failed in an *indeterminate* way is not. From the client +//! side a dropped connection is indistinguishable from a request that arrived +//! and executed, and a pod kill produces exactly these. If such a round woke, +//! the platform resolved the doubt in its own favour and the report says so; if +//! it did not, nothing is proven and the round is counted as inconclusive rather +//! than as a loss. +//! +//! ## The waiter that will not answer +//! +//! One case here is unlike anything the other scenarios see. A waiter is parked +//! *inside* an invocation, so a waiter that never wakes cannot answer a read +//! either — its `wakeups` read queues behind the `wait` that is still running. +//! +//! An unreadable agent is normally the weakest possible outcome: it means the +//! run cannot say. Here it is nearly the opposite. A waiter whose completion was +//! confirmed, which then stopped producing rounds *and* could not be read, is a +//! worker wedged on a promise that was resolved long ago. The report separates +//! that case ([`WakeupViolation::NeverWoke`], with the read failure as its +//! detail) from an agent that merely timed out while otherwise healthy. +//! +//! ## Two clocks, and which number to believe +//! +//! The waiter stamps `armedAt` and `wokenAt` from the executor's clock; the +//! driver stamps the completion from its own. So the headline delay — +//! completion accepted to waiter resumed — spans two clocks, exactly as the +//! scheduled-fire delay does in [`crate::chaos::fires`], and the same guard +//! applies: `minDelayMs` is reported per cell so skew shows up as a negative +//! number instead of quietly flattering a percentile. +//! +//! There is one cross-check S10 cannot make. `parkedMs` — armed to woken — is +//! stamped at both ends by the executor, so it carries no skew at all. It is not +//! the delay, because it also contains the round's deliberate dwell, but on a +//! healthy baseline `parked - dwell` and the cross-clock delay should agree. A +//! gap between them is skew, and the report carries both rather than picking. + +use crate::chaos::history::{OperationRecord, Outcome, Stream, WaiterWakeupLog, WakeupRecord}; +use crate::chaos::split::{FaultWindow, PodSplit, Window}; +use crate::chaos::summary::LatencyStats; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; +use std::time::Duration; + +/// Ceiling on how many findings the report carries. +/// +/// A run that lost every wakeup would otherwise produce tens of thousands of +/// them and an artifact nobody can open. The count is reported separately, so +/// truncation is stated rather than inferred from a suspiciously round number. +const MAX_FINDINGS: usize = 200; + +/// The method name the completion operations are recorded under. +const COMPLETE_METHOD: &str = "complete"; + +/// Whether a waiter was on the executor the fault killed. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum WaiterGroup { + /// Owned by the killed executor when the driver signalled readiness. + OnKilledExecutor, + /// Owned by an executor the fault left alone: the run's own control group. + Elsewhere, +} + +impl WaiterGroup { + pub fn as_str(self) -> &'static str { + match self { + WaiterGroup::OnKilledExecutor => "on-killed-executor", + WaiterGroup::Elsewhere => "elsewhere", + } + } +} + +impl std::fmt::Display for WaiterGroup { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// What went wrong with one round. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum WakeupViolation { + /// A completion the platform accepted whose waiter was never resumed. + NeverWoke, + /// One completion, two or more wakeups. A promise resolved twice, or a + /// recovery that replayed the resume without deduplicating it. + WokeMoreThanOnce, + /// A wakeup for a round whose completion the platform definitively refused. + WokeDespiteRejection, +} + +impl WakeupViolation { + pub fn as_str(self) -> &'static str { + match self { + WakeupViolation::NeverWoke => "never-woke", + WakeupViolation::WokeMoreThanOnce => "woke-more-than-once", + WakeupViolation::WokeDespiteRejection => "woke-despite-rejection", + } + } +} + +impl std::fmt::Display for WakeupViolation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// One violation, against one round. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct WakeupFinding { + pub violation: WakeupViolation, + pub token: String, + pub agent: String, + pub window: Window, + pub detail: String, +} + +/// Wakeup delay for one (group, window) cell. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct WakeupDelayStats { + pub group: WaiterGroup, + pub window: Window, + /// Percentiles over delays clamped at zero, so skew cannot flatter them. + pub delay: LatencyStats, + /// The most negative delay seen, which is the clock skew between the driver + /// and the executor rather than a waiter woken before it was completed. + pub min_delay_ms: i64, + /// Wakeups whose delay exceeded the configured budget. + pub over_budget: u64, + /// Armed-to-woken, on the executor's clock alone. Carries the round's dwell + /// as well as the delay, and carries no skew. + pub parked: LatencyStats, +} + +/// The promise-wakeup account. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct WakeupReport { + /// What resuming a suspended waiter is allowed to cost, from the suite YAML. + /// Recorded so a percentile in an archived result can be read years later + /// against the number it was judged by rather than against today's config. + pub wakeup_budget_ms: u64, + /// The dwell each round held before its completion, so `parked` can be read + /// without the suite YAML to hand. + pub dwell_ms: u64, + pub completions_confirmed: u64, + pub completions_indeterminate: u64, + pub completions_rejected: u64, + /// Wakeups the waiters recorded, including any whose token is unknown. + pub wakeups_recorded: u64, + /// Accepted completions paired with exactly one wakeup. + pub woke_once: u64, + /// Completions the driver was never sure of that woke anyway — doubt the + /// platform resolved in its own favour. + pub indeterminate_that_woke: u64, + /// Completions the driver was never sure of that never woke. Not a finding: + /// the completion may never have landed. + pub inconclusive: u64, + /// Completions whose waiter could not testify, because its log was + /// unreadable or truncated *and* nothing else about it looked wrong. + pub unverifiable: u64, + /// Wakeups whose token no completion claims. Zero on a healthy run: agent + /// names carry the run nonce, so nothing from an earlier run can appear. + pub unknown_tokens: u64, + /// Waiters the read-back could not reach at all. + pub waiters_unreadable: Vec, + /// Waiters whose wakeup log hit the component's cap. + pub waiters_truncated: Vec, + /// Waiters that stopped producing rounds during the run because a wakeup + /// never arrived, as the workload itself observed. The live half of the + /// oracle — see [`crate::chaos::waiters::WaiterHandle::stalled`]. + pub waiters_stood_down: u64, + /// Waiters that stood down *and* could not then be read: parked inside an + /// invocation, which is what a wedged worker looks like from outside. + pub waiters_wedged: Vec, + pub delay: Vec, + pub findings: Vec, + /// Findings past [`MAX_FINDINGS`], which the report drops rather than + /// carries. Non-zero means `findings` is a sample. + pub findings_omitted: u64, +} + +/// One round, as the pairing sees it. +struct Round<'a> { + token: &'a str, + agent: &'a str, + outcome: Outcome, + submitted_at: DateTime, +} + +impl WakeupReport { + /// Pairs completions against wakeups. + /// + /// `records` is the whole history; only the `complete` operations of the + /// waiter stream are considered — `arm` and `wait` are recorded for the + /// timeline, not for this account. + #[allow(clippy::too_many_arguments)] + pub fn build( + records: &[OperationRecord], + logs: &[WaiterWakeupLog], + split: &PodSplit, + fault: Option, + dwell: Duration, + wakeup_budget: Duration, + stood_down: u64, + ) -> Self { + let budget_ms = wakeup_budget.as_millis().min(u64::MAX as u128) as u64; + let dwell_ms = dwell.as_millis().min(u64::MAX as u128) as u64; + + let mut wakeups_by_token: BTreeMap<&str, Vec<&WakeupRecord>> = BTreeMap::new(); + let mut complete_log: BTreeMap<&str, bool> = BTreeMap::new(); + let mut waiters_unreadable = Vec::new(); + let mut waiters_truncated = Vec::new(); + let mut wakeups_recorded = 0u64; + + for log in logs { + complete_log.insert(log.agent.as_str(), log.is_complete()); + if log.error.is_some() { + waiters_unreadable.push(log.agent.clone()); + } else if !log.is_complete() { + waiters_truncated.push(log.agent.clone()); + } + for wakeup in &log.wakeups { + wakeups_recorded += 1; + wakeups_by_token + .entry(wakeup.token.as_str()) + .or_default() + .push(wakeup); + } + } + + let on_killed: BTreeSet<&str> = split.on_pod.iter().map(|s| s.as_str()).collect(); + let mut report = Self::empty(budget_ms, dwell_ms); + report.wakeups_recorded = wakeups_recorded; + report.waiters_stood_down = stood_down; + + // A stalled waiter that then could not be read is the wedged case. It is + // computed before the round loop because the loop uses it to decide + // whether an unreadable waiter excuses a missing wakeup or convicts it. + let unreadable: BTreeSet<&str> = waiters_unreadable.iter().map(|s| s.as_str()).collect(); + + let mut cells: BTreeMap<(WaiterGroup, Window), DelayCell> = BTreeMap::new(); + let mut claimed: BTreeSet<&str> = BTreeSet::new(); + + for round in completions(records) { + let group = if on_killed.contains(round.agent) { + WaiterGroup::OnKilledExecutor + } else { + WaiterGroup::Elsewhere + }; + let window = Window::of(round.submitted_at, fault); + let wakeups = wakeups_by_token + .get(round.token) + .cloned() + .unwrap_or_default(); + claimed.insert(round.token); + + match round.outcome { + Outcome::Confirmed => report.completions_confirmed += 1, + Outcome::Indeterminate => report.completions_indeterminate += 1, + Outcome::Rejected => report.completions_rejected += 1, + } + + if wakeups.len() > 1 { + report.push_finding(WakeupFinding { + violation: WakeupViolation::WokeMoreThanOnce, + token: round.token.to_string(), + agent: round.agent.to_string(), + window, + detail: format!( + "{} wakeups recorded for one completion, at {}", + wakeups.len(), + wakeups + .iter() + .map(|w| w.woken_at.to_rfc3339()) + .collect::>() + .join(", ") + ), + }); + } + + if let Some(wakeup) = wakeups.first() { + if round.outcome == Outcome::Rejected { + report.push_finding(WakeupFinding { + violation: WakeupViolation::WokeDespiteRejection, + token: round.token.to_string(), + agent: round.agent.to_string(), + window, + detail: format!( + "the completion was definitively refused, and the waiter woke at {}", + wakeup.woken_at.to_rfc3339() + ), + }); + } else { + if round.outcome == Outcome::Confirmed { + report.woke_once += 1; + } else { + report.indeterminate_that_woke += 1; + } + let delay_ms = (wakeup.woken_at - round.submitted_at).num_milliseconds(); + cells.entry((group, window)).or_default().push( + delay_ms, + wakeup.parked_ms(), + budget_ms, + ); + } + continue; + } + + // No wakeup. What that proves depends on the completion's outcome + // and on whether the waiter could testify at all. + match round.outcome { + Outcome::Rejected => {} + Outcome::Indeterminate => report.inconclusive += 1, + Outcome::Confirmed => { + let log_whole = complete_log.get(round.agent).copied().unwrap_or(false); + if log_whole { + report.push_finding(WakeupFinding { + violation: WakeupViolation::NeverWoke, + token: round.token.to_string(), + agent: round.agent.to_string(), + window, + detail: format!( + "the completion was accepted at {} and the waiter's whole wakeup \ + log has no entry for it", + round.submitted_at.to_rfc3339() + ), + }); + } else if unreadable.contains(round.agent) && stood_down > 0 { + // The wedged case: the workload watched this waiter stop + // producing, and the read-back then could not reach it. + // Both symptoms of one worker still parked on a promise + // that was resolved. + report.push_finding(WakeupFinding { + violation: WakeupViolation::NeverWoke, + token: round.token.to_string(), + agent: round.agent.to_string(), + window, + detail: format!( + "the completion was accepted at {} and the waiter has answered \ + nothing since, which is what a worker parked on a resolved \ + promise looks like from outside", + round.submitted_at.to_rfc3339() + ), + }); + if !report.waiters_wedged.iter().any(|w| w == round.agent) { + report.waiters_wedged.push(round.agent.to_string()); + } + } else { + report.unverifiable += 1; + } + } + } + } + + report.unknown_tokens = wakeups_by_token + .keys() + .filter(|token| !claimed.contains(*token)) + .count() as u64; + report.waiters_unreadable = waiters_unreadable; + report.waiters_truncated = waiters_truncated; + report.delay = cells + .into_iter() + .map(|((group, window), cell)| cell.into_stats(group, window)) + .collect(); + report + } + + fn empty(wakeup_budget_ms: u64, dwell_ms: u64) -> Self { + Self { + wakeup_budget_ms, + dwell_ms, + completions_confirmed: 0, + completions_indeterminate: 0, + completions_rejected: 0, + wakeups_recorded: 0, + woke_once: 0, + indeterminate_that_woke: 0, + inconclusive: 0, + unverifiable: 0, + unknown_tokens: 0, + waiters_unreadable: Vec::new(), + waiters_truncated: Vec::new(), + waiters_stood_down: 0, + waiters_wedged: Vec::new(), + delay: Vec::new(), + findings: Vec::new(), + findings_omitted: 0, + } + } + + fn push_finding(&mut self, finding: WakeupFinding) { + if self.findings.len() < MAX_FINDINGS { + self.findings.push(finding); + } else { + self.findings_omitted += 1; + } + } + + /// Whether the run found anything that fails it. + pub fn has_violations(&self) -> bool { + !self.findings.is_empty() || self.findings_omitted > 0 + } + + /// Total findings, including any the report dropped. + pub fn violations(&self) -> u64 { + self.findings.len() as u64 + self.findings_omitted + } + + /// The p99 wakeup delay on the killed executor's waiters, during the fault: + /// the one number this scenario exists to produce. + pub fn fault_window_p99_ms(&self) -> Option { + self.delay + .iter() + .find(|cell| { + cell.group == WaiterGroup::OnKilledExecutor && cell.window == Window::DuringFault + }) + .map(|cell| cell.delay.p99_ms) + } + + /// Lines an operator has to act on. + pub fn attention_lines(&self) -> Vec { + let mut lines = Vec::new(); + if self.has_violations() { + lines.push(format!( + "S11 found {} promise-wakeup violations: {}", + self.violations(), + self.findings + .iter() + .take(3) + .map(|f| format!("{} on {}", f.violation, f.token)) + .collect::>() + .join(", ") + )); + } + if !self.waiters_wedged.is_empty() { + lines.push(format!( + "S11 left {} waiters wedged: {} — each stopped producing rounds and then \ + answered no read, which is a worker still parked on a resolved promise", + self.waiters_wedged.len(), + self.waiters_wedged.join(", ") + )); + } + if !self.waiters_unreadable.is_empty() || !self.waiters_truncated.is_empty() { + lines.push(format!( + "S11 could not take a whole account from {} waiters ({} unreadable, {} \ + truncated), so {} accepted completions are unverified either way", + self.waiters_unreadable.len() + self.waiters_truncated.len(), + self.waiters_unreadable.len(), + self.waiters_truncated.len(), + self.unverifiable + )); + } + if self.unknown_tokens > 0 { + lines.push(format!( + "S11 recorded {} wakeups whose token no completion claims — agent names carry \ + the run nonce, so this should be impossible", + self.unknown_tokens + )); + } + lines + } + + /// Lines that explain the account without claiming anything is wrong. + pub fn note_lines(&self) -> Vec { + let mut lines = Vec::new(); + if self.waiters_stood_down > 0 && self.waiters_wedged.is_empty() { + lines.push(format!( + "S11 stood {} waiters down after a slow wakeup, and every one of them was \ + readable afterwards — late rather than lost", + self.waiters_stood_down + )); + } + if self.indeterminate_that_woke > 0 { + lines.push(format!( + "S11 had {} completions fail in a way that proves nothing, whose waiters woke \ + anyway", + self.indeterminate_that_woke + )); + } + if self.inconclusive > 0 { + lines.push(format!( + "S11 had {} completions that neither succeeded nor demonstrably landed, whose \ + waiters did not wake — not losses, because the completion may never have \ + arrived", + self.inconclusive + )); + } + lines + } +} + +/// Delays accumulated for one cell before they become percentiles. +#[derive(Default)] +struct DelayCell { + delays: Vec, + parked: Vec, + min_delay_ms: i64, + over_budget: u64, + any: bool, +} + +impl DelayCell { + fn push(&mut self, delay_ms: i64, parked_ms: i64, budget_ms: u64) { + if !self.any || delay_ms < self.min_delay_ms { + self.min_delay_ms = delay_ms; + } + self.any = true; + let clamped = delay_ms.max(0) as u64; + if clamped > budget_ms { + self.over_budget += 1; + } + self.delays.push(clamped); + self.parked.push(parked_ms.max(0) as u64); + } + + fn into_stats(self, group: WaiterGroup, window: Window) -> WakeupDelayStats { + WakeupDelayStats { + group, + window, + delay: LatencyStats::from_durations(self.delays), + min_delay_ms: self.min_delay_ms, + over_budget: self.over_budget, + parked: LatencyStats::from_durations(self.parked), + } + } +} + +/// The completion operations, which are the rounds this report is about. +fn completions(records: &[OperationRecord]) -> impl Iterator> { + records + .iter() + .filter(|r| r.stream == Stream::PromiseWait && r.method == COMPLETE_METHOD) + .map(|r| Round { + token: r.idempotency_key.as_str(), + agent: r.agent.as_str(), + outcome: r.outcome, + submitted_at: r.submitted_at, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::chaos::history::Phase; + use test_r::test; + + fn at(secs: i64) -> DateTime { + DateTime::from_timestamp(secs, 0).unwrap() + } + + fn at_ms(millis: i64) -> DateTime { + DateTime::from_timestamp_millis(millis).unwrap() + } + + /// The fault: injected at 100s, cleared at 200s. + fn fault() -> Option { + Some(FaultWindow { + injected_at: at(100), + recovered_at: Some(at(200)), + }) + } + + fn split_of(on_pod: &[&str], elsewhere: &[&str]) -> PodSplit { + PodSplit { + pod_address: "10.0.1.1:9000".to_string(), + pod_ip: "10.0.1.1".to_string(), + on_pod: on_pod.iter().map(|s| s.to_string()).collect(), + elsewhere: elsewhere.iter().map(|s| s.to_string()).collect(), + targets_per_pod: BTreeMap::new(), + number_of_shards: 1024, + } + } + + fn completion( + agent: &str, + token: &str, + submitted: DateTime, + outcome: Outcome, + ) -> OperationRecord { + OperationRecord { + op_id: 1, + stream: Stream::PromiseWait, + phase: Phase::Fault, + agent: agent.to_string(), + method: COMPLETE_METHOD.to_string(), + idempotency_key: token.to_string(), + submitted_at: submitted, + completed_at: Some(submitted), + attempts: 1, + outcome, + duration_ms: 0, + returned_value: None, + first_attempt_value: None, + error: None, + error_class: None, + attempt_log: Vec::new(), + } + } + + /// A whole log: `wakes` matches the entries, so an absent wakeup is a + /// statement rather than a gap. + fn log(agent: &str, wakeups: Vec) -> WaiterWakeupLog { + WaiterWakeupLog { + agent: agent.to_string(), + wakes: Some(wakeups.len() as u64), + wakeups, + error: None, + } + } + + fn wakeup(token: &str, armed: DateTime, woken: DateTime) -> WakeupRecord { + WakeupRecord { + token: token.to_string(), + armed_at: armed, + woken_at: woken, + } + } + + fn build( + records: &[OperationRecord], + logs: &[WaiterWakeupLog], + split: &PodSplit, + stood_down: u64, + ) -> WakeupReport { + WakeupReport::build( + records, + logs, + split, + fault(), + Duration::from_secs(5), + Duration::from_secs(60), + stood_down, + ) + } + + #[test] + fn a_completion_paired_with_one_wakeup_is_clean() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Confirmed)]; + let logs = vec![log("w-1", vec![wakeup("t-1", at(105), at(111))])]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert_eq!(report.completions_confirmed, 1); + assert_eq!(report.woke_once, 1); + assert!(!report.has_violations()); + } + + /// The headline failure. An accepted completion is a promise the platform + /// made; a whole log with no entry for it is the platform not keeping it. + #[test] + fn an_accepted_completion_that_never_woke_its_waiter_is_a_finding() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Confirmed)]; + let logs = vec![log("w-1", Vec::new())]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert_eq!(report.findings.len(), 1); + assert_eq!(report.findings[0].violation, WakeupViolation::NeverWoke); + assert_eq!(report.findings[0].token, "t-1"); + assert_eq!(report.findings[0].window, Window::DuringFault); + } + + /// A truncated log cannot testify, so the same missing wakeup proves + /// nothing. Calling it a loss would turn a component-side cap into a + /// platform defect. + #[test] + fn a_missing_wakeup_on_a_truncated_log_is_unverifiable_rather_than_lost() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Confirmed)]; + let logs = vec![WaiterWakeupLog { + agent: "w-1".to_string(), + wakes: Some(9000), + wakeups: Vec::new(), + error: None, + }]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert!(!report.has_violations()); + assert_eq!(report.unverifiable, 1); + assert_eq!(report.waiters_truncated, vec!["w-1".to_string()]); + } + + /// The wedged case, and the one place where an unreadable agent convicts + /// rather than excuses: the workload watched this waiter stop producing, and + /// the read-back then could not reach it. Both are symptoms of one worker + /// still parked on a promise that was resolved. + #[test] + fn a_waiter_that_stood_down_and_then_answered_nothing_is_a_finding() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Confirmed)]; + let logs = vec![WaiterWakeupLog { + agent: "w-1".to_string(), + wakes: None, + wakeups: Vec::new(), + error: Some("wakeups timed out after 60s".to_string()), + }]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 1); + + assert_eq!(report.findings.len(), 1); + assert_eq!(report.findings[0].violation, WakeupViolation::NeverWoke); + assert_eq!(report.waiters_wedged, vec!["w-1".to_string()]); + assert_eq!(report.unverifiable, 0); + } + + /// The same unreadable waiter, with nothing having stood down, is only an + /// unreadable waiter. A read that timed out on an otherwise healthy run says + /// nothing about whether the completion landed. + #[test] + fn an_unreadable_waiter_that_never_stood_down_is_unverifiable() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Confirmed)]; + let logs = vec![WaiterWakeupLog { + agent: "w-1".to_string(), + wakes: None, + wakeups: Vec::new(), + error: Some("wakeups timed out after 60s".to_string()), + }]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert!(!report.has_violations()); + assert_eq!(report.unverifiable, 1); + assert!(report.waiters_wedged.is_empty()); + } + + #[test] + fn one_completion_and_two_wakeups_is_a_finding() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Confirmed)]; + let logs = vec![log( + "w-1", + vec![ + wakeup("t-1", at(105), at(111)), + wakeup("t-1", at(105), at(160)), + ], + )]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert_eq!( + report.findings[0].violation, + WakeupViolation::WokeMoreThanOnce + ); + } + + #[test] + fn a_wakeup_for_a_refused_completion_is_a_finding() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Rejected)]; + let logs = vec![log("w-1", vec![wakeup("t-1", at(105), at(111))])]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert_eq!( + report.findings[0].violation, + WakeupViolation::WokeDespiteRejection + ); + assert_eq!(report.completions_rejected, 1); + } + + /// A pod kill produces indeterminate completions by the dozen. One that woke + /// anyway is the platform resolving the doubt in its own favour, and the + /// report says so rather than counting it beside the confirmed ones. + #[test] + fn an_indeterminate_completion_that_woke_is_recorded_separately() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Indeterminate)]; + let logs = vec![log("w-1", vec![wakeup("t-1", at(105), at(111))])]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert_eq!(report.indeterminate_that_woke, 1); + assert_eq!(report.woke_once, 0); + assert!(!report.has_violations()); + } + + /// And one that did not wake proves nothing at all: from the client side a + /// dropped connection is indistinguishable from a request that arrived. + #[test] + fn an_indeterminate_completion_that_never_woke_is_inconclusive_rather_than_lost() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Indeterminate)]; + let logs = vec![log("w-1", Vec::new())]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert_eq!(report.inconclusive, 1); + assert!(!report.has_violations()); + } + + /// The control group is the whole reason the kill is aimed. Mixing the two + /// would let a recovery that took its full budget hide behind the waiters + /// that were never touched. + #[test] + fn delays_are_split_by_group_and_by_window() { + let records = vec![ + completion("w-1", "t-1", at(110), Outcome::Confirmed), + completion("w-2", "t-2", at(110), Outcome::Confirmed), + completion("w-1", "t-3", at(50), Outcome::Confirmed), + ]; + let logs = vec![ + log( + "w-1", + vec![ + wakeup("t-1", at(105), at(150)), + wakeup("t-3", at(45), at(51)), + ], + ), + log("w-2", vec![wakeup("t-2", at(105), at(111))]), + ]; + let report = build(&records, &logs, &split_of(&["w-1"], &["w-2"]), 0); + + let killed_during = report + .delay + .iter() + .find(|c| c.group == WaiterGroup::OnKilledExecutor && c.window == Window::DuringFault) + .unwrap(); + assert_eq!(killed_during.delay.max_ms, 40_000); + + let control_during = report + .delay + .iter() + .find(|c| c.group == WaiterGroup::Elsewhere && c.window == Window::DuringFault) + .unwrap(); + assert_eq!(control_during.delay.max_ms, 1_000); + + assert!(report.delay.iter().any(|c| c.window == Window::BeforeFault)); + assert_eq!(report.fault_window_p99_ms(), Some(40_000)); + } + + /// The driver and the executor keep different clocks, so a wakeup can look + /// as though it happened before the completion that caused it. That is skew, + /// and it has to be visible rather than clamped silently into a percentile. + #[test] + fn clock_skew_shows_as_a_negative_minimum_rather_than_flattering_the_percentiles() { + let records = vec![completion("w-1", "t-1", at_ms(110_000), Outcome::Confirmed)]; + let logs = vec![log( + "w-1", + vec![wakeup("t-1", at_ms(105_000), at_ms(109_700))], + )]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + let cell = &report.delay[0]; + assert_eq!(cell.min_delay_ms, -300); + assert_eq!(cell.delay.max_ms, 0); + } + + /// `parked` is stamped at both ends by the executor, so it is the one number + /// in the report free of that skew — which is what makes it worth carrying + /// beside the delay rather than instead of it. + #[test] + fn the_parked_interval_is_reported_on_the_executors_own_clock() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Confirmed)]; + let logs = vec![log("w-1", vec![wakeup("t-1", at(105), at(150))])]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert_eq!(report.delay[0].parked.max_ms, 45_000); + assert_eq!(report.dwell_ms, 5_000); + } + + #[test] + fn a_delay_past_the_budget_is_counted_without_failing_the_run() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Confirmed)]; + let logs = vec![log("w-1", vec![wakeup("t-1", at(105), at(180))])]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert_eq!(report.delay[0].over_budget, 1); + assert!(!report.has_violations()); + } + + /// Agent names carry the run nonce, so a wakeup nobody asked for should be + /// impossible. If it happens the report says so instead of dropping it. + #[test] + fn a_wakeup_no_completion_claims_is_counted() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Confirmed)]; + let logs = vec![log( + "w-1", + vec![ + wakeup("t-1", at(105), at(111)), + wakeup("t-stray", at(105), at(112)), + ], + )]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert_eq!(report.unknown_tokens, 1); + assert_eq!(report.wakeups_recorded, 2); + } + + /// `arm` and `wait` are recorded for the timeline. Counting them as rounds + /// would treble every total in the report. + #[test] + fn only_the_completion_operations_are_counted_as_rounds() { + let mut armed = completion("w-1", "t-1-arm", at(109), Outcome::Confirmed); + armed.method = "arm".to_string(); + let mut waited = completion("w-1", "t-1-wait", at(109), Outcome::Confirmed); + waited.method = "wait".to_string(); + let records = vec![ + armed, + waited, + completion("w-1", "t-1", at(110), Outcome::Confirmed), + ]; + let logs = vec![log("w-1", vec![wakeup("t-1", at(105), at(111))])]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert_eq!(report.completions_confirmed, 1); + assert_eq!(report.woke_once, 1); + } + + /// A run that lost everything must still produce an artifact somebody can + /// open, and must say that it truncated rather than leaving a suspiciously + /// round number of findings. + #[test] + fn findings_beyond_the_cap_are_counted_rather_than_carried() { + let records: Vec = (0..MAX_FINDINGS + 10) + .map(|i| completion("w-1", &format!("t-{i}"), at(110), Outcome::Confirmed)) + .collect(); + let logs = vec![log("w-1", Vec::new())]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert_eq!(report.findings.len(), MAX_FINDINGS); + assert_eq!(report.findings_omitted, 10); + assert_eq!(report.violations(), MAX_FINDINGS as u64 + 10); + } + + /// Standing waiters down is normal on a run where recovery was slow. It only + /// becomes an attention line when those waiters then could not be read. + #[test] + fn waiters_that_stood_down_but_answered_afterwards_are_context_not_a_finding() { + let records = vec![completion("w-1", "t-1", at(110), Outcome::Confirmed)]; + let logs = vec![log("w-1", vec![wakeup("t-1", at(105), at(190))])]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 3); + + assert!(report.attention_lines().is_empty()); + assert!( + report + .note_lines() + .iter() + .any(|line| line.contains("late rather than lost")) + ); + } +} diff --git a/integration-tests/src/chaos/workload.rs b/integration-tests/src/chaos/workload.rs index dfce97f949..6c704a8f7c 100644 --- a/integration-tests/src/chaos/workload.rs +++ b/integration-tests/src/chaos/workload.rs @@ -181,6 +181,12 @@ impl WorkloadContext { format!("{}-scheduled-target-{index:04}", self.key_prefix) } + /// The suspended-waiter scenario's agent, in the promise component rather + /// than the counters one (GOL-377). + pub fn waiter_name(&self, index: u32) -> String { + format!("{}-promise-waiter-{index:04}", self.key_prefix) + } + /// The deterministic key for one operation. Same inputs, same key — that is /// what makes a retry the same operation rather than a new one. pub fn idempotency_key(&self, agent: &str, seq: u64) -> String { @@ -485,6 +491,12 @@ pub(crate) async fn submit_one(ctx: &WorkloadContext, stream: Stream, index: u32 Stream::PinnedHttp => { warn!("Chaos mixed workload cannot drive the pinned stream; see chaos::pinned"); } + // Driven by `crate::chaos::waiters` for the same reason: its agents run + // a round at their own pace rather than at a shared rate, and each one + // is blocked until its promise resolves. + Stream::PromiseWait => { + warn!("Chaos mixed workload cannot drive the waiter stream; see chaos::waiters"); + } Stream::Durable => { let agent = ctx.agent_name(Stream::Durable, index); let key = ctx.idempotency_key(&agent, seq); diff --git a/test-components/agent-rpc/golem-it-promise-agent-rust/src/lib.rs b/test-components/agent-rpc/golem-it-promise-agent-rust/src/lib.rs index 6767bfd62e..0edc3e3a5d 100644 --- a/test-components/agent-rpc/golem-it-promise-agent-rust/src/lib.rs +++ b/test-components/agent-rpc/golem-it-promise-agent-rust/src/lib.rs @@ -30,3 +30,114 @@ impl PromiseAgent for PromiseAgentImpl { golem_rust::complete_promise(&promise_id, &vec![0; payload_size as usize]) } } + +/// Ceiling on the wakeup log below. +/// +/// Far above what a chaos run produces — a few hundred wakeups per waiter — so +/// it only stops a misconfigured cadence from growing agent state without +/// bound. `wakes` keeps counting past it, which is what makes truncation +/// visible: a reader that gets fewer entries than wakes knows the log is short +/// rather than the wakeups missing. +const MAX_WAKEUP_LOG: usize = 10_000; + +/// Wall clock in milliseconds since the epoch. +/// +/// Read inside the agent rather than passed in, because the question S11 asks is +/// when the *platform* resumed the waiter. The read is durable, so an agent that +/// replays reports the original wakeup time instead of the replay's. +fn now_millis() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|since| since.as_millis() as u64) + .unwrap_or(0) +} + +/// A durable agent that parks on a promise and records being woken (GOL-377). +/// +/// [`PromiseAgent`] above creates and completes promises in one breath, which is +/// what the mixed workload's promise stream needs and says nothing about +/// recovery. S11 needs the other half: an agent genuinely suspended on a promise +/// whose executor is then killed, and a durable record of whether the completion +/// ever reached it. +/// +/// The record has to live in agent state rather than in the invocation's return +/// value, because the interesting case is exactly the one where the caller's +/// connection died with the executor. A caller that got an error learns nothing; +/// the log outlives the connection and answers anyway. +#[agent_definition] +pub trait PromiseWaiter { + fn new(name: String) -> Self; + + /// Creates a promise for this waiter and returns it without waiting. + /// + /// Separate from [`PromiseWaiter::wait`] because the completer needs the + /// promise id, and an invocation that blocks cannot return one. `token` is + /// the driver's idempotency key for this round, and is what pairs a wakeup + /// back to the completion that caused it. + fn arm(&mut self, token: String) -> PromiseId; + + /// Blocks until `promise_id` is completed, then records the wakeup. + /// + /// Suspends the agent, so the executor holds no thread for it: this is the + /// state S11 kills an executor in. + fn wait(&mut self, token: String, promise_id: PromiseId); + + /// The wakeup log: one `(token, armed_millis, woken_millis)` per wakeup, in + /// the order the waiter was resumed. + fn wakeups(&self) -> Vec<(String, u64, u64)>; + + /// How many wakeups happened, whether or not the log kept them. + fn wakes(&self) -> u32; +} + +struct PromiseWaiterImpl { + _name: String, + /// When each token was armed, so a wakeup can report the interval it was + /// parked without the driver having to join two clocks. + armed: Vec<(String, u64)>, + wakeups: Vec<(String, u64, u64)>, + wakes: u32, +} + +#[agent_implementation] +impl PromiseWaiter for PromiseWaiterImpl { + fn new(name: String) -> Self { + Self { + _name: name, + armed: Vec::new(), + wakeups: Vec::new(), + wakes: 0, + } + } + + fn arm(&mut self, token: String) -> PromiseId { + if self.armed.len() >= MAX_WAKEUP_LOG { + self.armed.remove(0); + } + self.armed.push((token, now_millis())); + golem_rust::create_promise() + } + + fn wait(&mut self, token: String, promise_id: PromiseId) { + let _ = golem_rust::blocking_await_promise(&promise_id); + self.wakes += 1; + if self.wakeups.len() < MAX_WAKEUP_LOG { + let armed_millis = self + .armed + .iter() + .rev() + .find(|(armed_token, _)| armed_token == &token) + .map(|(_, at)| *at) + .unwrap_or(0); + self.wakeups.push((token, armed_millis, now_millis())); + } + } + + fn wakeups(&self) -> Vec<(String, u64, u64)> { + self.wakeups.clone() + } + + fn wakes(&self) -> u32 { + self.wakes + } +} From b1591426d8f56dd9571fb61f58cc8be21465bb45 Mon Sep 17 00:00:00 2001 From: kmatasfp <33095685+kmatas@users.noreply.github.com> Date: Thu, 20 Aug 2026 16:04:52 -0700 Subject: [PATCH 08/40] Note why a stalled waiter's invocation is left running --- integration-tests/src/chaos/waiters.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/integration-tests/src/chaos/waiters.rs b/integration-tests/src/chaos/waiters.rs index 68a569811b..d0c0f44742 100644 --- a/integration-tests/src/chaos/waiters.rs +++ b/integration-tests/src/chaos/waiters.rs @@ -248,6 +248,10 @@ async fn run_waiter( tokio::time::sleep(dwell).await; complete(&ctx, &waiter, &token, &promise_id).await; + // The handle is dropped rather than aborted on the timeout path, which + // leaves the invocation running. That is deliberate: aborting it would + // throw away the one record that says how the round ended, and the agent + // is parked either way — the driver's task is not what is holding it. match tokio::time::timeout(stall_timeout, wait_task).await { Ok(_) => {} Err(_) => { From dc6d848dbee4ca6cd874384aa6fcd520f7291408 Mon Sep 17 00:00:00 2001 From: kmatasfp <33095685+kmatas@users.noreply.github.com> Date: Thu, 20 Aug 2026 16:14:29 -0700 Subject: [PATCH 09/40] Pin the promise-id encoding used to hand a promise back to an agent --- integration-tests/src/chaos/waiters.rs | 30 ++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/integration-tests/src/chaos/waiters.rs b/integration-tests/src/chaos/waiters.rs index d0c0f44742..ab8f22b171 100644 --- a/integration-tests/src/chaos/waiters.rs +++ b/integration-tests/src/chaos/waiters.rs @@ -550,4 +550,34 @@ mod tests { fn an_agent_timestamp_of_zero_stays_visible_as_the_epoch() { assert_eq!(from_millis(0).timestamp_millis(), 0); } + + /// A promise id has to survive the trip back *into* an agent. + /// + /// The driver parses what `arm` returns into a [`PromiseId`] so it can call + /// the external completion API with it, and passes that same parsed value + /// into `wait`. Only the first of those directions is exercised anywhere + /// else in this repository — the density benchmark keeps the raw + /// `ValueAndType` for its agent calls and never re-encodes one. So the + /// encoding side is pinned here rather than discovered on a cluster: a + /// `PromiseId` that does not round-trip would fail every round of S11 after + /// the run had already spent its baseline. + #[test] + fn a_promise_id_survives_the_round_trip_back_into_an_agent() { + use golem_common::model::{AgentId, OplogIndex}; + use golem_wasm::IntoValue; + + let parsed: ParsedAgentId = agent_id!(PROMISE_WAITER_AGENT, "w-0001".to_string()); + let promise_id = PromiseId { + agent_id: AgentId { + component_id: golem_common::model::component::ComponentId(uuid::Uuid::nil()), + agent_id: parsed.to_string(), + }, + oplog_idx: OplogIndex::from_u64(42), + }; + + let encoded = promise_id.clone().into_value(); + let decoded = PromiseId::from_value(encoded).expect("promise id should decode"); + assert_eq!(decoded, promise_id); + assert_eq!(decoded.oplog_idx.as_u64(), 42); + } } From d31720291e02ebc7e971a5e66e5a35dd0089e8ef Mon Sep 17 00:00:00 2001 From: kmatasfp <33095685+kmatas@users.noreply.github.com> Date: Thu, 20 Aug 2026 16:39:57 -0700 Subject: [PATCH 10/40] Pass the agent's own promise value back to wait, and smoke-test a round first --- integration-tests/src/chaos/scenarios/s11.rs | 18 ++ integration-tests/src/chaos/waiters.rs | 184 +++++++++++++++---- 2 files changed, 167 insertions(+), 35 deletions(-) diff --git a/integration-tests/src/chaos/scenarios/s11.rs b/integration-tests/src/chaos/scenarios/s11.rs index f5fd04f99f..a8ebae9649 100644 --- a/integration-tests/src/chaos/scenarios/s11.rs +++ b/integration-tests/src/chaos/scenarios/s11.rs @@ -215,6 +215,24 @@ pub async fn run( info!("S11: warmed {warmed} waiters, settling {WARMUP_SETTLE:?}"); tokio::time::sleep(WARMUP_SETTLE).await; + // ── Prove one whole round works ───────────────────────────────────────── + // Before aiming, before the baseline, before anything that costs the window. + // Arming and completing can both succeed while the parking in between is + // refused, and that combination looks entirely healthy in the operation + // totals — so the totals are not what gets asked. + if let Err(e) = waiters::smoke_test(&ctx, promise_config.dwell()).await { + warn!("S11: a single promise round does not work against this cluster: {e}"); + let records = history.snapshot(); + finish!( + TerminationReason::PlatformUnreachable { + detail: format!("promise round smoke test failed: {e}"), + }, + &records, + None + ); + } + info!("S11: smoke round armed, parked, completed and woke"); + // ── Aim the fault ─────────────────────────────────────────────────────── // Before the baseline, because a run that cannot be aimed should not spend a // maintenance window proving it. diff --git a/integration-tests/src/chaos/waiters.rs b/integration-tests/src/chaos/waiters.rs index ab8f22b171..61aecaca1e 100644 --- a/integration-tests/src/chaos/waiters.rs +++ b/integration-tests/src/chaos/waiters.rs @@ -76,7 +76,7 @@ use golem_common::base_model::agent::ParsedAgentId; use golem_common::model::PromiseId; use golem_common::{agent_id, data_value}; use golem_test_framework::dsl::TestDsl; -use golem_wasm::FromValue; +use golem_wasm::{FromValue, ValueAndType}; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::time::Duration; @@ -114,6 +114,15 @@ const READ_CONCURRENCY: usize = 16; /// wakeup the waiter recorded rather than one number. const READ_TIMEOUT: Duration = Duration::from_secs(60); +/// How long the smoke round waits for its one wakeup before giving up. +/// +/// Generous against a cluster that is merely cold, short enough that a broken +/// completion path costs seconds rather than a baseline. +const SMOKE_WAKE_TIMEOUT: Duration = Duration::from_secs(60); + +/// How often the smoke round asks whether its waiter came back. +const SMOKE_POLL_INTERVAL: Duration = Duration::from_secs(2); + /// A running waiter workload. /// /// As elsewhere, dropping the handle does not stop it: call @@ -172,6 +181,81 @@ pub async fn warm(ctx: &WorkloadContext, waiters: &[String]) -> usize { warmed } +/// Drives one whole round against a throwaway waiter, before the run commits to +/// anything. +/// +/// The first S11 run spent its entire baseline and then aborted on a count, +/// because `arm` and `complete` were both healthy and only `wait` was refused — +/// so the operation totals looked normal right up until nothing had woken. The +/// scenario knew something was wrong three minutes after it could have. +/// +/// This is the cheap version of that question, asked first and answered with the +/// platform's own error rather than with a zero. It uses an agent outside the +/// waiter pool, so a failed smoke test leaves the measured population untouched. +/// +/// Errors are returned rather than recorded: this is a pre-flight check, not +/// part of the workload, and its whole value is in saying *why*. +pub async fn smoke_test(ctx: &WorkloadContext, dwell: Duration) -> Result<(), String> { + let waiter = format!("{}-promise-smoke", ctx.key_prefix); + let parsed: ParsedAgentId = agent_id!(PROMISE_WAITER_AGENT, waiter.clone()); + let token = format!("{waiter}-smoke"); + + let created = ctx + .user + .invoke_and_await_agent(&ctx.promise, &parsed, "arm", data_value!(token.clone())) + .await + .map_err(|e| format!("arm failed: {e:#}"))?; + let promise = created + .into_return_value_and_type() + .ok_or_else(|| "arm returned no promise id".to_string())?; + let id = PromiseId::from_value(promise.value.clone()) + .map_err(|e| format!("arm returned a promise id the driver cannot parse: {e}"))?; + + // Fire-and-forget on purpose. A `wait` that parks correctly never returns + // while we are looking at it, so awaiting it here would prove nothing and + // block forever — but a `wait` the platform refuses fails at the API + // boundary, which is exactly the failure this is here to catch. + ctx.user + .invoke_agent( + &ctx.promise, + &parsed, + "wait", + data_value!(token.clone(), promise), + ) + .await + .map_err(|e| format!("wait was refused: {e:#}"))?; + + tokio::time::sleep(dwell).await; + ctx.user + .complete_promise(&id, COMPLETION_PAYLOAD.to_vec()) + .await + .map_err(|e| format!("complete_promise failed: {e:#}"))?; + + // The waiter has to actually come back. Polled rather than awaited for the + // same reason as above. + let deadline = tokio::time::Instant::now() + SMOKE_WAKE_TIMEOUT; + loop { + if let Ok(value) = ctx + .user + .invoke_and_await_agent(&ctx.promise, &parsed, "wakes", data_value!()) + .await + && let Some(wakes) = value + .into_return_value_and_type() + .and_then(|v| u32::from_value(v.value).ok()) + && wakes > 0 + { + return Ok(()); + } + if tokio::time::Instant::now() >= deadline { + return Err(format!( + "the smoke waiter was armed, told to wait and completed, and did not wake \ + within {SMOKE_WAKE_TIMEOUT:?}" + )); + } + tokio::time::sleep(SMOKE_POLL_INTERVAL).await; + } +} + /// Starts one loop per waiter. pub fn start(ctx: WorkloadContext, waiters: &[String], config: &PromiseConfig) -> WaiterHandle { let running = Arc::new(AtomicBool::new(true)); @@ -225,7 +309,7 @@ async fn run_waiter( round += 1; rounds.fetch_add(1, Ordering::Relaxed); - let Some(promise_id) = arm(&ctx, &waiter, &parsed, &token).await else { + let Some(armed) = arm(&ctx, &waiter, &parsed, &token).await else { // Nothing to complete and nobody parked. The next round tries again // after a dwell, which keeps a wholly unreachable platform from // spinning. @@ -241,12 +325,12 @@ async fn run_waiter( let waiter = waiter.clone(); let parsed = parsed.clone(); let token = token.clone(); - let promise_id = promise_id.clone(); - tokio::spawn(async move { wait(&ctx, &waiter, &parsed, &token, &promise_id).await }) + let promise = armed.value.clone(); + tokio::spawn(async move { wait(&ctx, &waiter, &parsed, &token, &promise).await }) }; tokio::time::sleep(dwell).await; - complete(&ctx, &waiter, &token, &promise_id).await; + complete(&ctx, &waiter, &token, &armed.id).await; // The handle is dropped rather than aborted on the timeout path, which // leaves the invocation running. That is deliberate: aborting it would @@ -266,9 +350,35 @@ async fn run_waiter( } } +/// One armed promise, in the two forms the round needs it. +/// +/// Both halves are kept because the two consumers want different things and +/// only one of them will accept a re-encoding. +#[derive(Clone)] +struct Armed { + /// For [`TestDsl::complete_promise`], the external REST API, which takes a + /// typed promise id and serialises it itself. + id: PromiseId, + /// For the `wait` invocation, which passes the promise back *into* an + /// agent. + /// + /// This has to be the value the agent handed out, verbatim. Re-encoding the + /// parsed [`PromiseId`] does not work: `data_value!` derives the parameter's + /// type from `IntoValue`, which names the record's fields with their WIT + /// spellings (`agent-id`, `oplog-idx`), while the agent's generated + /// parameter schema declares them in the component model's own spelling + /// (`agent_id`, `oplog_idx`). The type checker rejects the call outright. + /// Threading the original value through sidesteps the question, because its + /// type came from the agent in the first place. + /// + /// `crate::benchmarks::density::promise` keeps both halves for exactly this + /// reason. + value: ValueAndType, +} + /// Creates the round's promise, recording the invocation. /// -/// The promise id comes back through a cell rather than a return value because +/// The result comes back through a cell rather than a return value because /// [`workload::run_operation`] owns the retry rule and the failure /// classification for every stream in the suite, and it reports outcomes rather /// than payloads. Duplicating it here to get one value back would put the two @@ -278,8 +388,8 @@ async fn arm( waiter: &str, parsed: &ParsedAgentId, token: &str, -) -> Option { - let cell: Arc>> = Arc::new(std::sync::Mutex::new(None)); +) -> Option { + let cell: Arc>> = Arc::new(std::sync::Mutex::new(None)); let sink = cell.clone(); let ctx2 = ctx.clone(); let parsed2 = parsed.clone(); @@ -309,9 +419,9 @@ async fn arm( let value = created .into_return_value_and_type() .ok_or_else(|| anyhow::anyhow!("arm returned no promise id"))?; - let promise_id = PromiseId::from_value(value.value) + let id = PromiseId::from_value(value.value.clone()) .map_err(|e| anyhow::anyhow!("invalid promise id: {e}"))?; - *sink.lock().unwrap() = Some(promise_id); + *sink.lock().unwrap() = Some(Armed { id, value }); Ok(None) } }, @@ -327,7 +437,7 @@ async fn wait( waiter: &str, parsed: &ParsedAgentId, token: &str, - promise_id: &PromiseId, + promise: &ValueAndType, ) { let ctx2 = ctx.clone(); let parsed2 = parsed.clone(); @@ -341,7 +451,7 @@ async fn wait( let ctx = ctx2.clone(); let parsed = parsed2.clone(); let token = token.to_string(); - let promise_id = promise_id.clone(); + let promise = promise.clone(); async move { ctx.user .invoke_and_await_agent_with_key( @@ -349,7 +459,7 @@ async fn wait( &parsed, &key, "wait", - data_value!(token, promise_id), + data_value!(token, promise), ) .await?; Ok(None) @@ -551,33 +661,37 @@ mod tests { assert_eq!(from_millis(0).timestamp_millis(), 0); } - /// A promise id has to survive the trip back *into* an agent. + /// Why `wait` is handed the value `arm` returned rather than a re-encoded + /// [`PromiseId`]. + /// + /// This is not a style preference, and getting it wrong is not caught by + /// anything local: the first S11 run had every single `wait` refused by the + /// type checker in two milliseconds, with `arm` and `complete` both clean, + /// because a re-encoded promise id names its fields the way WIT does and the + /// agent's generated parameter schema names them the way the component model + /// does. The two spellings never meet. /// - /// The driver parses what `arm` returns into a [`PromiseId`] so it can call - /// the external completion API with it, and passes that same parsed value - /// into `wait`. Only the first of those directions is exercised anywhere - /// else in this repository — the density benchmark keeps the raw - /// `ValueAndType` for its agent calls and never re-encodes one. So the - /// encoding side is pinned here rather than discovered on a cluster: a - /// `PromiseId` that does not round-trip would fail every round of S11 after - /// the run had already spent its baseline. + /// So the rule is: a value that came *out* of an agent goes back *into* one + /// verbatim. This test pins the mismatch that makes the rule necessary, so + /// that a future change to either derive shows up here rather than on a + /// cluster. #[test] - fn a_promise_id_survives_the_round_trip_back_into_an_agent() { - use golem_common::model::{AgentId, OplogIndex}; + fn a_re_encoded_promise_id_does_not_name_its_fields_the_way_an_agent_declares_them() { + use golem_common::model::PromiseId; use golem_wasm::IntoValue; + use golem_wasm::analysis::AnalysedType; - let parsed: ParsedAgentId = agent_id!(PROMISE_WAITER_AGENT, "w-0001".to_string()); - let promise_id = PromiseId { - agent_id: AgentId { - component_id: golem_common::model::component::ComponentId(uuid::Uuid::nil()), - agent_id: parsed.to_string(), - }, - oplog_idx: OplogIndex::from_u64(42), + let AnalysedType::Record(record) = PromiseId::get_type() else { + panic!("a promise id is a record"); }; + let names: Vec<&str> = record.fields.iter().map(|f| f.name.as_str()).collect(); + + // What `data_value!` would ship: the WIT spellings. + assert!(names.contains(&"agent-id"), "got {names:?}"); + assert!(names.contains(&"oplog-idx"), "got {names:?}"); - let encoded = promise_id.clone().into_value(); - let decoded = PromiseId::from_value(encoded).expect("promise id should decode"); - assert_eq!(decoded, promise_id); - assert_eq!(decoded.oplog_idx.as_u64(), 42); + // What the agent's parameter schema actually asks for, and does not get. + assert!(!names.contains(&"agent_id"), "got {names:?}"); + assert!(!names.contains(&"oplog_idx"), "got {names:?}"); } } From c38fb86a87c8080319b3e9f7bbed5a64e7edf56f Mon Sep 17 00:00:00 2001 From: kmatasfp <33095685+kmatas@users.noreply.github.com> Date: Thu, 20 Aug 2026 17:58:34 -0700 Subject: [PATCH 11/40] Report the caller's view of a wakeup beside the platform's --- integration-tests/src/chaos/scenarios/s11.rs | 48 +++++- integration-tests/src/chaos/wakeups.rs | 167 ++++++++++++++++++- 2 files changed, 205 insertions(+), 10 deletions(-) diff --git a/integration-tests/src/chaos/scenarios/s11.rs b/integration-tests/src/chaos/scenarios/s11.rs index a8ebae9649..0668a0fe15 100644 --- a/integration-tests/src/chaos/scenarios/s11.rs +++ b/integration-tests/src/chaos/scenarios/s11.rs @@ -355,9 +355,6 @@ pub async fn run( phases.fault = Some(PhaseWindow::started(injected.injected_at)); let on_pod: BTreeSet = chosen.on_pod.iter().cloned().collect(); - let parked = parked_at_injection(&history.snapshot(), injected.injected_at, &on_pod); - attention_extra.push(parked.note()); - info!("S11: {}", parked.describe()); let recovered = match signals.await_fault_recovered(config.signal_timeout()).await { Ok(recovered) => recovered, @@ -399,6 +396,15 @@ pub async fn run( tokio::time::sleep(settle).await; let records = history.snapshot(); + + // Only now, with every `wait` record landed, can the parked population be + // counted — see [`parked_at_injection`]. + if let Some(injected_at) = fault_injected_at { + let parked = parked_at_injection(&records, injected_at, &on_pod); + attention_extra.push(parked.note()); + info!("S11: {}", parked.describe()); + } + let logs = waiters::read_logs(&ctx, &waiter_names).await; // Archived alongside the operations, not just reduced into the report: the // reduced numbers cannot be recomputed later, and a correction to how a @@ -476,9 +482,12 @@ impl ParkedAtInjection { /// Whether the kill landed anywhere near the mechanism under test. /// /// A run that caught nothing parked proves nothing however clean the rest of - /// its numbers look, which is the one thing here a human has to act on. + /// its numbers look, which is the one thing here a human has to act on. So + /// does a run that caught waiters but none of them *on the pod it killed*: + /// the affected group would be empty and every number in the report would + /// describe an undisturbed cluster. pub fn needs_attention(&self) -> bool { - self.total == 0 + self.total == 0 || self.on_killed_executor == 0 } /// The same line as [`Self::describe`], classified. @@ -492,6 +501,14 @@ impl ParkedAtInjection { this run says nothing about promise-completion recovery" .to_string(); } + if self.on_killed_executor == 0 { + return format!( + "WARNING: {} waiters were suspended when the executor died but none of them \ + were on it, so the affected group is empty and this run says nothing about \ + promise-completion recovery", + self.total + ); + } format!( "S11 killed the executor with {} waiters suspended on promises, {} of them on \ waiters it owned", @@ -506,6 +523,13 @@ impl ParkedAtInjection { /// returned by then. Not derived from the round arithmetic, because a platform /// that had slowed down would have armed fewer rounds than the cadence says, and /// the whole point of this number is to say whether the kill landed in anything. +/// +/// **This must be computed from the finished history, not from a snapshot taken +/// at injection time.** [`crate::chaos::workload::run_operation`] appends a +/// record only once the operation has completed, so a snapshot taken at the +/// moment of the kill contains none of the invocations that were open across it +/// — which is precisely the population being counted. The first S11 run reported +/// 5 waiters parked when 200 were, and read as a run that had tested nothing. pub fn parked_at_injection( records: &[OperationRecord], injected_at: DateTime, @@ -608,6 +632,20 @@ mod tests { assert_eq!(parked_at_injection(&records, at(100), &on_pod).total, 1); } + /// A kill that caught waiters but none of its *own* is exactly as + /// uninformative as one that caught none at all — the affected group is + /// empty either way — and the first completed S11 run reported this shape as + /// ordinary context because the rule only looked at the total. + #[test] + fn a_kill_that_caught_none_of_its_own_waiters_needs_attention() { + let parked = ParkedAtInjection { + total: 200, + on_killed_executor: 0, + }; + assert!(parked.needs_attention()); + assert!(parked.describe().contains("none of them")); + } + #[test] fn a_kill_that_caught_nothing_parked_needs_attention() { let parked = ParkedAtInjection { diff --git a/integration-tests/src/chaos/wakeups.rs b/integration-tests/src/chaos/wakeups.rs index 212e4fea25..55506a84e2 100644 --- a/integration-tests/src/chaos/wakeups.rs +++ b/integration-tests/src/chaos/wakeups.rs @@ -84,6 +84,13 @@ const MAX_FINDINGS: usize = 200; /// The method name the completion operations are recorded under. const COMPLETE_METHOD: &str = "complete"; +/// The method name the parking invocations are recorded under. +const WAIT_METHOD: &str = "wait"; + +/// What [`WAIT_METHOD`] operations append to their round's token to form their +/// own idempotency key. +const WAIT_KEY_SUFFIX: &str = "-wait"; + /// Whether a waiter was on the executor the fault killed. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] @@ -165,6 +172,17 @@ pub struct WakeupDelayStats { /// Armed-to-woken, on the executor's clock alone. Carries the round's dwell /// as well as the delay, and carries no skew. pub parked: LatencyStats, + /// How much longer the *caller* waited than the platform actually took. + /// + /// The `wait` invocation's own duration, less the round's dwell, less the + /// delay the waiter recorded. On a healthy round this is a few milliseconds + /// of round trip. A large value means the platform woke the agent on time + /// and the answer did not come back — which is a different defect from a + /// slow wakeup, and invisible in [`Self::delay`]. + pub client_excess: LatencyStats, + /// Rounds whose caller waited longer than the whole wakeup budget *after* + /// the waiter had already woken. + pub client_stalled: u64, } /// The promise-wakeup account. @@ -209,6 +227,14 @@ pub struct WakeupReport { /// invocation, which is what a wedged worker looks like from outside. pub waiters_wedged: Vec, pub delay: Vec, + /// Rounds across every cell whose caller waited past the budget after the + /// wakeup had already happened. See [`WakeupDelayStats::client_excess`]. + pub client_stalled_total: u64, + /// The worst such gap, in milliseconds. + pub client_stall_worst_ms: u64, + /// How many of those callers had to retry to get their answer at all. A + /// stall that only ends on a retry is a request that was never coming back. + pub client_stall_retried: u64, pub findings: Vec, /// Findings past [`MAX_FINDINGS`], which the report drops rather than /// carries. Non-zero means `findings` is a sample. @@ -276,6 +302,9 @@ impl WakeupReport { let mut cells: BTreeMap<(WaiterGroup, Window), DelayCell> = BTreeMap::new(); let mut claimed: BTreeSet<&str> = BTreeSet::new(); + let waits = wait_observations(records); + let mut stall_worst_ms = 0u64; + let mut stall_retried = 0u64; for round in completions(records) { let group = if on_killed.contains(round.agent) { @@ -333,11 +362,23 @@ impl WakeupReport { report.indeterminate_that_woke += 1; } let delay_ms = (wakeup.woken_at - round.submitted_at).num_milliseconds(); - cells.entry((group, window)).or_default().push( - delay_ms, - wakeup.parked_ms(), - budget_ms, - ); + let cell = cells.entry((group, window)).or_default(); + cell.push(delay_ms, wakeup.parked_ms(), budget_ms); + + // The caller's own view of the same round. `wait` covers the + // dwell as well as the wakeup, so the dwell comes off before + // the two are compared. + if let Some(observed) = waits.get(round.token) { + let client_ms = observed.duration_ms.saturating_sub(dwell_ms); + let excess = client_ms.saturating_sub(delay_ms.max(0) as u64); + cell.push_client(excess, budget_ms); + if excess > budget_ms { + stall_worst_ms = stall_worst_ms.max(excess); + if observed.attempts > 1 { + stall_retried += 1; + } + } + } } continue; } @@ -398,6 +439,9 @@ impl WakeupReport { .into_iter() .map(|((group, window), cell)| cell.into_stats(group, window)) .collect(); + report.client_stalled_total = report.delay.iter().map(|c| c.client_stalled).sum(); + report.client_stall_worst_ms = stall_worst_ms; + report.client_stall_retried = stall_retried; report } @@ -419,6 +463,9 @@ impl WakeupReport { waiters_stood_down: 0, waiters_wedged: Vec::new(), delay: Vec::new(), + client_stalled_total: 0, + client_stall_worst_ms: 0, + client_stall_retried: 0, findings: Vec::new(), findings_omitted: 0, } @@ -486,6 +533,14 @@ impl WakeupReport { self.unverifiable )); } + if self.client_stalled_total > 0 { + lines.push(format!( + "S11: {} rounds woke on time and the caller was not told for up to {}ms — {} of \ + them only got an answer by retrying. The waiters' own logs say the platform \ + resumed them promptly, so this is the response path, not the wakeup", + self.client_stalled_total, self.client_stall_worst_ms, self.client_stall_retried + )); + } if self.unknown_tokens > 0 { lines.push(format!( "S11 recorded {} wakeups whose token no completion claims — agent names carry \ @@ -530,6 +585,8 @@ impl WakeupReport { struct DelayCell { delays: Vec, parked: Vec, + client_excess: Vec, + client_stalled: u64, min_delay_ms: i64, over_budget: u64, any: bool, @@ -549,6 +606,14 @@ impl DelayCell { self.parked.push(parked_ms.max(0) as u64); } + /// Records how much longer the caller waited than the platform took. + fn push_client(&mut self, excess_ms: u64, budget_ms: u64) { + if excess_ms > budget_ms { + self.client_stalled += 1; + } + self.client_excess.push(excess_ms); + } + fn into_stats(self, group: WaiterGroup, window: Window) -> WakeupDelayStats { WakeupDelayStats { group, @@ -557,10 +622,42 @@ impl DelayCell { min_delay_ms: self.min_delay_ms, over_budget: self.over_budget, parked: LatencyStats::from_durations(self.parked), + client_excess: LatencyStats::from_durations(self.client_excess), + client_stalled: self.client_stalled, } } } +/// What the driver's own `wait` invocation cost, per round. +/// +/// Keyed by the round's token: the `wait` operation is recorded under +/// `{token}-wait`, which is what lets the caller's view be joined to the +/// waiter's own. +struct WaitObservation { + duration_ms: u64, + attempts: u32, +} + +fn wait_observations(records: &[OperationRecord]) -> BTreeMap<&str, WaitObservation> { + records + .iter() + .filter(|r| r.stream == Stream::PromiseWait && r.method == WAIT_METHOD) + .filter_map(|r| { + r.idempotency_key + .strip_suffix(WAIT_KEY_SUFFIX) + .map(|token| { + ( + token, + WaitObservation { + duration_ms: r.duration_ms, + attempts: r.attempts, + }, + ) + }) + }) + .collect() +} + /// The completion operations, which are the rounds this report is about. fn completions(records: &[OperationRecord]) -> impl Iterator> { records @@ -944,6 +1041,66 @@ mod tests { assert_eq!(report.violations(), MAX_FINDINGS as u64 + 10); } + /// The caller's view and the waiter's own can disagree, and when they do the + /// waiter's is the one that describes the platform. + /// + /// This is not hypothetical: the first completed S11 run woke every one of + /// 21,863 completions on time, and 89 of those callers were not told for 125 + /// seconds. A report with only [`WakeupReport::delay`] in it would have + /// called that run flawless. + #[test] + fn a_wakeup_the_caller_was_not_told_about_is_counted_separately_from_the_delay() { + let mut waited = completion("w-1", "t-1-wait", at(110), Outcome::Confirmed); + waited.method = "wait".to_string(); + // Parked 5s, then 125s before the caller heard anything back. + waited.duration_ms = 130_000; + waited.attempts = 2; + + let records = vec![ + completion("w-1", "t-1", at(110), Outcome::Confirmed), + waited, + ]; + // The waiter itself woke one second after the completion. + let logs = vec![log("w-1", vec![wakeup("t-1", at(105), at(111))])]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + // The platform did its job, and the delay table says so. + assert_eq!(report.woke_once, 1); + assert_eq!(report.delay[0].delay.max_ms, 1_000); + assert_eq!(report.delay[0].over_budget, 0); + assert!(!report.has_violations()); + + // And the caller still waited two minutes past that. + assert_eq!(report.client_stalled_total, 1); + assert_eq!(report.client_stall_worst_ms, 124_000); + assert_eq!(report.client_stall_retried, 1); + assert!( + report + .attention_lines() + .iter() + .any(|l| l.contains("response path, not the wakeup")) + ); + } + + /// A healthy round's caller waits the dwell plus the wakeup and nothing + /// more, so it must not be counted as stalled. + #[test] + fn a_prompt_round_records_no_client_stall() { + let mut waited = completion("w-1", "t-1-wait", at(110), Outcome::Confirmed); + waited.method = "wait".to_string(); + waited.duration_ms = 5_040; + + let records = vec![ + completion("w-1", "t-1", at(110), Outcome::Confirmed), + waited, + ]; + let logs = vec![log("w-1", vec![wakeup("t-1", at(105), at(110))])]; + let report = build(&records, &logs, &split_of(&["w-1"], &[]), 0); + + assert_eq!(report.client_stalled_total, 0); + assert!(report.attention_lines().is_empty()); + } + /// Standing waiters down is normal on a run where recovery was slow. It only /// becomes an attention line when those waiters then could not be read. #[test] From dfd6403266f79da206c28be2b6e2d3d59cb72790 Mon Sep 17 00:00:00 2001 From: kmatasfp <33095685+kmatas@users.noreply.github.com> Date: Mon, 24 Aug 2026 13:46:21 -0700 Subject: [PATCH 12/40] Add S3 chaos scenario driver and reachability account --- golem-test-framework/src/benchmark/config.rs | 2 + .../chaos_suites/cloud-chaos.yaml | 93 ++ integration-tests/src/benchmarks/all.rs | 4 + integration-tests/src/chaos/mod.rs | 89 +- integration-tests/src/chaos/reachability.rs | 862 ++++++++++++++++++ integration-tests/src/chaos/result.rs | 91 +- integration-tests/src/chaos/scenarios/mod.rs | 6 + integration-tests/src/chaos/scenarios/s1.rs | 1 + integration-tests/src/chaos/scenarios/s10.rs | 1 + integration-tests/src/chaos/scenarios/s11.rs | 1 + integration-tests/src/chaos/scenarios/s12.rs | 1 + integration-tests/src/chaos/scenarios/s13.rs | 1 + integration-tests/src/chaos/scenarios/s3.rs | 453 +++++++++ integration-tests/src/chaos/scenarios/s5.rs | 1 + integration-tests/src/chaos/scenarios/s8.rs | 1 + integration-tests/src/chaos/split.rs | 17 +- integration-tests/src/chaos/steady.rs | 122 +++ integration-tests/src/chaos/summary.rs | 21 + integration-tests/src/chaos/workload.rs | 2 +- 19 files changed, 1765 insertions(+), 4 deletions(-) create mode 100644 integration-tests/src/chaos/reachability.rs create mode 100644 integration-tests/src/chaos/scenarios/s3.rs create mode 100644 integration-tests/src/chaos/steady.rs diff --git a/golem-test-framework/src/benchmark/config.rs b/golem-test-framework/src/benchmark/config.rs index 5706f48f26..729ac3f625 100644 --- a/golem-test-framework/src/benchmark/config.rs +++ b/golem-test-framework/src/benchmark/config.rs @@ -246,6 +246,8 @@ pub enum ChaosScenarioArg { S10, /// Executor pod kill while agents are suspended on promises being completed. S11, + /// Executor cut off from worker-service while it keeps its shards. + S3, } /// Density subcommand action. diff --git a/integration-tests/chaos_suites/cloud-chaos.yaml b/integration-tests/chaos_suites/cloud-chaos.yaml index 6dcdf4ca1a..c7ad4944f5 100644 --- a/integration-tests/chaos_suites/cloud-chaos.yaml +++ b/integration-tests/chaos_suites/cloud-chaos.yaml @@ -529,3 +529,96 @@ scenarios: delaySecs: 5 signalTimeoutSecs: 1800 + + # S3 — executor / worker-service network partition (GOL-370). + # + # The only fault in the suite the platform never learns about. S1 cuts the + # shard-manager off from an executor and the cluster reacts: health checks are + # missed and the shards move. S3 cuts the link worker-service uses, leaves the + # shard-manager link alone, and so the executor keeps its shards and stays in + # the routing table. worker-service is told, correctly, that this pod owns + # those agents, and keeps trying to reach a pod it cannot reach. There is no + # route around it, because as far as the platform is concerned nothing is + # wrong. + # + # Like S8, S10 and S11 the driver names the pod rather than letting Chaos Mesh + # pick one, and keeps driving the agents on the other executor as the control + # group. That control group is where the interesting finding would be: a + # partition it is not part of should cost it nothing. + - code: S3 + name: executor-worker-service-partition + enabled: true + + fault: + kind: network-partition + target: worker-executor + # The workflow narrows the selector to the pod the driver named; `one` + # stays as the belt-and-braces bound, exactly as in S8, S10 and S11. + # + # Only the executor side is pinned. The other side of the cut is every + # worker-service replica, selected by label — a partition from two of the + # three would leave the agents reachable through the third and the run + # would measure nothing. + mode: one + durationSecs: 180 + + phases: + # Long enough for cold starts and route warm-up to settle, so the cut + # lands on a population that has been cycling steadily. + baselineSecs: 300 + # Has to comfortably exceed the caller's 120s attempt timeout, or no + # operation would ever reach the timeout inside the window and the + # pending-then-timeout behaviour the scenario exists to show would fall + # outside it. 180s leaves room for the first attempt to time out, the + # one permitted retry to go out, and that retry to still be waiting when + # the link comes back. + faultSecs: 180 + # Every isolated agent has to get several whole operations after the heal, + # so one that recovered slowly is distinguishable from one that never did. + recoverySecs: 420 + + isolation: + # 200 agents, split by shard ownership across the two executors, so ~100 + # end up isolated and ~100 are the control group. Also the resolution of + # the report: a lost agent localises to one out of two hundred. + agents: 200 + # One operation per agent per second, so ~200 ops/s offered while + # everything is reachable. Higher than the other scenarios' 100/s because + # half of it stops the moment the fault lands: the number that has to be + # large is the *control* group's, since a collateral-damage finding is a + # percentage change in its throughput and a small sample cannot show one. + intervalMillis: 1000 + # The most of its baseline the isolated group may keep for the partition + # to count as observed. Not zero: an operation submitted late in the fault + # window can still be waiting when the link returns, and then confirms. + # At this cadence that is a handful against a baseline of hundreds, so + # anything near 25% means the cut did not take hold. + isolatedCeilingPercent: 25 + # The least of its baseline the control group must keep. Below this, + # serving an unreachable executor cost the agents that had nothing to do + # with it — the sharpest finding this scenario can produce. + # + # 75% rather than 95%: worker-service invalidates one process-wide routing + # table entry on every failed call, so some shared cost is expected by + # construction. `invalidation_min_delay` bounds it to twice a second, + # which should keep it far above this line. + controlFloorPercent: 75 + # What being served again may cost an isolated agent once the link is + # back. Recorded, not asserted: nothing has to be recovered here — no + # shard moves and no worker restarts — so this is really a measure of how + # long worker-service's own retry loop takes to notice, and how long that + # may be is a judgement. + recoveryBudgetSecs: 60 + + retryPolicy: + # Identical to the others and load-bearing for the same reason, with one + # consequence specific to S3: the retry is what turns a single 120s stall + # into a second attempt under the same key. If the link heals in between, + # that retry succeeds against work the first attempt may already have + # started — which is exactly the case the exactly-once probe is here to + # rule on. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 diff --git a/integration-tests/src/benchmarks/all.rs b/integration-tests/src/benchmarks/all.rs index 0f61fc1bed..07b362a81b 100644 --- a/integration-tests/src/benchmarks/all.rs +++ b/integration-tests/src/benchmarks/all.rs @@ -594,6 +594,7 @@ async fn run_chaos( ChaosScenarioArg::S13 => chaos::ScenarioCode::S13, ChaosScenarioArg::S10 => chaos::ScenarioCode::S10, ChaosScenarioArg::S11 => chaos::ScenarioCode::S11, + ChaosScenarioArg::S3 => chaos::ScenarioCode::S3, }; let config = suite .scenario(code, allow_disabled) @@ -631,6 +632,9 @@ async fn run_chaos( chaos::ScenarioCode::S11 => { chaos::scenarios::s11::run(&config, &manifest, &deps, &signals, &outputs).await } + chaos::ScenarioCode::S3 => { + chaos::scenarios::s3::run(&config, &manifest, &deps, &signals, &outputs).await + } }; deps.kill_all().await; diff --git a/integration-tests/src/chaos/mod.rs b/integration-tests/src/chaos/mod.rs index 9d4f9fcf77..77ff1a80ec 100644 --- a/integration-tests/src/chaos/mod.rs +++ b/integration-tests/src/chaos/mod.rs @@ -39,11 +39,13 @@ pub mod ownership; pub mod pinned; pub mod prep; pub mod probe; +pub mod reachability; pub mod result; pub mod scenarios; pub mod scheduled; pub mod signal; pub mod split; +pub mod steady; pub mod summary; pub mod waiters; pub mod wakeups; @@ -72,6 +74,8 @@ pub enum ScenarioCode { S10, /// Executor pod kill while agents are suspended on promises being completed. S11, + /// Executor cut off from worker-service while it keeps its shards. + S3, } impl ScenarioCode { @@ -84,14 +88,16 @@ impl ScenarioCode { ScenarioCode::S13 => "S13", ScenarioCode::S10 => "S10", ScenarioCode::S11 => "S11", + ScenarioCode::S3 => "S3", } } /// Every scenario this driver implements. The suite YAML is checked against /// this list, so a scenario cannot be enabled in YAML without code behind /// it, nor implemented without an operational switch in front of it. - pub const ALL: [ScenarioCode; 7] = [ + pub const ALL: [ScenarioCode; 8] = [ ScenarioCode::S1, + ScenarioCode::S3, ScenarioCode::S5, ScenarioCode::S8, ScenarioCode::S10, @@ -403,6 +409,69 @@ impl PromiseConfig { } } +/// Shape of the reachability workload (GOL-370). +/// +/// The fifth experiment shape, and the only one where nothing about the +/// platform is broken at all. [`PinnedConfig`] asks what happens to operations +/// running on a pod that dies; this one asks what happens to operations bound +/// for a pod that is perfectly healthy and simply cannot be reached from the +/// tier that routes to it. The executor keeps its shards for the whole fault, +/// because the link it needs in order to keep them — to the shard-manager — is +/// not the one that was cut. +/// +/// Every agent gets its own emitter holding at most one operation, rather than +/// the shared per-stream budget [`WorkloadConfig`] drives. That is load-bearing +/// here and not a style choice: the stall this scenario induces is bounded by +/// *which executor owns the agent*, not by which stream it belongs to, so a +/// shared budget would be drained by the isolated half and would stop the +/// reachable half submitting too. The run would then report the control group +/// degrading, and the cause would be the driver. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct IsolationConfig { + /// Durable counter agents, split by shard ownership into the ones the + /// isolated executor holds and the ones it does not. Also the resolution of + /// the report: a finding localises to one agent out of this many. + pub agents: u32, + /// Milliseconds between one agent's operations, measured from the end of + /// the previous one. The offered rate is `agents / interval`, and an agent + /// whose operation is stalled offers nothing at all — which is the signal, + /// not a gap in it. + pub interval_millis: u64, + /// The most of its own baseline throughput the isolated group may keep + /// during the fault, as a percentage, for the partition to count as + /// observed. + /// + /// A run above this line did not cut the executor off, whatever the fault + /// status says, and every other number in the report is then a measurement + /// of an undisturbed cluster. That is reported as inconclusive rather than + /// clean: a healthy-looking result from a fault that never landed is the + /// worst artifact this suite can produce. + pub isolated_ceiling_percent: f64, + /// The least of its own baseline throughput the control group must keep + /// during the fault, as a percentage. + /// + /// The sharpest thing S3 can find. The agents on the reachable executor + /// have nothing to do with the partition, so a drop here is collateral + /// damage from how worker-service handles an unreachable pod — its routing + /// table is one process-wide entry, and every stalled caller invalidating it + /// costs every other caller a shard-manager round trip. + pub control_floor_percent: f64, + /// What resuming an isolated agent may cost once the link is back, and the + /// number the recovery gap is reported against. Recorded rather than + /// asserted, like every other budget in the suite. + pub recovery_budget_secs: u64, +} + +impl IsolationConfig { + pub fn interval(&self) -> Duration { + Duration::from_millis(self.interval_millis) + } + pub fn recovery_budget(&self) -> Duration { + Duration::from_secs(self.recovery_budget_secs) + } +} + /// One step of the executor scale schedule the workflow runs during the fault. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -481,6 +550,9 @@ pub struct ScenarioConfig { /// The suspended-waiter workload. Absent for scenarios that do not run one. #[serde(default)] pub promise: Option, + /// The reachability workload. Absent for scenarios that do not run one. + #[serde(default)] + pub isolation: Option, /// Shard-ownership oracle settings. Absent for scenarios that do not sample /// executor assignments. #[serde(default)] @@ -554,6 +626,16 @@ impl ScenarioConfig { }) } + /// The reachability workload block. See [`Self::require_workload`]. + pub fn require_isolation(&self) -> anyhow::Result<&IsolationConfig> { + self.isolation.as_ref().ok_or_else(|| { + anyhow::anyhow!( + "chaos scenario {} needs an `isolation` block in the suite YAML", + self.code + ) + }) + } + /// The pinned workload block. See [`Self::require_workload`]. pub fn require_pinned(&self) -> anyhow::Result<&PinnedConfig> { self.pinned.as_ref().ok_or_else(|| { @@ -689,6 +771,7 @@ mod tests { pinned: None, scheduled: None, promise: None, + isolation: None, ownership: None, scale_during_fault: None, retry_policy: RetryPolicy::default(), @@ -761,6 +844,7 @@ mod tests { assert_eq!(ScenarioCode::parse("S12"), Some(ScenarioCode::S12)); assert_eq!(ScenarioCode::parse("s8"), Some(ScenarioCode::S8)); assert_eq!(ScenarioCode::parse("s1"), Some(ScenarioCode::S1)); + assert_eq!(ScenarioCode::parse("s3"), Some(ScenarioCode::S3)); assert_eq!(ScenarioCode::parse("S99"), None); } @@ -793,6 +877,9 @@ mod tests { ScenarioCode::S11 => { entry.require_promise().unwrap(); } + ScenarioCode::S3 => { + entry.require_isolation().unwrap(); + } } } } diff --git a/integration-tests/src/chaos/reachability.rs b/integration-tests/src/chaos/reachability.rs new file mode 100644 index 0000000000..2db5770c0a --- /dev/null +++ b/integration-tests/src/chaos/reachability.rs @@ -0,0 +1,862 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! What a partition between worker-service and one executor cost (GOL-370). +//! +//! Three questions, in the order they have to be answered: +//! +//! 1. **Did the fault land?** The agents on the isolated executor must stop +//! being served. If they did not, nothing else in the report means anything, +//! and it says so — see [`ReachabilityViolation::PartitionNotObserved`]. +//! 2. **What did it cost the agents it was not aimed at?** The other executor +//! is reachable throughout and its agents should be untouched. They share a +//! worker-service with the stalled half, and worker-service keeps one +//! process-wide routing table that every stalled caller invalidates, so +//! "untouched" is a claim worth measuring rather than assuming. +//! 3. **Did the isolated agents come back?** Every one of them, and how long +//! after the link was restored. +//! +//! Throughput, not success rate, is what the first two are measured on, and the +//! difference matters. An emitter holds one operation at a time +//! ([`crate::chaos::steady`]), so an agent whose executor is unreachable does +//! not fail repeatedly — it fails *slowly*, once, and offers nothing else for +//! two minutes. A success rate would read that as one failure out of one +//! attempt and call the group 0% degraded. Confirmed operations per second is +//! the number that collapses, so it is the number the report is built on. +//! +//! ### Reading a non-zero isolated cell +//! +//! Operations are placed in the window they were *submitted* in, which is when +//! the platform was asked to do the work. An isolated operation submitted late +//! in the fault window can still be waiting when the link comes back, and then +//! confirms — so the isolated group's during-fault throughput is small rather +//! than exactly zero. At one operation per agent per second against a +//! two-minute client timeout, that is a handful of confirmations against a +//! baseline of hundreds. A cell anywhere near the ceiling means something else. + +use crate::chaos::history::{OperationRecord, Outcome, Stream}; +use crate::chaos::split::{FaultWindow, Group, PodSplit, Window}; +use crate::chaos::summary::LatencyStats; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; + +/// The most findings the report carries. Past this it says how many it dropped +/// rather than growing without bound: 200 agents that all failed to recover is +/// one fact, not 200. +const MAX_FINDINGS: usize = 50; + +/// What a reachability finding is about. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum ReachabilityViolation { + /// The isolated executor kept serving its agents through the fault. The + /// partition did not take hold where the run says it did, and every other + /// number here describes an undisturbed cluster. + PartitionNotObserved, + /// The agents on the *reachable* executor lost throughput while the other + /// executor was cut off. They were never partitioned from anything. + ControlDegraded, + /// An isolated agent produced no confirmed operation at all once the link + /// was restored. + NeverRecovered, +} + +impl ReachabilityViolation { + pub fn as_str(self) -> &'static str { + match self { + ReachabilityViolation::PartitionNotObserved => "partition-not-observed", + ReachabilityViolation::ControlDegraded => "control-degraded", + ReachabilityViolation::NeverRecovered => "never-recovered", + } + } +} + +/// One finding. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ReachabilityFinding { + pub violation: ReachabilityViolation, + /// The agent it localises to, for the findings that localise to one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub agent: Option, + pub detail: String, +} + +/// What one group of agents managed in one window. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ThroughputCell { + /// `on-pod` is the group the isolated executor owns; `elsewhere` is the + /// control group. The names are the suite's, shared with every other + /// scenario that divides its agents around one pod — see + /// [`crate::chaos::split`]. + pub group: Group, + pub window: Window, + /// Agents of this group that offered at least one operation in this window. + /// Below the group's size means emitters were stalled across the whole + /// window rather than merely slowed. + pub agents_active: usize, + pub submitted: u64, + pub confirmed: u64, + pub rejected: u64, + pub indeterminate: u64, + /// Attempts that hit the client's attempt timeout rather than answering. + /// The pending-then-timeout behaviour the scenario exists to make visible. + pub attempts_timed_out: u64, + pub window_secs: f64, + pub confirmed_per_sec: f64, + /// This cell's rate against the same group's own before-fault rate. `None` + /// for the before-fault cell itself, and for a group that never had a + /// baseline to compare against. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub share_of_baseline_percent: Option, + pub latency: LatencyStats, +} + +/// The reachability account. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ReachabilityReport { + /// The executor the partition was aimed at, as the shard-manager names it. + pub isolated_pod: String, + pub isolated_agents: usize, + pub reachable_agents: usize, + /// The thresholds from the suite YAML, recorded so an archived cell can be + /// read years later against the numbers it was judged by rather than + /// against today's config. + pub isolated_ceiling_percent: f64, + pub control_floor_percent: f64, + pub recovery_budget_ms: u64, + pub cells: Vec, + /// Per isolated agent, how long after the link was restored its first + /// confirmed operation landed. + pub recovery: LatencyStats, + pub recovery_over_budget: u64, + /// Isolated agents that never confirmed anything after the heal. + pub agents_never_recovered: Vec, + /// Records whose agent the selection never saw. Zero on a healthy run; + /// non-zero means the split and the workload disagree about who was driven. + pub records_outside_the_split: u64, + pub findings: Vec, + /// Findings past [`MAX_FINDINGS`], dropped rather than carried. Non-zero + /// means `findings` is a sample. + pub findings_omitted: u64, +} + +/// One group's per-window accumulation, before it becomes a cell. +#[derive(Default)] +struct Tally { + agents: BTreeSet, + submitted: u64, + confirmed: u64, + rejected: u64, + indeterminate: u64, + attempts_timed_out: u64, + durations: Vec, +} + +impl ReachabilityReport { + /// Builds the account from the operation history. + /// + /// `fault` is what the workflow reported. Without it every record lands in + /// [`Window::Unknown`] and the report carries counts but no verdict, which + /// is the honest outcome for a run that never learned when the fault was: + /// the thresholds are all defined relative to a before-and-during + /// comparison that cannot be made. + pub fn build( + records: &[OperationRecord], + split: &PodSplit, + fault: Option, + isolated_ceiling_percent: f64, + control_floor_percent: f64, + recovery_budget: std::time::Duration, + ) -> Self { + let mut tallies: BTreeMap<(Group, Window), Tally> = BTreeMap::new(); + let mut records_outside_the_split = 0u64; + let mut first_submitted: Option> = None; + let mut last_completed: Option> = None; + + for record in records.iter().filter(|r| r.stream == Stream::Durable) { + let Some(group) = split.group_of(&record.agent) else { + records_outside_the_split += 1; + continue; + }; + let window = Window::of(record.submitted_at, fault); + let tally = tallies.entry((group, window)).or_default(); + + tally.agents.insert(record.agent.clone()); + tally.submitted += 1; + match record.outcome { + Outcome::Confirmed => { + tally.confirmed += 1; + tally.durations.push(record.duration_ms); + } + Outcome::Rejected => tally.rejected += 1, + Outcome::Indeterminate => tally.indeterminate += 1, + } + tally.attempts_timed_out += record + .attempt_log + .iter() + .filter(|a| { + a.error + .as_deref() + .is_some_and(|e| e.contains("attempt timed out")) + }) + .count() as u64; + + first_submitted = Some(match first_submitted { + Some(at) if at <= record.submitted_at => at, + _ => record.submitted_at, + }); + if let Some(completed) = record.completed_at { + last_completed = Some(match last_completed { + Some(at) if at >= completed => at, + _ => completed, + }); + } + } + + // Baselines first: every other cell is expressed as a share of its own + // group's before-fault rate, so a lopsided split cannot make one group + // look better than the other. + let mut baseline_rate: BTreeMap = BTreeMap::new(); + let mut cells: Vec = Vec::new(); + for ((group, window), tally) in &tallies { + let secs = window_secs(*window, fault, first_submitted, last_completed); + let rate = if secs > 0.0 { + tally.confirmed as f64 / secs + } else { + 0.0 + }; + if *window == Window::BeforeFault { + baseline_rate.insert(*group, rate); + } + cells.push(ThroughputCell { + group: *group, + window: *window, + agents_active: tally.agents.len(), + submitted: tally.submitted, + confirmed: tally.confirmed, + rejected: tally.rejected, + indeterminate: tally.indeterminate, + attempts_timed_out: tally.attempts_timed_out, + window_secs: round2(secs), + confirmed_per_sec: round2(rate), + share_of_baseline_percent: None, + latency: LatencyStats::from_durations(tally.durations.clone()), + }); + } + + for cell in &mut cells { + if cell.window == Window::BeforeFault { + continue; + } + if let Some(baseline) = baseline_rate.get(&cell.group).filter(|r| **r > 0.0) { + cell.share_of_baseline_percent = + Some(round2(cell.confirmed_per_sec / baseline * 100.0)); + } + } + cells.sort_by_key(|c| (c.group, c.window)); + + // ── Recovery, per isolated agent ──────────────────────────────────── + let recovered_at = fault.and_then(|w| w.recovered_at); + let mut gaps: Vec = Vec::new(); + let mut over_budget = 0u64; + let mut agents_never_recovered: Vec = Vec::new(); + if let Some(healed) = recovered_at { + for agent in &split.on_pod { + let first = records + .iter() + .filter(|r| { + r.stream == Stream::Durable + && &r.agent == agent + && r.outcome == Outcome::Confirmed + }) + .filter_map(|r| r.completed_at) + .filter(|at| *at >= healed) + .min(); + match first { + Some(at) => { + let gap = (at - healed).num_milliseconds().max(0) as u64; + if gap > recovery_budget.as_millis() as u64 { + over_budget += 1; + } + gaps.push(gap); + } + None => agents_never_recovered.push(agent.clone()), + } + } + } + + let mut report = ReachabilityReport { + isolated_pod: split.pod_address.clone(), + isolated_agents: split.on_pod.len(), + reachable_agents: split.elsewhere.len(), + isolated_ceiling_percent, + control_floor_percent, + recovery_budget_ms: recovery_budget.as_millis() as u64, + cells, + recovery: LatencyStats::from_durations(gaps), + recovery_over_budget: over_budget, + agents_never_recovered, + records_outside_the_split, + findings: Vec::new(), + findings_omitted: 0, + }; + report.judge(); + report + } + + /// A cell by group and window, if the run produced one. + pub fn cell(&self, group: Group, window: Window) -> Option<&ThroughputCell> { + self.cells + .iter() + .find(|c| c.group == group && c.window == window) + } + + fn judge(&mut self) { + let mut findings: Vec = Vec::new(); + + // Did the fault land? Asked first, because a "no" makes the rest of the + // report a description of a cluster nothing happened to. + if let Some(share) = self + .cell(Group::OnPod, Window::DuringFault) + .and_then(|c| c.share_of_baseline_percent) + && share > self.isolated_ceiling_percent + { + findings.push(ReachabilityFinding { + violation: ReachabilityViolation::PartitionNotObserved, + agent: None, + detail: format!( + "the {} agents on the isolated executor kept {share:.1}% of their baseline \ + throughput during the fault, above the {:.0}% ceiling: the partition did \ + not cut worker-service off from {}. Nothing else in this report describes \ + a disturbed cluster.", + self.isolated_agents, self.isolated_ceiling_percent, self.isolated_pod + ), + }); + } + + // What did it cost the half it was not aimed at? + if let Some(share) = self + .cell(Group::Elsewhere, Window::DuringFault) + .and_then(|c| c.share_of_baseline_percent) + && share < self.control_floor_percent + { + findings.push(ReachabilityFinding { + violation: ReachabilityViolation::ControlDegraded, + agent: None, + detail: format!( + "the {} agents on the reachable executor kept only {share:.1}% of their \ + baseline throughput while {} was cut off, below the {:.0}% floor. They \ + were never partitioned from anything, so this is what serving an \ + unreachable executor cost the rest of the cluster.", + self.reachable_agents, self.isolated_pod, self.control_floor_percent + ), + }); + } + + for agent in &self.agents_never_recovered { + findings.push(ReachabilityFinding { + violation: ReachabilityViolation::NeverRecovered, + agent: Some(agent.clone()), + detail: format!( + "{agent} confirmed no operation at all after the link to {} was restored", + self.isolated_pod + ), + }); + } + + self.findings_omitted = findings.len().saturating_sub(MAX_FINDINGS) as u64; + findings.truncate(MAX_FINDINGS); + self.findings = findings; + } + + /// The lines that need a human. + pub fn attention_lines(&self) -> Vec { + let mut lines: Vec = self + .findings + .iter() + .map(|f| format!("S3 {}: {}", f.violation.as_str(), f.detail)) + .collect(); + + if self.findings_omitted > 0 { + lines.push(format!( + "S3: {} further reachability finding(s) were dropped from the report", + self.findings_omitted + )); + } + if self.records_outside_the_split > 0 { + lines.push(format!( + "S3: {} operation(s) ran against agents the ownership split never saw, so they \ + are in no group and in no cell — the split and the workload disagree about \ + who was driven", + self.records_outside_the_split + )); + } + // Over budget is not a finding. How long a partition heal may take + // before an agent is served again is a judgement, and the driver is not + // the one to make it. + if self.recovery_over_budget > 0 { + lines.push(format!( + "S3: {} of {} isolated agents took longer than the {}ms recovery budget to \ + confirm anything after the heal (p99 {}ms, worst {}ms)", + self.recovery_over_budget, + self.isolated_agents, + self.recovery_budget_ms, + self.recovery.p99_ms, + self.recovery.max_ms + )); + } + lines + } + + /// The lines a reader needs in order to interpret the run, which are not + /// themselves problems. + pub fn note_lines(&self) -> Vec { + let mut lines = vec![format!( + "S3: {} agents on the isolated executor {}, {} elsewhere", + self.isolated_agents, self.isolated_pod, self.reachable_agents + )]; + + for group in [Group::OnPod, Group::Elsewhere] { + for window in [Window::BeforeFault, Window::DuringFault, Window::AfterFault] { + if let Some(cell) = self.cell(group, window) { + lines.push(format!( + "S3 {} {}: {:.2} confirmed/s{}, {} submitted, {} indeterminate, {} \ + attempt(s) timed out, {} of {} agents active", + group.as_str(), + window.as_str(), + cell.confirmed_per_sec, + cell.share_of_baseline_percent + .map(|s| format!(" ({s:.1}% of baseline)")) + .unwrap_or_default(), + cell.submitted, + cell.indeterminate, + cell.attempts_timed_out, + cell.agents_active, + if group == Group::OnPod { + self.isolated_agents + } else { + self.reachable_agents + }, + )); + } + } + } + + if self.recovery.count > 0 { + lines.push(format!( + "S3: isolated agents were served again p50 {}ms / p99 {}ms / worst {}ms after \ + the heal", + self.recovery.p50_ms, self.recovery.p99_ms, self.recovery.max_ms + )); + } + lines + } +} + +/// How long a window lasted, in seconds. +/// +/// The fault's own windows come from the workflow's timestamps, which is what +/// makes them comparable across runs. The two open-ended ones are bounded by +/// the workload instead: the baseline starts at the first operation offered, +/// and recovery ends at the last one that came back. +fn window_secs( + window: Window, + fault: Option, + first_submitted: Option>, + last_completed: Option>, +) -> f64 { + let seconds = |from: DateTime, to: DateTime| { + (to - from).num_milliseconds().max(0) as f64 / 1000.0 + }; + match (window, fault) { + (Window::BeforeFault, Some(w)) => first_submitted + .map(|first| seconds(first, w.injected_at)) + .unwrap_or(0.0), + (Window::DuringFault, Some(w)) => match (w.recovered_at, last_completed) { + (Some(recovered), _) => seconds(w.injected_at, recovered), + // A run that never saw the heal: the fault ran to whatever the last + // operation saw, which is the most that can be claimed. + (None, Some(last)) => seconds(w.injected_at, last), + (None, None) => 0.0, + }, + ( + Window::AfterFault, + Some(FaultWindow { + recovered_at: Some(recovered), + .. + }), + ) => last_completed + .map(|last| seconds(recovered, last)) + .unwrap_or(0.0), + _ => 0.0, + } +} + +fn round2(value: f64) -> f64 { + (value * 100.0).round() / 100.0 +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::chaos::errors::ErrorClass; + use crate::chaos::history::{AttemptRecord, Phase}; + use chrono::TimeDelta; + use std::time::Duration; + use test_r::test; + + const ISOLATED: &str = "chaos-s3-durable-0000"; + const CONTROL: &str = "chaos-s3-durable-0001"; + + fn t0() -> DateTime { + DateTime::parse_from_rfc3339("2026-08-24T12:00:00Z") + .unwrap() + .with_timezone(&Utc) + } + + fn split() -> PodSplit { + PodSplit { + pod_address: "10.0.1.1:9000".to_string(), + pod_ip: "10.0.1.1".to_string(), + on_pod: vec![ISOLATED.to_string()], + elsewhere: vec![CONTROL.to_string()], + targets_per_pod: BTreeMap::new(), + number_of_shards: 1024, + } + } + + fn fault() -> FaultWindow { + FaultWindow { + injected_at: t0(), + recovered_at: Some(t0() + TimeDelta::seconds(180)), + } + } + + /// One operation, submitted `offset` seconds from the moment the partition + /// was injected. Negative offsets are the baseline. + fn op(agent: &str, offset_secs: i64, outcome: Outcome) -> OperationRecord { + let submitted_at = t0() + TimeDelta::seconds(offset_secs); + OperationRecord { + op_id: 0, + stream: Stream::Durable, + phase: Phase::Baseline, + agent: agent.to_string(), + method: "increment".to_string(), + idempotency_key: format!("{agent}-{offset_secs}"), + submitted_at, + completed_at: Some(submitted_at + TimeDelta::milliseconds(20)), + attempts: 1, + outcome, + duration_ms: 20, + returned_value: Some(1), + first_attempt_value: None, + error: None, + error_class: None, + attempt_log: vec![AttemptRecord { + attempt: 1, + started_at: submitted_at, + duration_ms: 20, + returned_value: Some(1), + succeeded: outcome == Outcome::Confirmed, + error_class: None, + error: None, + }], + } + } + + /// An operation that hung until the client gave up, twice: the shape every + /// isolated invocation takes while the link is cut. + fn stalled(agent: &str, offset_secs: i64) -> OperationRecord { + let mut record = op(agent, offset_secs, Outcome::Indeterminate); + record.duration_ms = 245_000; + record.completed_at = Some(record.submitted_at + TimeDelta::seconds(245)); + record.returned_value = None; + record.attempts = 2; + record.error_class = Some(ErrorClass::Transport); + record.attempt_log = (1..=2) + .map(|attempt| AttemptRecord { + attempt, + started_at: record.submitted_at, + duration_ms: 120_000, + returned_value: None, + succeeded: false, + error_class: Some(ErrorClass::Transport), + error: Some("attempt timed out after 120s".to_string()), + }) + .collect(); + record + } + + /// A baseline both groups share, then a fault the isolated group is cut off + /// by and the control group sails through. `during_isolated` is how many + /// operations the isolated group still managed. + fn history(during_isolated: usize, during_control: usize) -> Vec { + let mut records = Vec::new(); + // 300s of baseline, one operation per second per agent. + for second in 1..=300 { + for agent in [ISOLATED, CONTROL] { + records.push(op(agent, -second, Outcome::Confirmed)); + } + } + // 180s of fault. + for i in 0..during_isolated { + records.push(op(ISOLATED, i as i64, Outcome::Confirmed)); + } + for i in 0..during_control { + records.push(op(CONTROL, i as i64, Outcome::Confirmed)); + } + // 240s of recovery, both groups back to cadence. + for second in 181..=420 { + for agent in [ISOLATED, CONTROL] { + records.push(op(agent, second, Outcome::Confirmed)); + } + } + records + } + + fn build(records: &[OperationRecord]) -> ReachabilityReport { + ReachabilityReport::build( + records, + &split(), + Some(fault()), + 25.0, + 75.0, + Duration::from_secs(60), + ) + } + + /// The healthy shape: the cut half stops, the other half does not, and the + /// report says so without raising anything. + #[test] + fn a_partition_that_lands_and_costs_the_control_group_nothing_has_no_findings() { + let report = build(&history(2, 180)); + + assert!( + report.findings.is_empty(), + "expected no findings, got {:?}", + report.findings + ); + let isolated = report.cell(Group::OnPod, Window::DuringFault).unwrap(); + assert!( + isolated.share_of_baseline_percent.unwrap() < 25.0, + "the isolated group should have collapsed, got {isolated:?}" + ); + let control = report.cell(Group::Elsewhere, Window::DuringFault).unwrap(); + assert!( + control.share_of_baseline_percent.unwrap() >= 75.0, + "the control group should have held, got {control:?}" + ); + } + + /// The inconclusive case, and the most important one in this module: a + /// partition that never took hold produces a report full of healthy numbers, + /// and it has to read as "this run tested nothing" rather than as a pass. + #[test] + fn an_isolated_group_that_kept_working_says_the_partition_never_landed() { + let report = build(&history(180, 180)); + + assert_eq!( + report + .findings + .iter() + .map(|f| f.violation) + .collect::>(), + vec![ReachabilityViolation::PartitionNotObserved] + ); + assert!( + report + .attention_lines() + .iter() + .any(|line| line.contains("Nothing else in this report")), + "the operator has to be told the rest of the report is meaningless" + ); + } + + /// The finding S3 exists to hunt: agents that were never partitioned from + /// anything losing throughput because their worker-service was busy waiting + /// on a pod they do not use. + #[test] + fn a_control_group_that_degraded_alongside_the_isolated_one_is_a_finding() { + // Half the control group's baseline rate, well under the 75% floor. + let report = build(&history(2, 90)); + + assert!( + report + .findings + .iter() + .any(|f| f.violation == ReachabilityViolation::ControlDegraded), + "expected collateral damage to be reported, got {:?}", + report.findings + ); + } + + /// Throughput, not raw counts. The baseline is 300s long and the fault + /// window 180s, so comparing totals would call an untouched group degraded + /// by 40% on window length alone. + #[test] + fn a_group_that_held_its_rate_is_not_penalised_for_a_shorter_fault_window() { + // 180 operations across a 180s window is exactly the baseline rate. + let report = build(&history(2, 180)); + let control = report.cell(Group::Elsewhere, Window::DuringFault).unwrap(); + + assert_eq!(control.submitted, 180); + assert!( + (control.share_of_baseline_percent.unwrap() - 100.0).abs() < 1.0, + "an unchanged rate should read as ~100% of baseline, got {:?}", + control.share_of_baseline_percent + ); + } + + /// The pending-then-timeout behaviour the ticket asks to see in the history. + #[test] + fn attempts_that_hit_the_client_timeout_are_counted_per_cell() { + let mut records = history(0, 180); + records.push(stalled(ISOLATED, 10)); + let report = build(&records); + + let isolated = report.cell(Group::OnPod, Window::DuringFault).unwrap(); + assert_eq!(isolated.attempts_timed_out, 2); + assert_eq!(isolated.indeterminate, 1); + assert_eq!(isolated.confirmed, 0); + } + + /// Recovery is measured from the heal, not from when the operation was + /// submitted: an agent whose call was already in flight when the link came + /// back was served promptly, and the number has to say so. + #[test] + fn recovery_is_measured_from_the_heal() { + let mut records = history(0, 180); + // Submitted mid-fault, answered five seconds after the heal. + let mut late = op(ISOLATED, 100, Outcome::Confirmed); + late.completed_at = Some(t0() + TimeDelta::seconds(185)); + records.retain(|r| !(r.agent == ISOLATED && r.submitted_at > t0())); + records.push(late); + + let report = build(&records); + assert_eq!(report.recovery.count, 1); + assert_eq!(report.recovery.max_ms, 5_000); + assert!(report.agents_never_recovered.is_empty()); + } + + /// An isolated agent that never answered again is the strongest finding this + /// report can make: the link is back and nothing else is wrong. + #[test] + fn an_isolated_agent_that_never_came_back_is_a_finding() { + let records: Vec = history(0, 180) + .into_iter() + .filter(|r| !(r.agent == ISOLATED && r.submitted_at >= t0())) + .collect(); + + let report = build(&records); + assert_eq!(report.agents_never_recovered, vec![ISOLATED.to_string()]); + assert!( + report + .findings + .iter() + .any(|f| f.violation == ReachabilityViolation::NeverRecovered + && f.agent.as_deref() == Some(ISOLATED)) + ); + } + + /// Without the fault window there is no before-and-during to compare, so + /// every threshold in this report is undefined. It must count what it saw + /// and claim nothing — a verdict here would be invented. + #[test] + fn a_run_that_never_learned_when_the_fault_was_reports_counts_but_no_verdict() { + let report = ReachabilityReport::build( + &history(180, 180), + &split(), + None, + 25.0, + 75.0, + Duration::from_secs(60), + ); + + assert!(report.findings.is_empty()); + assert!(report.cells.iter().all(|c| c.window == Window::Unknown)); + assert!(report.cells.iter().all(|c| c.confirmed_per_sec == 0.0)); + assert_eq!(report.recovery.count, 0); + // And nothing is silently blamed on the agents themselves. + assert!(report.agents_never_recovered.is_empty()); + } + + /// An agent the selection never saw is reported, not folded into a group. + /// It means the split and the workload disagree about who was driven, and + /// silently counting it as a control would corrupt the one comparison the + /// whole scenario rests on. + #[test] + fn operations_against_an_unknown_agent_are_reported_rather_than_grouped() { + let mut records = history(2, 180); + records.push(op("chaos-s3-durable-9999", -10, Outcome::Confirmed)); + + let report = build(&records); + assert_eq!(report.records_outside_the_split, 1); + assert!( + report + .attention_lines() + .iter() + .any(|line| line.contains("the ownership split never saw")) + ); + } + + /// Two hundred agents that all failed to recover is one fact, not two + /// hundred, and an artifact carrying all of them is unreadable. + #[test] + fn findings_beyond_the_cap_are_counted_rather_than_carried() { + let many: Vec = (0..MAX_FINDINGS + 10) + .map(|i| format!("chaos-s3-durable-{i:04}")) + .collect(); + let mut split = split(); + split.on_pod = many.clone(); + + let report = ReachabilityReport::build( + &[], + &split, + Some(fault()), + 25.0, + 75.0, + Duration::from_secs(60), + ); + + assert_eq!(report.agents_never_recovered.len(), MAX_FINDINGS + 10); + assert_eq!(report.findings.len(), MAX_FINDINGS); + assert_eq!(report.findings_omitted, 10); + } + + /// Over budget is context, not a finding: how long worker-service's retry + /// loop may take to notice a healed link is a judgement, and the driver is + /// not the one to make it. + #[test] + fn a_slow_recovery_is_an_attention_line_rather_than_a_finding() { + let mut records = history(0, 180); + let mut slow = op(ISOLATED, 100, Outcome::Confirmed); + slow.completed_at = Some(t0() + TimeDelta::seconds(300)); + records.retain(|r| !(r.agent == ISOLATED && r.submitted_at > t0())); + records.push(slow); + + let report = build(&records); + assert_eq!(report.recovery_over_budget, 1); + assert!(report.findings.is_empty()); + assert!( + report + .attention_lines() + .iter() + .any(|line| line.contains("recovery budget")) + ); + } +} diff --git a/integration-tests/src/chaos/result.rs b/integration-tests/src/chaos/result.rs index cc61bfc9fa..4b4994e7c0 100644 --- a/integration-tests/src/chaos/result.rs +++ b/integration-tests/src/chaos/result.rs @@ -29,7 +29,8 @@ use crate::chaos::scheduled::ScheduledSelection; use crate::chaos::split::PodSplit; use crate::chaos::summary::{ChaosSummary, TerminationReason}; use crate::chaos::{ - FaultConfig, PinnedConfig, PromiseConfig, RetryPolicy, ScheduledConfig, WorkloadConfig, + FaultConfig, IsolationConfig, PinnedConfig, PromiseConfig, RetryPolicy, ScheduledConfig, + WorkloadConfig, }; use chrono::{DateTime, Utc}; use golem_test_framework::benchmark::RunMetadata; @@ -152,6 +153,15 @@ pub struct ChaosResult { /// population from the control group. #[serde(default, skip_serializing_if = "Option::is_none")] pub promise_selection: Option, + /// The reachability workload the run was configured with, if any. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub isolation: Option, + /// How the agents divided around the executor the partition cut off. + /// Present only for S3. Load-bearing for the same reason as + /// `promiseSelection`, and for one more: S3's whole verdict is a comparison + /// between the two groups, so a report without this cannot be re-checked. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub isolation_selection: Option, pub retry_policy: RetryPolicy, pub scope: RunScope, pub summary: ChaosSummary, @@ -223,6 +233,8 @@ mod tests { scheduled_selection: None, promise: None, promise_selection: None, + isolation: None, + isolation_selection: None, retry_policy: RetryPolicy::default(), scope: RunScope { environment_id: "env-1".to_string(), @@ -323,6 +335,79 @@ mod tests { assert!(parsed.promise_selection.is_some()); } + /// The S3 shape. Same contract as the S10 and S11 tests, for the same + /// reason: `ci-scripts/chaos-investigation-report.py` in golem-cloud reads + /// these by name, and the two repositories cannot be changed atomically. + #[test] + fn an_s3_result_carries_the_reachability_fields_the_investigation_report_reads() { + use crate::chaos::reachability::ReachabilityReport; + use crate::chaos::split::{FaultWindow, PodSplit}; + + let now = Utc::now(); + let split = PodSplit { + pod_address: "10.0.1.1:9000".to_string(), + pod_ip: "10.0.1.1".to_string(), + on_pod: vec!["chaos-s3-durable-0000".to_string()], + elsewhere: vec!["chaos-s3-durable-0001".to_string()], + targets_per_pod: std::collections::BTreeMap::new(), + number_of_shards: 1024, + }; + + let mut result = sample_result(TerminationReason::Completed); + result.scenario_code = "S3".to_string(); + result.isolation = Some(crate::chaos::IsolationConfig { + agents: 200, + interval_millis: 1000, + isolated_ceiling_percent: 25.0, + control_floor_percent: 75.0, + recovery_budget_secs: 60, + }); + result.isolation_selection = Some(split.clone()); + result.summary = ChaosSummary::build(&[], Vec::new(), Vec::new(), Some(now)) + .with_reachability(ReachabilityReport::build( + &[], + &split, + Some(FaultWindow { + injected_at: now, + recovered_at: Some(now + chrono::Duration::seconds(180)), + }), + 25.0, + 75.0, + std::time::Duration::from_secs(60), + )); + + let json = serde_json::to_value(&result).unwrap(); + let reachability = &json["summary"]["reachability"]; + for key in [ + "isolatedPod", + "isolatedAgents", + "reachableAgents", + "isolatedCeilingPercent", + "controlFloorPercent", + "recoveryBudgetMs", + "cells", + "recovery", + "recoveryOverBudget", + "agentsNeverRecovered", + "recordsOutsideTheSplit", + "findings", + "findingsOmitted", + ] { + assert!( + !reachability[key].is_null(), + "summary.reachability.{key} is what the investigation report reads" + ); + } + assert_eq!(json["isolation"]["controlFloorPercent"], 75.0); + assert_eq!(json["isolationSelection"]["podIp"], "10.0.1.1"); + + // And it still round-trips, so an archived S3 result stays readable. + let parsed: ChaosResult = serde_json::from_str(&json.to_string()).unwrap(); + assert_eq!(parsed.scenario_code, "S3"); + assert!(parsed.summary.reachability.is_some()); + assert!(parsed.isolation_selection.is_some()); + } + /// The S10 shape, whose report is read by a script in another repository. /// /// `ci-scripts/chaos-investigation-report.py` in golem-cloud renders these @@ -658,6 +743,8 @@ mod sample_artifact { scheduled_selection: None, promise: None, promise_selection: None, + isolation: None, + isolation_selection: None, retry_policy: RetryPolicy::default(), scope: RunScope { environment_id: "0192f000-0000-7000-8000-000000000001".to_string(), @@ -872,6 +959,8 @@ mod sample_artifact { }), promise: None, promise_selection: None, + isolation: None, + isolation_selection: None, retry_policy: RetryPolicy::default(), scope: RunScope { environment_id: "0192f000-0000-7000-8000-000000000001".to_string(), diff --git a/integration-tests/src/chaos/scenarios/mod.rs b/integration-tests/src/chaos/scenarios/mod.rs index 4bdad51533..6c8a25dd83 100644 --- a/integration-tests/src/chaos/scenarios/mod.rs +++ b/integration-tests/src/chaos/scenarios/mod.rs @@ -30,6 +30,7 @@ pub mod s10; pub mod s11; pub mod s12; pub mod s13; +pub mod s3; pub mod s5; pub mod s8; @@ -85,6 +86,9 @@ pub struct ScenarioOutcome { /// Present only for S11, which divides its waiters around the executor the /// fault was aimed at the same way S10 divides its targets. pub promise_selection: Option, + /// Present only for S3, which divides its agents around the executor the + /// partition cuts off rather than around one that dies. + pub isolation_selection: Option, } /// Assembles the archived result. @@ -112,6 +116,8 @@ pub fn build_result(config: &ScenarioConfig, outcome: ScenarioOutcome) -> ChaosR scheduled_selection: outcome.scheduled_selection, promise: config.promise.clone(), promise_selection: outcome.promise_selection, + isolation: config.isolation.clone(), + isolation_selection: outcome.isolation_selection, retry_policy: config.retry_policy.clone(), scope: outcome.scope, summary: outcome.summary, diff --git a/integration-tests/src/chaos/scenarios/s1.rs b/integration-tests/src/chaos/scenarios/s1.rs index 798f52826b..e6ffd7fef8 100644 --- a/integration-tests/src/chaos/scenarios/s1.rs +++ b/integration-tests/src/chaos/scenarios/s1.rs @@ -237,6 +237,7 @@ pub async fn run( pinned_selection: None, scheduled_selection: None, promise_selection: None, + isolation_selection: None, }, ); write_outputs(&result, &history, outputs)?; diff --git a/integration-tests/src/chaos/scenarios/s10.rs b/integration-tests/src/chaos/scenarios/s10.rs index 5352ec0bd1..2831ac67a0 100644 --- a/integration-tests/src/chaos/scenarios/s10.rs +++ b/integration-tests/src/chaos/scenarios/s10.rs @@ -223,6 +223,7 @@ pub async fn run( pinned_selection: None, scheduled_selection: selection.clone(), promise_selection: None, + isolation_selection: None, }, ); write_outputs(&result, &history, outputs)?; diff --git a/integration-tests/src/chaos/scenarios/s11.rs b/integration-tests/src/chaos/scenarios/s11.rs index 0668a0fe15..641f8927eb 100644 --- a/integration-tests/src/chaos/scenarios/s11.rs +++ b/integration-tests/src/chaos/scenarios/s11.rs @@ -197,6 +197,7 @@ pub async fn run( pinned_selection: None, scheduled_selection: None, promise_selection: selection.clone(), + isolation_selection: None, }, ); write_outputs(&result, &history, outputs)?; diff --git a/integration-tests/src/chaos/scenarios/s12.rs b/integration-tests/src/chaos/scenarios/s12.rs index e2865f5357..3dd67fe1e7 100644 --- a/integration-tests/src/chaos/scenarios/s12.rs +++ b/integration-tests/src/chaos/scenarios/s12.rs @@ -134,6 +134,7 @@ pub async fn run( pinned_selection: None, scheduled_selection: None, promise_selection: None, + isolation_selection: None, }, ); write_outputs(&result, &history, outputs)?; diff --git a/integration-tests/src/chaos/scenarios/s13.rs b/integration-tests/src/chaos/scenarios/s13.rs index ab796a5753..f749e52c3c 100644 --- a/integration-tests/src/chaos/scenarios/s13.rs +++ b/integration-tests/src/chaos/scenarios/s13.rs @@ -205,6 +205,7 @@ pub async fn run( pinned_selection: None, scheduled_selection: None, promise_selection: None, + isolation_selection: None, }, ); write_outputs(&result, &history, outputs)?; diff --git a/integration-tests/src/chaos/scenarios/s3.rs b/integration-tests/src/chaos/scenarios/s3.rs new file mode 100644 index 0000000000..843a62fa3a --- /dev/null +++ b/integration-tests/src/chaos/scenarios/s3.rs @@ -0,0 +1,453 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! S3 — executor / worker-service network partition (GOL-370). +//! +//! S1 cuts the shard-manager off from an executor, and the cluster reacts: the +//! shard-manager stops hearing health checks and moves the shards. S3 cuts the +//! *other* link, and the cluster does not react at all — which is the point. +//! +//! The executor keeps talking to the shard-manager, so it keeps its shards and +//! stays in the routing table. worker-service therefore keeps being told, quite +//! correctly, that this executor owns those agents, and keeps trying to reach a +//! pod it cannot reach. There is no route around it, because as far as the +//! platform is concerned nothing is wrong. Every other scenario in the suite +//! ends with the platform recovering; this one asks what it does when there is +//! nothing to recover *from*, only something to wait out. +//! +//! ### What worker-service actually does +//! +//! Read out of `golem-worker-service/src/service/worker/routing_logic.rs`. A +//! call to an unreachable pod fails the 10s connect timeout, which is retriable, +//! so `call_worker_executor` invalidates the routing table and tries again — +//! forever. `get_delay` stops extending the backoff after five attempts and the +//! loop then settles at the 3s ceiling rather than giving up. The freshly +//! fetched table names the same unreachable pod every time, because the +//! shard-manager still believes in it. +//! +//! Two consequences, and S3 measures both: +//! +//! * **For the agents on that executor**, an invocation does not fail fast. It +//! hangs, until the caller's own attempt timeout ends it. That is the +//! "acceptance degradation with pending and timeout behaviour" the ticket +//! asks to see, and it is visible in the history as attempts that timed out +//! rather than as refusals. +//! * **For everyone else**, the routing table is one process-wide cache entry +//! per worker-service replica. Every stalled caller invalidating it costs +//! *every* caller a shard-manager round trip. `invalidation_min_delay` bounds +//! that to twice a second, so the cost should be small — but "should be" is +//! what the control group is there to check. +//! +//! ### The choreography +//! +//! 1. **Warm up** every agent, so the partition lands on a live population. +//! 2. **Select** the executor owning the largest share of them, exactly as S10 +//! and S11 pick their target, and name it for the workflow. +//! 3. **Baseline** — one emitter per agent, one operation each at a time. +//! 4. **Fault** — keep driving. Sample the routing table part-way in: S3's +//! premise is that the assignment does *not* move, and a run where it did is +//! a different experiment that has to be read differently. +//! 5. **Heal**, then keep driving so the isolated agents visibly come back. +//! 6. **Read back and probe** — the same completion and exactly-once oracles +//! every other scenario ends with. + +use crate::chaos::history::{OperationHistory, OperationRecord, Outcome, Phase, Stream}; +use crate::chaos::ownership::OwnershipSample; +use crate::chaos::prep::ChaosPrepManifest; +use crate::chaos::probe; +use crate::chaos::reachability::ReachabilityReport; +use crate::chaos::result::{ChaosResult, PhaseWindow, Phases, RunScope}; +use crate::chaos::scenarios::{ + OutputPaths, ReadKind, ScenarioOutcome, WARMUP_SETTLE, build_result, exactly_once_termination, + read_back_agents, read_counters, sample_ownership, signal_termination, snapshot_routing, + wait_for_settled_routing, write_outputs, +}; +use crate::chaos::signal::{BaselineReady, FaultSignals, FaultTarget}; +use crate::chaos::split::{self, FaultWindow, PodSplit}; +use crate::chaos::steady; +use crate::chaos::summary::{ + AgentReadback, ChaosSummary, ExactlyOnceReport, Note, TerminationReason, +}; +use crate::chaos::workload::{PhaseMarker, WorkloadContext}; +use crate::chaos::{ScenarioCode, ScenarioConfig}; +use chrono::Utc; +use golem_test_framework::config::BenchmarkTestDependencies; +use golem_test_framework::dsl::TestDsl; +use std::time::Duration; +use tracing::{info, warn}; + +/// How long to wait after stopping the workload before reading durable state. +/// Same reasoning as every other scenario: an increment still in flight has to +/// land, and reading early reports a mismatch that says nothing. +const SETTLE_BEFORE_READBACK: Duration = Duration::from_secs(30); + +/// How far into the fault window the assignment sample is taken, as a fraction +/// of it, and the ceiling on that. +/// +/// Unlike S1 this sample is checking that nothing moved, so it is taken late +/// rather than early: a table that still looks untouched two minutes in is a +/// much stronger statement than one that looks untouched immediately. +const DURING_FAULT_SAMPLE_FRACTION: f64 = 0.75; +const DURING_FAULT_SAMPLE_CAP: Duration = Duration::from_secs(150); + +/// Runs S3 end to end. +pub async fn run( + config: &ScenarioConfig, + manifest: &ChaosPrepManifest, + deps: &BenchmarkTestDependencies, + signals: &FaultSignals, + outputs: &OutputPaths, +) -> anyhow::Result { + let started_at = Utc::now(); + let isolation = config.require_isolation()?; + let history = OperationHistory::new(ScenarioCode::S3.as_str()); + let key_prefix = crate::chaos::scenario_key_prefix(ScenarioCode::S3); + + let user = manifest.user_context(deps); + let counters = user + .get_latest_component_revision(&manifest.counters_component_id) + .await?; + let promise = user + .get_latest_component_revision(&manifest.promise_component_id) + .await?; + + let ctx = WorkloadContext { + user, + counters, + promise, + history: history.clone(), + retry: config.retry_policy.clone(), + phase: PhaseMarker::new(Phase::Baseline), + key_prefix: key_prefix.clone(), + }; + + let scope = RunScope { + environment_id: manifest.environment_id.0.to_string(), + component_ids: vec![manifest.counters_component_id.0.to_string()], + agent_id_prefix: key_prefix.clone(), + idempotency_key_prefix: format!("{key_prefix}-"), + }; + + let agents = steady::agent_names(&ctx, isolation.agents); + + let mut phases = Phases::default(); + let mut routing_snapshots = Vec::new(); + let mut ownership: Vec = Vec::new(); + let mut fault_injected_at = None; + let mut fault_recovered_at = None; + let mut fault_id = None; + let mut fault_target_observed = None; + let mut selection: Option = None; + let mut attention_extra: Vec = Vec::new(); + + macro_rules! finish { + ($reason:expr, $records:expr, $readback:expr, $exactly_once:expr, $reachability:expr) => {{ + let mut summary = ChaosSummary::build( + $records, + $readback, + routing_snapshots.clone(), + fault_injected_at, + ) + .with_ownership(ownership.clone()); + summary.absorb(attention_extra.clone()); + if let Some(report) = $exactly_once { + summary = summary.with_exactly_once(report); + } + if let Some(report) = $reachability { + summary = summary.with_reachability(report); + } + let result = build_result( + config, + ScenarioOutcome { + started_at, + phases: phases.clone(), + fault_injected_at, + fault_recovered_at, + fault_id: fault_id.clone(), + fault_target_observed: fault_target_observed.clone(), + scope: scope.clone(), + summary, + termination_reason: $reason, + pinned_selection: None, + scheduled_selection: None, + promise_selection: None, + isolation_selection: selection.clone(), + }, + ); + write_outputs(&result, &history, outputs)?; + return Ok(result); + }}; + } + + // ── Warm-up ───────────────────────────────────────────────────────────── + // + // Construct every agent before measuring, for the reason S1 spells out: a + // cold start looks exactly like a stall from outside, and this scenario's + // entire signal is a comparison of throughput before and during the fault. + // A baseline that was still cold-starting would understate itself and make + // the fault look milder than it was. + // + // Reads, not increments. An increment here would be invisible to the + // operation history and would leave every read-back off by one. + routing_snapshots.push(snapshot_routing(deps, "before-warmup").await); + attention_extra.push(wait_for_settled_routing(deps, &mut routing_snapshots).await); + + info!("S3: warming up {} counter agents", agents.len()); + let warm: Vec<(Stream, String, ReadKind)> = agents + .iter() + .map(|agent| (Stream::Durable, agent.clone(), ReadKind::Counter)) + .collect(); + let _ = read_back_agents(&ctx, &[], warm).await; + info!( + "S3: warmed {} agents, settling {WARMUP_SETTLE:?}", + agents.len() + ); + tokio::time::sleep(WARMUP_SETTLE).await; + + // ── Aim ───────────────────────────────────────────────────────────────── + // + // Chaos Mesh's `mode: one` would pick an executor at random, and a + // partition that cut off the executor owning six agents out of two hundred + // would still produce a confident-looking report. The driver names the pod + // instead; the workflow turns the IP into a pod name. + let subject = split::counter_subject(&ctx); + let split = match split::select(subject, deps, &agents).await { + Ok(split) => split, + Err(e) => { + warn!("S3: cannot aim the partition: {e:#}"); + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!("{e:#}"), + }, + &records, + Vec::new(), + None, + None + ); + } + }; + selection = Some(split.clone()); + + // ── Baseline ──────────────────────────────────────────────────────────── + info!( + "S3: baseline phase, running {} emitters for {:?}", + agents.len(), + config.phases.baseline() + ); + phases.baseline = Some(PhaseWindow::started(Utc::now())); + let handle = steady::start(ctx.clone(), isolation.agents, isolation.interval()); + tokio::time::sleep(config.phases.baseline()).await; + routing_snapshots.push(snapshot_routing(deps, "before-fault").await); + ownership.push(sample_ownership(deps, "before-fault", ownership.last(), false).await); + if let Some(window) = phases.baseline.as_mut() { + window.end(Utc::now()); + } + + let baseline_operations = history.confirmed_in_phase(Phase::Baseline); + if baseline_operations == 0 { + warn!("S3: baseline produced no confirmed operations, aborting before injection"); + handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::PlatformUnreachable { + detail: "no operation succeeded during the baseline phase".to_string(), + }, + &records, + Vec::new(), + None, + None + ); + } + + // A rebalance between selection and injection would leave the run naming + // the control group as the affected one and vice versa — a report that is + // not merely wrong but confidently inverted. + if let Err(e) = split::verify_ownership(subject, deps, &split).await { + warn!("S3: ownership drifted between selection and injection: {e:#}"); + handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!("{e:#}"), + }, + &records, + Vec::new(), + None, + None + ); + } + + info!( + "S3: baseline complete ({baseline_operations} confirmed ops), naming {} and signalling \ + readiness", + split.pod_address + ); + signals.write_baseline_ready(&BaselineReady { + scenario_code: ScenarioCode::S3.as_str().to_string(), + ready_at: Utc::now(), + baseline_operations, + fault_target: Some(FaultTarget { + pod_address: split.pod_address.clone(), + pod_ip: split.pod_ip.clone(), + owned_agents: split.on_pod.clone(), + }), + })?; + + // ── Fault ─────────────────────────────────────────────────────────────── + let injected = match signals.await_fault_injected(config.signal_timeout()).await { + Ok(injected) => injected, + Err(e) => { + warn!("S3: no fault-injected signal arrived: {e}"); + handle.stop().await; + let records = history.snapshot(); + finish!(signal_termination(&e), &records, Vec::new(), None, None); + } + }; + info!( + "S3: fault {} ({} on {}) reported active at {}", + injected.fault_id, injected.kind, injected.target, injected.injected_at + ); + fault_injected_at = Some(injected.injected_at); + fault_id = Some(injected.fault_id.clone()); + fault_target_observed = Some(injected.target.clone()); + ctx.phase.set(Phase::Fault); + phases.fault = Some(PhaseWindow::started(injected.injected_at)); + + // Evidence for the premise, not a verdict. The partitioned link is not the + // one the shard-manager uses, so the assignment is expected to be identical + // to the baseline. If it moved, the fault was wider than intended and every + // reading below has a second explanation. + let observe_after = config + .phases + .fault() + .mul_f64(DURING_FAULT_SAMPLE_FRACTION) + .min(DURING_FAULT_SAMPLE_CAP); + info!("S3: sampling assignment {observe_after:?} into the fault window"); + tokio::time::sleep(observe_after).await; + ownership.push(sample_ownership(deps, "during-fault", ownership.last(), false).await); + + let recovered = match signals.await_fault_recovered(config.signal_timeout()).await { + Ok(recovered) => recovered, + Err(e) => { + warn!("S3: no fault-recovered signal arrived: {e}"); + handle.stop().await; + let records = history.snapshot(); + finish!(signal_termination(&e), &records, Vec::new(), None, None); + } + }; + info!( + "S3: partition healed at {} ({})", + recovered.recovered_at, recovered.termination_reason + ); + fault_recovered_at = Some(recovered.recovered_at); + if let Some(window) = phases.fault.as_mut() { + window.end(recovered.recovered_at); + } + + // ── Recovery ──────────────────────────────────────────────────────────── + ctx.phase.set(Phase::Recovery); + phases.recovery = Some(PhaseWindow::started(Utc::now())); + info!( + "S3: recovery phase, running for {:?}", + config.phases.recovery() + ); + tokio::time::sleep(config.phases.recovery()).await; + + handle.stop().await; + if let Some(window) = phases.recovery.as_mut() { + window.end(Utc::now()); + } + routing_snapshots.push(snapshot_routing(deps, "after-recovery").await); + ownership.push(sample_ownership(deps, "after-recovery", ownership.last(), true).await); + + // ── Read-back ─────────────────────────────────────────────────────────── + info!("S3: letting the platform settle for {SETTLE_BEFORE_READBACK:?} before read-back"); + tokio::time::sleep(SETTLE_BEFORE_READBACK).await; + + let records = history.snapshot(); + + let readback = read_back(&ctx, &records, &agents).await; + let before_probe = read_counters(&ctx, &records).await; + let probes = probe::probe_keys(&ctx, &records, Stream::Durable).await; + let after_probe = read_counters(&ctx, &records).await; + + let exactly_once = ExactlyOnceReport::build( + &records, + &probes, + Stream::Durable, + &before_probe, + &after_probe, + ); + info!( + "S3: exactly-once account — {} keys checked, {} with a final result, {} recovered by the \ + probe, {} findings", + exactly_once.keys_checked, + exactly_once.keys_with_final_result, + exactly_once.keys_recovered_by_probe, + exactly_once.findings.len() + ); + + let reachability = ReachabilityReport::build( + &records, + &split, + fault_injected_at.map(|injected_at| FaultWindow { + injected_at, + recovered_at: fault_recovered_at, + }), + isolation.isolated_ceiling_percent, + isolation.control_floor_percent, + isolation.recovery_budget(), + ); + info!( + "S3: reachability account — {} findings, {} isolated agents never recovered", + reachability.findings.len(), + reachability.agents_never_recovered.len() + ); + + // The assertion is the same one S1 makes, and for the same reason: a key + // that executed twice is the only harm visible from outside the cluster. + // Everything the reachability report says is reported for a human to judge, + // because "worker-service waited rather than failing fast" is a design + // question and not a defect the driver is entitled to rule on. + let reason = exactly_once_termination(&exactly_once).unwrap_or_else(|| { + if records.iter().all(|r| r.outcome != Outcome::Confirmed) { + TerminationReason::StreamNeverSucceeded { + stream: Stream::Durable.to_string(), + } + } else { + TerminationReason::Completed + } + }); + + finish!( + reason, + &records, + readback, + Some(exactly_once), + Some(reachability) + ); +} + +async fn read_back( + ctx: &WorkloadContext, + records: &[OperationRecord], + agents: &[String], +) -> Vec { + let targets = agents + .iter() + .map(|agent| (Stream::Durable, agent.clone(), ReadKind::Counter)) + .collect(); + read_back_agents(ctx, records, targets).await +} diff --git a/integration-tests/src/chaos/scenarios/s5.rs b/integration-tests/src/chaos/scenarios/s5.rs index f07775afc1..71eb018393 100644 --- a/integration-tests/src/chaos/scenarios/s5.rs +++ b/integration-tests/src/chaos/scenarios/s5.rs @@ -177,6 +177,7 @@ pub async fn run( pinned_selection: None, scheduled_selection: None, promise_selection: None, + isolation_selection: None, }, ); write_outputs(&result, &history, outputs)?; diff --git a/integration-tests/src/chaos/scenarios/s8.rs b/integration-tests/src/chaos/scenarios/s8.rs index 10636310bd..b9515ad16e 100644 --- a/integration-tests/src/chaos/scenarios/s8.rs +++ b/integration-tests/src/chaos/scenarios/s8.rs @@ -154,6 +154,7 @@ pub async fn run( pinned_selection: selection.clone(), scheduled_selection: None, promise_selection: None, + isolation_selection: None, }, ); write_outputs(&result, &history, outputs)?; diff --git a/integration-tests/src/chaos/split.rs b/integration-tests/src/chaos/split.rs index 4ebe6f565c..a7ec87d8ff 100644 --- a/integration-tests/src/chaos/split.rs +++ b/integration-tests/src/chaos/split.rs @@ -158,7 +158,8 @@ impl PodSplit { } /// Which side of the kill an agent was on. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] pub enum Group { /// Owned by the executor the fault was aimed at. OnPod, @@ -323,6 +324,20 @@ pub fn schedule_subject<'a>(ctx: &'a WorkloadContext) -> Subject<'a> { } } +/// The counters component's durable agents, as S3 aims at them. +/// +/// The same agent type the mixed workload's durable stream drives, because it +/// is the same population: S3's emitters exist to pace it per agent, not to +/// invent a new kind of agent. +pub fn counter_subject<'a>(ctx: &'a WorkloadContext) -> Subject<'a> { + Subject { + scenario: "S3", + component: &ctx.counters, + agent_type: crate::chaos::workload::COUNTER_AGENT, + noun: "counter agents", + } +} + /// The promise component's waiters, as S11 aims at them. pub fn waiter_subject<'a>(ctx: &'a WorkloadContext) -> Subject<'a> { Subject { diff --git a/integration-tests/src/chaos/steady.rs b/integration-tests/src/chaos/steady.rs new file mode 100644 index 0000000000..f2d4a1247c --- /dev/null +++ b/integration-tests/src/chaos/steady.rs @@ -0,0 +1,122 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! One emitter per agent, each holding at most one operation (GOL-370). +//! +//! [`crate::chaos::workload`] drives streams: a shared rate, a shared in-flight +//! budget, and agents picked round-robin out of a pool. That is the right shape +//! when the thing being disturbed is a *stream*, and the wrong one when it is a +//! *place*. +//! +//! S3 cuts one executor off from worker-service. Every agent that executor owns +//! stalls, and those agents are spread evenly through every stream, so a shared +//! budget would be consumed by stalled operations within seconds and the agents +//! on the reachable executor would stop being submitted too. The run would then +//! show the undisturbed half degrading in lockstep with the disturbed one, and +//! the cause would be the driver rather than the platform. The mixed workload +//! already carries a comment about exactly this failure along the stream axis; +//! this module is the same lesson along the ownership axis. +//! +//! So: one task per agent, one operation in flight at a time, and a cadence +//! measured from the end of the previous operation rather than from a shared +//! clock. An agent whose executor is unreachable then contributes nothing and +//! costs nothing, which is what makes the two groups' throughput comparable. + +use crate::chaos::history::Stream; +use crate::chaos::workload::{self, WorkloadContext}; +use std::sync::Arc; +use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; +use std::time::Duration; +use tokio::task::JoinSet; +use tracing::info; + +/// The agents a run of `count` emitters drives, in index order. +/// +/// Delegates to [`WorkloadContext::agent_name`] rather than formatting a name +/// of its own, so the two cannot drift. That matters: the shared read-back and +/// exactly-once machinery both select on [`Stream::Durable`] and on names of +/// exactly that shape, and a second copy of the format here would empty both +/// silently rather than fail. +pub fn agent_names(ctx: &WorkloadContext, count: u32) -> Vec { + (0..count) + .map(|index| ctx.agent_name(Stream::Durable, index)) + .collect() +} + +/// A running steady workload. Dropping the handle does not stop it — call +/// [`SteadyHandle::stop`], so operations in flight record themselves rather than +/// being cancelled. During a partition those are precisely the operations the +/// run exists to describe. +pub struct SteadyHandle { + stop: Arc, + tasks: JoinSet<()>, + submitted: Arc, +} + +impl SteadyHandle { + pub fn submitted(&self) -> u64 { + self.submitted.load(Ordering::Relaxed) + } + + pub async fn stop(mut self) { + self.stop.store(1, Ordering::Relaxed); + while self.tasks.join_next().await.is_some() {} + info!( + "Chaos steady workload stopped after {} operations", + self.submitted() + ); + } +} + +/// Starts one emitter per agent and keeps them running until +/// [`SteadyHandle::stop`]. +pub fn start(ctx: WorkloadContext, agents: u32, interval: Duration) -> SteadyHandle { + let stop = Arc::new(AtomicU8::new(0)); + let submitted = Arc::new(AtomicU64::new(0)); + let mut tasks = JoinSet::new(); + + info!( + "Chaos steady workload starting: {agents} emitters, one operation each, {interval:?} \ + between them" + ); + + for index in 0..agents { + let ctx = ctx.clone(); + let stop = stop.clone(); + let submitted = submitted.clone(); + + tasks.spawn(async move { + let mut seq: u64 = 0; + while stop.load(Ordering::Relaxed) == 0 { + submitted.fetch_add(1, Ordering::Relaxed); + workload::submit_one(&ctx, Stream::Durable, index, seq).await; + seq += 1; + // From the end of the operation, not from a fixed clock. An + // agent that just spent two minutes stalled must not then fire + // a burst of catch-up operations: that would put its recovery + // throughput above its baseline and make the recovery cell + // unreadable. + if stop.load(Ordering::Relaxed) == 0 { + tokio::time::sleep(interval).await; + } + } + }); + } + + SteadyHandle { + stop, + tasks, + submitted, + } +} diff --git a/integration-tests/src/chaos/summary.rs b/integration-tests/src/chaos/summary.rs index 3a47aaf0c5..51d69c3aa6 100644 --- a/integration-tests/src/chaos/summary.rs +++ b/integration-tests/src/chaos/summary.rs @@ -52,6 +52,7 @@ use crate::chaos::fires::ScheduleFireReport; use crate::chaos::history::{Outcome, Phase, Stream}; use crate::chaos::ownership::OwnershipSample; use crate::chaos::probe::KeyProbe; +use crate::chaos::reachability::ReachabilityReport; use crate::chaos::wakeups::WakeupReport; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; @@ -572,6 +573,11 @@ pub struct ChaosSummary { /// not, for the same reason as `scheduleFires`. #[serde(default, skip_serializing_if = "Option::is_none")] pub promise_wakeups: Option, + /// The reachability account, for scenarios that cut one executor off from + /// the tier that routes to it. Absent for scenarios that do not, for the + /// same reason as `scheduleFires`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reachability: Option, /// Shard-ownership samples, in the order they were taken. Empty for /// scenarios that do not sample executor assignments. /// @@ -703,6 +709,7 @@ impl ChaosSummary { exactly_once: None, schedule_fires: None, promise_wakeups: None, + reachability: None, ownership: Vec::new(), attention, notes: Vec::new(), @@ -762,6 +769,20 @@ impl ChaosSummary { self } + /// Attaches the reachability account and hoists everything it wants a human + /// to see into [`Self::attention`]. + /// + /// Same split as [`Self::with_schedule_fires`]. The one line worth calling + /// out is the inconclusive case: a partition that never cut the executor off + /// produces a report full of healthy numbers, and that has to read as + /// "this run tested nothing" rather than as a pass. + pub fn with_reachability(mut self, report: ReachabilityReport) -> Self { + self.attention.extend(report.attention_lines()); + self.notes.extend(report.note_lines()); + self.reachability = Some(report); + self + } + /// Attaches the shard-ownership samples and hoists their findings into /// [`Self::attention`]. /// diff --git a/integration-tests/src/chaos/workload.rs b/integration-tests/src/chaos/workload.rs index 6c704a8f7c..b847a5548c 100644 --- a/integration-tests/src/chaos/workload.rs +++ b/integration-tests/src/chaos/workload.rs @@ -63,7 +63,7 @@ use tokio::task::JoinSet; use tracing::{debug, info, warn}; /// Agent type names exported by the counters component. -const COUNTER_AGENT: &str = "Counter"; +pub(crate) const COUNTER_AGENT: &str = "Counter"; const EPHEMERAL_COUNTER_AGENT: &str = "EphemeralCounter"; pub(crate) const SCHEDULE_EMITTER_AGENT: &str = "ScheduleEmitter"; pub(crate) const SCHEDULE_COUNTER_AGENT: &str = "ScheduleCounter"; From d6f431524e0c689a016b59c8b24d74e76b3bc4aa Mon Sep 17 00:00:00 2001 From: kmatasfp <33095685+kmatas@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:10:34 -0700 Subject: [PATCH 13/40] Name the golem-bench-results bucket instead of saying S3 --- integration-tests/src/benchmarks/all.rs | 2 +- integration-tests/src/benchmarks/density/agent.rs | 3 ++- integration-tests/src/benchmarks/density/prep.rs | 3 ++- integration-tests/src/chaos/mod.rs | 6 ++++-- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/integration-tests/src/benchmarks/all.rs b/integration-tests/src/benchmarks/all.rs index 07b362a81b..22af3d8f3d 100644 --- a/integration-tests/src/benchmarks/all.rs +++ b/integration-tests/src/benchmarks/all.rs @@ -867,7 +867,7 @@ async fn run_density( // Emit the cell result as a single-result suite so the JSON shape // matches cloud-perf (BenchmarkSuiteResultCollection), and the - // buildspec can upload it directly to S3. + // buildspec can upload it directly to golem-bench-results. let mut suite_result = BenchmarkSuiteResult::new(&format!("density-{section}")); suite_result.add(result); if let Some(run_id) = cloud_bench_run_id() { diff --git a/integration-tests/src/benchmarks/density/agent.rs b/integration-tests/src/benchmarks/density/agent.rs index 96dba8f7bc..0b3fcc28fd 100644 --- a/integration-tests/src/benchmarks/density/agent.rs +++ b/integration-tests/src/benchmarks/density/agent.rs @@ -185,7 +185,8 @@ pub struct CellConfig { } impl CellConfig { - /// The cell's name, used in S3 paths and result identifiers. Encodes the + /// The cell's name, used in `golem-bench-results` keys and result identifiers. + /// Encodes the /// full axis set in human-readable terms (no cryptic single letters). pub fn cell_name(&self) -> String { let mut parts = vec![ diff --git a/integration-tests/src/benchmarks/density/prep.rs b/integration-tests/src/benchmarks/density/prep.rs index 209076716e..f5e208b473 100644 --- a/integration-tests/src/benchmarks/density/prep.rs +++ b/integration-tests/src/benchmarks/density/prep.rs @@ -20,7 +20,8 @@ //! //! The buildspec runs prep exactly once at suite start (against a freshly-wiped //! cluster) and writes the resulting [`PrepManifest`] to a file that is -//! uploaded to S3 and passed to every per-cell invocation via +//! uploaded to the `golem-bench-results` bucket and passed to every per-cell +//! invocation via //! `--prep-manifest`. The manifest carries the account token and all component //! IDs so per-cell invocations need no by-name lookup and no re-tokenization — //! this is also the resume mechanism: a resumed run reloads the same manifest. diff --git a/integration-tests/src/chaos/mod.rs b/integration-tests/src/chaos/mod.rs index 77ff1a80ec..68cebd582e 100644 --- a/integration-tests/src/chaos/mod.rs +++ b/integration-tests/src/chaos/mod.rs @@ -18,8 +18,10 @@ //! Cloud-mode Golem while a bounded fault is injected, then reports what //! happened. The suite's shape follows density: the workflow drives one scenario //! per invocation, each scenario is independently selectable through a YAML -//! `enabled` flag, and results are archived to S3 per scenario so an interrupted -//! run resumes rather than restarts. +//! `enabled` flag, and results are archived to the `golem-bench-results` bucket +//! per scenario so an interrupted run resumes rather than restarts. +//! +//! `S3` in this module always means the scenario code, never the bucket. //! //! Two boundaries define this module: //! From 8678730f3a872a68b6b22376cb826f8b576f413f Mon Sep 17 00:00:00 2001 From: kmatasfp <33095685+kmatas@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:30:04 -0700 Subject: [PATCH 14/40] Report the operations a partition caught in flight, and how long a group stayed quiet --- integration-tests/src/chaos/reachability.rs | 404 +++++++++++++++++++- 1 file changed, 394 insertions(+), 10 deletions(-) diff --git a/integration-tests/src/chaos/reachability.rs b/integration-tests/src/chaos/reachability.rs index 2db5770c0a..b40b755494 100644 --- a/integration-tests/src/chaos/reachability.rs +++ b/integration-tests/src/chaos/reachability.rs @@ -122,9 +122,69 @@ pub struct ThroughputCell { /// baseline to compare against. #[serde(default, skip_serializing_if = "Option::is_none")] pub share_of_baseline_percent: Option, + /// How long into this window the group waited before offering anything. + /// + /// The number that stops a small non-zero rate being read as residual + /// service. An emitter holds one operation at a time, so a group whose + /// executor is unreachable offers nothing at all until something finally + /// answers it: a during-fault cell can therefore show a handful of + /// confirmations that all arrived in the last seconds of the window, once + /// the fault was already coming down. `quietMs` next to `windowSecs` says + /// which of the two happened, and it needs no threshold to do it. + /// + /// `None` when the window has no fixed start, or when the group offered + /// nothing in it at all. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub quiet_ms: Option, pub latency: LatencyStats, } +/// The operations the fault landed in the middle of. +/// +/// The population S3 actually disturbed, and the one number a reader wants +/// first. It cannot be read off the cells: these operations were *submitted* +/// before the cut, so every trace of them — their timeouts, their duration — +/// is attributed to the `before-fault` row, which is the last place anyone +/// looks for the damage. +/// +/// The control group's entry is the comparison that makes it mean something, +/// but read it on **duration**, not on count. How many operations a healthy +/// group has in flight at any instant is its duty cycle — operation time over +/// interval — so a group answering in 45ms on a one-second cadence has about +/// one agent in twenty busy. A stalled group accumulates instead: every emitter +/// ends up holding an operation that will not return, so its count climbs to +/// the size of the group. A real run showed 113 of 113 against 3 of 87, and the +/// gap between 47ms and 182 seconds is the finding, not the gap between 3 and +/// 113. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CaughtInFlight { + pub group: Group, + /// Operations submitted before the fault that were still running when it + /// landed. + pub operations: u64, + /// Distinct agents they belonged to. Equal to the group size when every + /// emitter was mid-operation, which is what one-in-flight-per-agent makes + /// the normal case. + pub agents: usize, + pub confirmed: u64, + pub rejected: u64, + pub indeterminate: u64, + /// Submission to final outcome, across every attempt. + pub duration: LatencyStats, + /// Attempts that hit the client's attempt timeout rather than answering. + pub attempts_timed_out: u64, + /// The most attempts any one of them needed. + /// + /// Load-bearing rather than trivia. An operation that stalled and then + /// answered on a later attempt was rescued by the caller's retry, not + /// returned by the platform: with retries off it would have ended + /// indeterminate. That distinction is invisible in the outcome alone. + pub max_attempts: u32, + /// How many were still unresolved when the fault was reported healed. + pub outlived_the_fault: u64, +} + /// The reachability account. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -140,6 +200,10 @@ pub struct ReachabilityReport { pub control_floor_percent: f64, pub recovery_budget_ms: u64, pub cells: Vec, + /// What the fault landed in the middle of, per group. Empty for a run that + /// never learned when the fault was. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub caught_in_flight: Vec, /// Per isolated agent, how long after the link was restored its first /// confirmed operation landed. pub recovery: LatencyStats, @@ -159,6 +223,8 @@ pub struct ReachabilityReport { #[derive(Default)] struct Tally { agents: BTreeSet, + /// Earliest submission in this cell, for [`ThroughputCell::quiet_ms`]. + first_submitted: Option>, submitted: u64, confirmed: u64, rejected: u64, @@ -197,6 +263,10 @@ impl ReachabilityReport { let tally = tallies.entry((group, window)).or_default(); tally.agents.insert(record.agent.clone()); + tally.first_submitted = Some(match tally.first_submitted { + Some(at) if at <= record.submitted_at => at, + _ => record.submitted_at, + }); tally.submitted += 1; match record.outcome { Outcome::Confirmed => { @@ -206,15 +276,7 @@ impl ReachabilityReport { Outcome::Rejected => tally.rejected += 1, Outcome::Indeterminate => tally.indeterminate += 1, } - tally.attempts_timed_out += record - .attempt_log - .iter() - .filter(|a| { - a.error - .as_deref() - .is_some_and(|e| e.contains("attempt timed out")) - }) - .count() as u64; + tally.attempts_timed_out += timed_out(record); first_submitted = Some(match first_submitted { Some(at) if at <= record.submitted_at => at, @@ -243,6 +305,9 @@ impl ReachabilityReport { if *window == Window::BeforeFault { baseline_rate.insert(*group, rate); } + let quiet_ms = window_start(*window, fault, first_submitted) + .zip(tally.first_submitted) + .map(|(start, first)| (first - start).num_milliseconds().max(0) as u64); cells.push(ThroughputCell { group: *group, window: *window, @@ -255,6 +320,7 @@ impl ReachabilityReport { window_secs: round2(secs), confirmed_per_sec: round2(rate), share_of_baseline_percent: None, + quiet_ms, latency: LatencyStats::from_durations(tally.durations.clone()), }); } @@ -300,6 +366,53 @@ impl ReachabilityReport { } } + // ── What the fault landed in the middle of ────────────────────────── + // + // Computed from the whole history rather than from the cells, because + // the cells cannot answer it: these operations were submitted before + // the cut, so every trace of them sits in the `before-fault` row. + let mut caught_in_flight: Vec = Vec::new(); + if let Some(window) = fault { + let mut by_group: BTreeMap> = BTreeMap::new(); + for record in records.iter().filter(|r| r.stream == Stream::Durable) { + let Some(group) = split.group_of(&record.agent) else { + continue; + }; + let still_running = record + .completed_at + .is_none_or(|at| at >= window.injected_at); + if record.submitted_at < window.injected_at && still_running { + by_group.entry(group).or_default().push(record); + } + } + for (group, caught) in by_group { + let agents: BTreeSet<&str> = caught.iter().map(|r| r.agent.as_str()).collect(); + caught_in_flight.push(CaughtInFlight { + group, + operations: caught.len() as u64, + agents: agents.len(), + confirmed: count_of(&caught, Outcome::Confirmed), + rejected: count_of(&caught, Outcome::Rejected), + indeterminate: count_of(&caught, Outcome::Indeterminate), + duration: LatencyStats::from_durations( + caught.iter().map(|r| r.duration_ms).collect(), + ), + attempts_timed_out: caught.iter().map(|r| timed_out(r)).sum(), + max_attempts: caught.iter().map(|r| r.attempts).max().unwrap_or(0), + outlived_the_fault: caught + .iter() + .filter(|r| match (r.completed_at, window.recovered_at) { + // Never finished at all, so it certainly outlived it. + (None, _) => true, + (Some(done), Some(healed)) => done >= healed, + // No heal was ever reported; nothing can be said. + (Some(_), None) => false, + }) + .count() as u64, + }); + } + } + let mut report = ReachabilityReport { isolated_pod: split.pod_address.clone(), isolated_agents: split.on_pod.len(), @@ -308,6 +421,7 @@ impl ReachabilityReport { control_floor_percent, recovery_budget_ms: recovery_budget.as_millis() as u64, cells, + caught_in_flight, recovery: LatencyStats::from_durations(gaps), recovery_over_budget: over_budget, agents_never_recovered, @@ -406,6 +520,24 @@ impl ReachabilityReport { self.records_outside_the_split )); } + // Work the fault caught and the run cannot account for. Not a finding — + // an indeterminate operation is doubt, not damage, and the read-back + // and exactly-once accounts are what resolve it — but the operator has + // to see it next to the clean cells rather than infer it from them. + for caught in &self.caught_in_flight { + let unresolved = caught.indeterminate + caught.rejected; + if unresolved > 0 { + lines.push(format!( + "S3: {unresolved} of the {} operations the cut caught in flight on {} did \ + not confirm ({} indeterminate, {} rejected)", + caught.operations, + caught.group.as_str(), + caught.indeterminate, + caught.rejected + )); + } + } + // Over budget is not a finding. How long a partition heal may take // before an agent is served again is a judgement, and the driver is not // the one to make it. @@ -431,11 +563,30 @@ impl ReachabilityReport { self.isolated_agents, self.isolated_pod, self.reachable_agents )]; + // The caught population first: it is what the fault actually disturbed, + // and nothing in the cells below points at it. + for caught in &self.caught_in_flight { + lines.push(format!( + "S3 {}: {} operations across {} agents were in flight when the cut landed — \ + p50 {}ms / p99 {}ms / worst {}ms, {} attempt(s) timed out, up to {} attempts \ + each, {} still unresolved at the heal", + caught.group.as_str(), + caught.operations, + caught.agents, + caught.duration.p50_ms, + caught.duration.p99_ms, + caught.duration.max_ms, + caught.attempts_timed_out, + caught.max_attempts, + caught.outlived_the_fault, + )); + } + for group in [Group::OnPod, Group::Elsewhere] { for window in [Window::BeforeFault, Window::DuringFault, Window::AfterFault] { if let Some(cell) = self.cell(group, window) { lines.push(format!( - "S3 {} {}: {:.2} confirmed/s{}, {} submitted, {} indeterminate, {} \ + "S3 {} {}: {:.2} confirmed/s{}, {} submitted{}, {} indeterminate, {} \ attempt(s) timed out, {} of {} agents active", group.as_str(), window.as_str(), @@ -444,6 +595,18 @@ impl ReachabilityReport { .map(|s| format!(" ({s:.1}% of baseline)")) .unwrap_or_default(), cell.submitted, + // Silence is the reading that stops a small rate being + // mistaken for residual service. + cell.quiet_ms + .filter(|_| cell.window_secs > 0.0) + .map(|q| { + format!( + ", first offered {:.1}s into a {:.1}s window", + q as f64 / 1000.0, + cell.window_secs + ) + }) + .unwrap_or_default(), cell.indeterminate, cell.attempts_timed_out, cell.agents_active, @@ -468,6 +631,24 @@ impl ReachabilityReport { } } +/// When a window began, for the windows that have a fixed start. +/// +/// The fault's own boundaries come from the workflow's timestamps, which is +/// what makes them comparable across runs. The baseline has no boundary of its +/// own, so it starts at the first operation the run offered. +fn window_start( + window: Window, + fault: Option, + first_submitted: Option>, +) -> Option> { + match (window, fault) { + (Window::BeforeFault, Some(_)) => first_submitted, + (Window::DuringFault, Some(w)) => Some(w.injected_at), + (Window::AfterFault, Some(w)) => w.recovered_at, + _ => None, + } +} + /// How long a window lasted, in seconds. /// /// The fault's own windows come from the workflow's timestamps, which is what @@ -507,6 +688,29 @@ fn window_secs( } } +/// Operations of one outcome. +fn count_of(records: &[&OperationRecord], outcome: Outcome) -> u64 { + records.iter().filter(|r| r.outcome == outcome).count() as u64 +} + +/// Attempts of one operation that hit the client's attempt timeout. +/// +/// Matched on the message `crate::chaos::workload` writes for a timed-out +/// attempt. A structured flag would be better, but the attempt log is an +/// archived shape shared with every other scenario and this reads it without +/// changing it. +fn timed_out(record: &OperationRecord) -> u64 { + record + .attempt_log + .iter() + .filter(|a| { + a.error + .as_deref() + .is_some_and(|e| e.contains("attempt timed out")) + }) + .count() as u64 +} + fn round2(value: f64) -> f64 { (value * 100.0).round() / 100.0 } @@ -859,4 +1063,184 @@ mod tests { .any(|line| line.contains("recovery budget")) ); } + + /// An operation the cut landed in the middle of: submitted just before it, + /// still running when it landed, answered only once the link returned. + fn caught(agent: &str, timed_out_attempts: u32) -> OperationRecord { + let mut record = op(agent, -1, Outcome::Confirmed); + record.duration_ms = 178_000; + record.completed_at = Some(record.submitted_at + TimeDelta::milliseconds(178_000)); + record.attempts = timed_out_attempts + 1; + record.attempt_log = (1..=timed_out_attempts) + .map(|attempt| AttemptRecord { + attempt, + started_at: record.submitted_at, + duration_ms: 120_000, + returned_value: None, + succeeded: false, + error_class: Some(ErrorClass::Transport), + error: Some("attempt timed out after 120s".to_string()), + }) + .chain(std::iter::once(AttemptRecord { + attempt: timed_out_attempts + 1, + started_at: record.submitted_at, + duration_ms: 56_000, + returned_value: Some(1), + succeeded: true, + error_class: None, + error: None, + })) + .collect(); + record + } + + /// The population the fault actually disturbed, which no cell can show: + /// these were submitted before the cut, so their timeouts and their + /// duration are attributed to the `before-fault` row. + #[test] + fn the_operations_the_cut_caught_are_reported_apart_from_the_cells() { + let mut records = history(0, 180); + records.retain(|r| { + !(r.agent == ISOLATED + && r.submitted_at >= t0() + && r.submitted_at < t0() + TimeDelta::seconds(180)) + }); + records.push(caught(ISOLATED, 1)); + + let report = build(&records); + let caught = report + .caught_in_flight + .iter() + .find(|c| c.group == Group::OnPod) + .expect("the isolated group had an operation in flight"); + + assert_eq!(caught.operations, 1); + assert_eq!(caught.agents, 1); + assert_eq!(caught.confirmed, 1); + assert_eq!(caught.duration.max_ms, 178_000); + assert_eq!(caught.attempts_timed_out, 1); + // The retry is what landed it. With retries off it would have been + // indeterminate, and the outcome alone cannot say so. + assert_eq!(caught.max_attempts, 2); + // It answered before the heal was stamped, so it did not outlive it. + assert_eq!(caught.outlived_the_fault, 0); + + assert!( + report + .note_lines() + .iter() + .any(|l| l.contains("were in flight when the cut landed")), + "the caught population has to be in front of the reader" + ); + } + + /// An operation still unanswered when the link came back is a different + /// statement from one that resolved inside the window, and the report has + /// to keep them apart. + #[test] + fn an_operation_still_running_at_the_heal_is_counted_as_outliving_it() { + let mut records = history(0, 180); + records.retain(|r| { + !(r.agent == ISOLATED + && r.submitted_at >= t0() + && r.submitted_at < t0() + TimeDelta::seconds(180)) + }); + let mut late = caught(ISOLATED, 1); + late.completed_at = Some(t0() + TimeDelta::seconds(200)); + late.duration_ms = 201_000; + records.push(late); + + let report = build(&records); + let caught = report + .caught_in_flight + .iter() + .find(|c| c.group == Group::OnPod) + .unwrap(); + assert_eq!(caught.outlived_the_fault, 1); + } + + /// The number that stops a small during-fault rate reading as residual + /// service. A real run showed 1.1% of baseline from operations that all + /// arrived in the last four seconds of a 182-second window, once the fault + /// was already coming down. + #[test] + fn a_group_silent_until_the_heal_reports_how_long_it_offered_nothing() { + let mut records = history(0, 180); + records.retain(|r| { + !(r.agent == ISOLATED + && r.submitted_at >= t0() + && r.submitted_at < t0() + TimeDelta::seconds(180)) + }); + // A late tail, exactly as the heal window produces. + for offset in 176..180 { + records.push(op(ISOLATED, offset, Outcome::Confirmed)); + } + + let report = build(&records); + let cell = report.cell(Group::OnPod, Window::DuringFault).unwrap(); + + assert_eq!(cell.submitted, 4); + assert_eq!(cell.quiet_ms, Some(176_000)); + assert!( + cell.quiet_ms.unwrap() as f64 / 1000.0 > cell.window_secs * 0.9, + "silence has to dominate the window, not trail it" + ); + // Measured from the window's own start, not from the run's first + // operation: the baseline began 300s earlier. + let baseline = report.cell(Group::OnPod, Window::BeforeFault).unwrap(); + assert_eq!(baseline.quiet_ms, Some(0)); + + assert!( + report + .note_lines() + .iter() + .any(|l| l.contains("first offered 176.0s into a 180.0s window")), + "notes were {:?}", + report.note_lines() + ); + } + + /// Work the cut caught and the run cannot account for. Not a finding, but + /// it must sit next to the clean cells rather than be inferred from them. + #[test] + fn caught_work_that_never_confirmed_is_raised_to_the_operator() { + let mut records = history(0, 180); + records.retain(|r| { + !(r.agent == ISOLATED + && r.submitted_at >= t0() + && r.submitted_at < t0() + TimeDelta::seconds(180)) + }); + let mut lost = caught(ISOLATED, 2); + lost.outcome = Outcome::Indeterminate; + records.push(lost); + + let report = build(&records); + assert!( + report + .attention_lines() + .iter() + .any(|l| l.contains("caught in flight") && l.contains("did not confirm")), + "attention was {:?}", + report.attention_lines() + ); + // Still not a finding: doubt is not damage, and the exactly-once and + // read-back accounts are what resolve it. + assert!(report.findings.is_empty()); + } + + /// A run with no fault window cannot say what was in flight when the cut + /// landed, because it does not know when that was. + #[test] + fn a_run_without_a_fault_window_claims_nothing_was_caught() { + let report = ReachabilityReport::build( + &history(2, 180), + &split(), + None, + 25.0, + 75.0, + Duration::from_secs(60), + ); + assert!(report.caught_in_flight.is_empty()); + assert!(report.cells.iter().all(|c| c.quiet_ms.is_none())); + } } From f12eb038a35f40fb038354019125e7903e41400b Mon Sep 17 00:00:00 2001 From: kmatasfp <33095685+kmatas@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:51:49 -0700 Subject: [PATCH 15/40] Add S7 chaos scenario driver and truncation account --- golem-test-framework/src/benchmark/config.rs | 2 + .../chaos_suites/cloud-chaos.yaml | 78 ++ integration-tests/src/benchmarks/all.rs | 4 + integration-tests/src/chaos/history.rs | 15 +- integration-tests/src/chaos/mod.rs | 167 +++- integration-tests/src/chaos/result.rs | 100 ++- integration-tests/src/chaos/reverts.rs | 314 ++++++++ integration-tests/src/chaos/scenarios/mod.rs | 6 + integration-tests/src/chaos/scenarios/s1.rs | 1 + integration-tests/src/chaos/scenarios/s10.rs | 1 + integration-tests/src/chaos/scenarios/s11.rs | 1 + integration-tests/src/chaos/scenarios/s12.rs | 1 + integration-tests/src/chaos/scenarios/s13.rs | 1 + integration-tests/src/chaos/scenarios/s3.rs | 1 + integration-tests/src/chaos/scenarios/s5.rs | 1 + integration-tests/src/chaos/scenarios/s7.rs | 449 +++++++++++ integration-tests/src/chaos/scenarios/s8.rs | 1 + integration-tests/src/chaos/split.rs | 10 + integration-tests/src/chaos/summary.rs | 39 +- integration-tests/src/chaos/truncation.rs | 731 ++++++++++++++++++ integration-tests/src/chaos/workload.rs | 62 +- 21 files changed, 1979 insertions(+), 6 deletions(-) create mode 100644 integration-tests/src/chaos/reverts.rs create mode 100644 integration-tests/src/chaos/scenarios/s7.rs create mode 100644 integration-tests/src/chaos/truncation.rs diff --git a/golem-test-framework/src/benchmark/config.rs b/golem-test-framework/src/benchmark/config.rs index 729ac3f625..24bdfd6471 100644 --- a/golem-test-framework/src/benchmark/config.rs +++ b/golem-test-framework/src/benchmark/config.rs @@ -248,6 +248,8 @@ pub enum ChaosScenarioArg { S11, /// Executor cut off from worker-service while it keeps its shards. S3, + /// Executor pod kill while agents are having their state reverted. + S7, } /// Density subcommand action. diff --git a/integration-tests/chaos_suites/cloud-chaos.yaml b/integration-tests/chaos_suites/cloud-chaos.yaml index c7ad4944f5..645fd7571a 100644 --- a/integration-tests/chaos_suites/cloud-chaos.yaml +++ b/integration-tests/chaos_suites/cloud-chaos.yaml @@ -622,3 +622,81 @@ scenarios: delaySecs: 5 signalTimeoutSecs: 1800 + + # S7 — executor pod kill during agent state revert (GOL-371). + # + # The only scenario that disturbs work the platform is trying to *undo*. Each + # agent builds its counter up with a run of increments and then asks for some + # of them back, over and over, while an executor is killed underneath. + # + # It is also the only read-back in the suite with no band of doubt in it. The + # last increment of a round returns the counter's value, so the driver knows + # exactly what the agent was worth before the revert and exactly how many + # invocations it asked to take back. Afterwards there are two legal values and + # nothing between them, which is why S7's findings fail the run outright. + # + # Like S8, S10, S11 and S3 the driver names the pod and keeps driving the + # agents on the other executor as a control group. + - code: S7 + name: executor-crash-during-revert + enabled: true + + fault: + kind: pod-kill + target: worker-executor + # The workflow narrows the selector to the pod the driver named; `one` + # stays as the belt-and-braces bound, as in every other pinned scenario. + mode: one + durationSecs: 60 + + phases: + # Long enough for cold starts to settle, so the kill lands on a population + # that has been cycling rounds steadily rather than one still arriving. + baselineSecs: 180 + # Covers the kill, the reschedule, and the shard reassignment that has to + # happen before anything can revert an agent the dead executor owned. + faultSecs: 120 + # Rounds continue throughout. Long enough that every agent runs several + # whole rounds after the kill, so a round disturbed by it is followed by + # ones that were not. + recoverySecs: 300 + + revert: + # 200 agents, split by shard ownership across the two executors. Also the + # resolution of the report: a torn revert localises to one agent out of + # two hundred. + agents: 200 + # Four increments then a revert of two, so a completed round is worth +2. + # + # `revertInvocations` must not exceed `incrementsPerRound`: a revert that + # reached further back would land in the region an earlier revert already + # deleted, and the platform refuses that outright — every round would fail + # and the run would measure nothing. The driver checks this before + # starting rather than discovering it per round. + incrementsPerRound: 4 + revertInvocations: 2 + # Between rounds. With five operations per round this keeps roughly one + # agent in six mid-round at any instant, which is the population a kill + # has something to land in. It also has to leave the round long enough + # that the workflow's inject-and-verify path — signal poll (5s) plus + # `kubectl apply` plus waiting for `AllInjected` — does not fall entirely + # between two rounds. + intervalMillis: 500 + # What recovering a reverted agent may cost. Recorded, not asserted: the + # floor is a shard reassignment plus the worker recovery that replays the + # oplog up to the revert, and how much more than that is acceptable is a + # judgement. + recoveryBudgetSecs: 60 + + retryPolicy: + # Applies to the increments only. **The revert is never retried**, and + # that exception is load-bearing rather than an oversight: a revert has no + # idempotency key and is not idempotent — asking twice for "the last two + # invocations" takes back four. A retried revert that had actually landed + # would look exactly like the platform tearing a truncation, which is the + # finding this scenario exists to make. See `chaos/reverts.rs`. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 diff --git a/integration-tests/src/benchmarks/all.rs b/integration-tests/src/benchmarks/all.rs index 22af3d8f3d..497189d5e3 100644 --- a/integration-tests/src/benchmarks/all.rs +++ b/integration-tests/src/benchmarks/all.rs @@ -595,6 +595,7 @@ async fn run_chaos( ChaosScenarioArg::S10 => chaos::ScenarioCode::S10, ChaosScenarioArg::S11 => chaos::ScenarioCode::S11, ChaosScenarioArg::S3 => chaos::ScenarioCode::S3, + ChaosScenarioArg::S7 => chaos::ScenarioCode::S7, }; let config = suite .scenario(code, allow_disabled) @@ -635,6 +636,9 @@ async fn run_chaos( chaos::ScenarioCode::S3 => { chaos::scenarios::s3::run(&config, &manifest, &deps, &signals, &outputs).await } + chaos::ScenarioCode::S7 => { + chaos::scenarios::s7::run(&config, &manifest, &deps, &signals, &outputs).await + } }; deps.kill_all().await; diff --git a/integration-tests/src/chaos/history.rs b/integration-tests/src/chaos/history.rs index 8b33f3fa0b..302f661e68 100644 --- a/integration-tests/src/chaos/history.rs +++ b/integration-tests/src/chaos/history.rs @@ -67,6 +67,17 @@ pub enum Stream { /// one known executor, so mixing them into the durable population would /// blur two different experiments. PinnedHttp, + /// `Counter.increment` rounds that are deliberately taken back again, and + /// the `revert` calls that take them (GOL-371). + /// + /// Distinct from [`Stream::Durable`] even though both land on `Counter` + /// agents, and for a reason that is not cosmetic: a reverted increment is + /// acknowledged work that the platform was then *asked* to forget, so the + /// generic read-back — which compares a counter against everything + /// confirmed against it — would report every one of them as lost. S7 + /// computes each agent's exact expected value from its own round history + /// instead, which is strictly stronger than a range. + Revert, /// `PromiseWaiter.arm` / `wait` / the external completion that resolves it /// (GOL-377). Distinct from `Promise` even though both land on the promise /// component: that stream creates and resolves a promise in one breath with @@ -85,6 +96,7 @@ impl Stream { Stream::Quota => "quota", Stream::PinnedHttp => "pinned-http", Stream::PromiseWait => "promise-wait", + Stream::Revert => "revert", } } @@ -110,7 +122,7 @@ impl Stream { ) } - pub const ALL: [Stream; 7] = [ + pub const ALL: [Stream; 8] = [ Stream::Durable, Stream::Ephemeral, Stream::Scheduled, @@ -118,6 +130,7 @@ impl Stream { Stream::Quota, Stream::PinnedHttp, Stream::PromiseWait, + Stream::Revert, ]; } diff --git a/integration-tests/src/chaos/mod.rs b/integration-tests/src/chaos/mod.rs index 68cebd582e..ef9e240cc8 100644 --- a/integration-tests/src/chaos/mod.rs +++ b/integration-tests/src/chaos/mod.rs @@ -43,12 +43,14 @@ pub mod prep; pub mod probe; pub mod reachability; pub mod result; +pub mod reverts; pub mod scenarios; pub mod scheduled; pub mod signal; pub mod split; pub mod steady; pub mod summary; +pub mod truncation; pub mod waiters; pub mod wakeups; pub mod workload; @@ -78,6 +80,8 @@ pub enum ScenarioCode { S11, /// Executor cut off from worker-service while it keeps its shards. S3, + /// Executor pod kill while agents are having their state reverted. + S7, } impl ScenarioCode { @@ -91,16 +95,18 @@ impl ScenarioCode { ScenarioCode::S10 => "S10", ScenarioCode::S11 => "S11", ScenarioCode::S3 => "S3", + ScenarioCode::S7 => "S7", } } /// Every scenario this driver implements. The suite YAML is checked against /// this list, so a scenario cannot be enabled in YAML without code behind /// it, nor implemented without an operational switch in front of it. - pub const ALL: [ScenarioCode; 8] = [ + pub const ALL: [ScenarioCode; 9] = [ ScenarioCode::S1, ScenarioCode::S3, ScenarioCode::S5, + ScenarioCode::S7, ScenarioCode::S8, ScenarioCode::S10, ScenarioCode::S11, @@ -474,6 +480,61 @@ impl IsolationConfig { } } +/// Shape of the revert workload (GOL-371). +/// +/// The sixth experiment shape, and the only one that asks the platform to +/// *destroy* durable state on purpose. Every other scenario disturbs work that +/// is trying to happen; this one disturbs work that is trying to be undone. +/// +/// Each agent repeats a round: increment `increments_per_round` times, then +/// revert the last `revert_invocations` of them. Both numbers are exact, and +/// that is the point — the driver knows the counter's value before the revert +/// from the last increment's own return value, so the value afterwards has +/// exactly two legitimate answers and no band of doubt between them. +/// +/// Reverting needs the worker stopped (`lock_stopped_worker` in +/// `golem-worker-executor/src/worker/mod.rs`), so a revert is not one atomic +/// instant but a stop, a commit and a status reattach. The truncation itself is +/// a single oplog entry and cannot tear; the window worth killing into is the +/// one around it. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RevertConfig { + /// Counter agents running rounds, split by shard ownership around the + /// executor the kill is aimed at. Also the resolution of the report: a + /// torn revert localises to one agent out of this many. + pub agents: u32, + /// Increments before each revert. Must be at least `revert_invocations`, + /// or the revert would reach back into an already-deleted oplog region and + /// the platform would refuse it — see `find_nth_invocation_from_end`. + pub increments_per_round: u32, + /// How many of those increments each revert takes back. + pub revert_invocations: u32, + /// Milliseconds an agent waits between rounds. The share of the population + /// standing mid-revert at any instant is roughly one round-step in + /// `increments_per_round + 1`, so this and the round length together decide + /// how much of the mechanism a kill can land in. + pub interval_millis: u64, + /// What recovering a reverted agent may cost, and the number the + /// resume delay is reported against. Recorded rather than asserted, like + /// every other budget in the suite. + pub recovery_budget_secs: u64, +} + +impl RevertConfig { + pub fn interval(&self) -> Duration { + Duration::from_millis(self.interval_millis) + } + pub fn recovery_budget(&self) -> Duration { + Duration::from_secs(self.recovery_budget_secs) + } + /// What one completed round adds to a counter. + pub fn net_per_round(&self) -> u32 { + self.increments_per_round + .saturating_sub(self.revert_invocations) + } +} + /// One step of the executor scale schedule the workflow runs during the fault. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -555,6 +616,9 @@ pub struct ScenarioConfig { /// The reachability workload. Absent for scenarios that do not run one. #[serde(default)] pub isolation: Option, + /// The revert workload. Absent for scenarios that do not run one. + #[serde(default)] + pub revert: Option, /// Shard-ownership oracle settings. Absent for scenarios that do not sample /// executor assignments. #[serde(default)] @@ -638,6 +702,36 @@ impl ScenarioConfig { }) } + /// The revert workload block. See [`Self::require_workload`]. + pub fn require_revert(&self) -> anyhow::Result<&RevertConfig> { + let config = self.revert.as_ref().ok_or_else(|| { + anyhow::anyhow!( + "chaos scenario {} needs a `revert` block in the suite YAML", + self.code + ) + })?; + // Checked here rather than discovered mid-run: a revert reaching past + // its own round lands in an already-deleted oplog region and the + // platform refuses it, so every round would fail and the scenario would + // measure nothing. + if config.revert_invocations > config.increments_per_round { + anyhow::bail!( + "chaos scenario {}: revertInvocations ({}) exceeds incrementsPerRound ({}), \ + so every revert would reach into an already-deleted oplog region", + self.code, + config.revert_invocations, + config.increments_per_round + ); + } + if config.revert_invocations == 0 { + anyhow::bail!( + "chaos scenario {}: revertInvocations is 0, so the scenario would revert nothing", + self.code + ); + } + Ok(config) + } + /// The pinned workload block. See [`Self::require_workload`]. pub fn require_pinned(&self) -> anyhow::Result<&PinnedConfig> { self.pinned.as_ref().ok_or_else(|| { @@ -774,6 +868,7 @@ mod tests { scheduled: None, promise: None, isolation: None, + revert: None, ownership: None, scale_during_fault: None, retry_policy: RetryPolicy::default(), @@ -786,6 +881,72 @@ mod tests { assert!(suite.scenario(ScenarioCode::S12, true).is_ok()); } + fn revert_config(increments: u32, revert: u32) -> ScenarioConfig { + ScenarioConfig { + code: "S7".to_string(), + name: "executor-crash-during-revert".to_string(), + enabled: true, + fault: FaultConfig { + kind: "pod-kill".to_string(), + target: "worker-executor".to_string(), + mode: "one".to_string(), + target_count: None, + duration_secs: 60, + }, + phases: PhaseConfig { + baseline_secs: 1, + fault_secs: 1, + recovery_secs: 1, + }, + workload: None, + pinned: None, + scheduled: None, + promise: None, + isolation: None, + revert: Some(RevertConfig { + agents: 10, + increments_per_round: increments, + revert_invocations: revert, + interval_millis: 500, + recovery_budget_secs: 60, + }), + ownership: None, + scale_during_fault: None, + retry_policy: RetryPolicy::default(), + signal_timeout_secs: 1, + } + } + + /// A revert reaching further back than its own round lands in the region an + /// earlier revert already deleted, and the platform refuses it outright. + /// Every round would fail and the run would measure nothing, so this is + /// caught before the maintenance window is spent rather than per round. + #[test] + fn a_revert_deeper_than_its_own_round_is_refused_before_the_run_starts() { + let error = revert_config(2, 3) + .require_revert() + .unwrap_err() + .to_string(); + assert!( + error.contains("already-deleted oplog region"), + "the message has to say why, got: {error}" + ); + } + + /// Reverting nothing would leave a scenario that builds state up and takes + /// none of it back, which is S8 with extra steps. + #[test] + fn a_revert_of_nothing_is_refused() { + assert!(revert_config(4, 0).require_revert().is_err()); + } + + /// The boundary case is legal: a round may take back everything it added. + #[test] + fn a_revert_of_exactly_one_round_is_allowed() { + let config = revert_config(3, 3); + assert_eq!(config.require_revert().unwrap().net_per_round(), 0); + } + /// The retry defaults are load-bearing for correctness, not just for load. #[test] fn retry_policy_defaults_to_one_same_key_transport_only_retry() { @@ -847,6 +1008,7 @@ mod tests { assert_eq!(ScenarioCode::parse("s8"), Some(ScenarioCode::S8)); assert_eq!(ScenarioCode::parse("s1"), Some(ScenarioCode::S1)); assert_eq!(ScenarioCode::parse("s3"), Some(ScenarioCode::S3)); + assert_eq!(ScenarioCode::parse("s7"), Some(ScenarioCode::S7)); assert_eq!(ScenarioCode::parse("S99"), None); } @@ -882,6 +1044,9 @@ mod tests { ScenarioCode::S3 => { entry.require_isolation().unwrap(); } + ScenarioCode::S7 => { + entry.require_revert().unwrap(); + } } } } diff --git a/integration-tests/src/chaos/result.rs b/integration-tests/src/chaos/result.rs index 4b4994e7c0..3f03fa3104 100644 --- a/integration-tests/src/chaos/result.rs +++ b/integration-tests/src/chaos/result.rs @@ -29,8 +29,8 @@ use crate::chaos::scheduled::ScheduledSelection; use crate::chaos::split::PodSplit; use crate::chaos::summary::{ChaosSummary, TerminationReason}; use crate::chaos::{ - FaultConfig, IsolationConfig, PinnedConfig, PromiseConfig, RetryPolicy, ScheduledConfig, - WorkloadConfig, + FaultConfig, IsolationConfig, PinnedConfig, PromiseConfig, RetryPolicy, RevertConfig, + ScheduledConfig, WorkloadConfig, }; use chrono::{DateTime, Utc}; use golem_test_framework::benchmark::RunMetadata; @@ -162,6 +162,13 @@ pub struct ChaosResult { /// between the two groups, so a report without this cannot be re-checked. #[serde(default, skip_serializing_if = "Option::is_none")] pub isolation_selection: Option, + /// The revert workload the run was configured with, if any. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub revert: Option, + /// How the revert agents divided around the executor the kill was aimed at. + /// Present only for S7. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub revert_selection: Option, pub retry_policy: RetryPolicy, pub scope: RunScope, pub summary: ChaosSummary, @@ -235,6 +242,8 @@ mod tests { promise_selection: None, isolation: None, isolation_selection: None, + revert: None, + revert_selection: None, retry_policy: RetryPolicy::default(), scope: RunScope { environment_id: "env-1".to_string(), @@ -335,6 +344,89 @@ mod tests { assert!(parsed.promise_selection.is_some()); } + /// The S7 shape. Same contract as the S3, S10 and S11 tests: the + /// investigation report in golem-cloud reads these fields by name. + #[test] + fn an_s7_result_carries_the_truncation_fields_the_investigation_report_reads() { + use crate::chaos::reverts::RevertRound; + use crate::chaos::split::{FaultWindow, PodSplit}; + use crate::chaos::truncation::TruncationReport; + + let now = Utc::now(); + let agent = "chaos-s7-revert-0000".to_string(); + let split = PodSplit { + pod_address: "10.0.1.1:9000".to_string(), + pod_ip: "10.0.1.1".to_string(), + on_pod: vec![agent.clone()], + elsewhere: Vec::new(), + targets_per_pod: std::collections::BTreeMap::new(), + number_of_shards: 1024, + }; + + let mut result = sample_result(TerminationReason::Completed); + result.scenario_code = "S7".to_string(); + result.revert = Some(crate::chaos::RevertConfig { + agents: 200, + increments_per_round: 4, + revert_invocations: 2, + interval_millis: 500, + recovery_budget_secs: 60, + }); + result.revert_selection = Some(split.clone()); + result.summary = ChaosSummary::build(&[], Vec::new(), Vec::new(), Some(now)) + .with_truncation(TruncationReport::build( + &[RevertRound { + agent, + round: 0, + before_revert: Some(10), + asked_to_revert: 2, + outcome: crate::chaos::history::Outcome::Confirmed, + submitted_at: now, + completed_at: Some(now), + observed_after: Some(8), + }], + &split, + Some(FaultWindow { + injected_at: now, + recovered_at: None, + }), + 4, + 2, + )); + + let json = serde_json::to_value(&result).unwrap(); + let truncation = &json["summary"]["truncation"]; + for key in [ + "incrementsPerRound", + "revertInvocations", + "roundsRecorded", + "revertsConfirmed", + "revertsIndeterminate", + "revertsRejected", + "appliedExactly", + "indeterminateThatApplied", + "indeterminateThatDidNot", + "unjudgeable", + "unprobed", + "cells", + "caughtByTheKill", + "findings", + "findingsOmitted", + ] { + assert!( + !truncation[key].is_null(), + "summary.truncation.{key} is what the investigation report reads" + ); + } + assert_eq!(json["revert"]["revertInvocations"], 2); + assert_eq!(json["revertSelection"]["podIp"], "10.0.1.1"); + + let parsed: ChaosResult = serde_json::from_str(&json.to_string()).unwrap(); + assert_eq!(parsed.scenario_code, "S7"); + assert!(parsed.summary.truncation.is_some()); + assert!(parsed.revert_selection.is_some()); + } + /// The S3 shape. Same contract as the S10 and S11 tests, for the same /// reason: `ci-scripts/chaos-investigation-report.py` in golem-cloud reads /// these by name, and the two repositories cannot be changed atomically. @@ -745,6 +837,8 @@ mod sample_artifact { promise_selection: None, isolation: None, isolation_selection: None, + revert: None, + revert_selection: None, retry_policy: RetryPolicy::default(), scope: RunScope { environment_id: "0192f000-0000-7000-8000-000000000001".to_string(), @@ -961,6 +1055,8 @@ mod sample_artifact { promise_selection: None, isolation: None, isolation_selection: None, + revert: None, + revert_selection: None, retry_policy: RetryPolicy::default(), scope: RunScope { environment_id: "0192f000-0000-7000-8000-000000000001".to_string(), diff --git a/integration-tests/src/chaos/reverts.rs b/integration-tests/src/chaos/reverts.rs new file mode 100644 index 0000000000..e609e2bb70 --- /dev/null +++ b/integration-tests/src/chaos/reverts.rs @@ -0,0 +1,314 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Agents that build state up and then ask the platform to take it back +//! (GOL-371). +//! +//! One emitter per agent, running rounds. A round is `increments_per_round` +//! increments followed by one `revert` of the last `revert_invocations` of +//! them. Both numbers are exact, which is what gives this scenario an oracle +//! with no band of doubt in it: the last increment of a round *returns* the +//! counter's value, so the driver knows exactly what the agent was worth before +//! the revert, and afterwards there are exactly two legitimate answers. +//! +//! ### The revert must never be retried +//! +//! Every other operation in this suite retries once, under the original +//! idempotency key, and that retry is load-bearing: it is what exposes +//! duplicate execution. A revert has no such key and is not idempotent. +//! Reverting "the last two invocations" twice takes back four. So a revert +//! whose response was lost but which actually landed would, on retry, revert +//! again — and the result would be indistinguishable from the platform tearing +//! a truncation, which is precisely the finding this scenario exists to make. +//! +//! The retry is therefore switched off for the revert call alone, and left on +//! for the increments around it. This is the only place in the suite that does +//! that, and it is not an oversight anywhere else. +//! +//! ### How a round is judged +//! +//! Not by reading the counter: a read is an invocation, and it would land in +//! the oplog between the increments and the next revert, shifting what "the +//! last N invocations" means. The **next round's first increment** is the probe +//! instead. It returns the new value, so the value the revert left behind is +//! that minus one, and it costs nothing extra. + +use crate::chaos::history::{Outcome, Stream}; +use crate::chaos::workload::{self, WorkloadContext}; +use crate::chaos::{RetryPolicy, RevertConfig}; +use chrono::{DateTime, Utc}; +use golem_common::model::worker::{RevertLastInvocations, RevertWorkerTarget}; +use golem_test_framework::dsl::TestDsl; +use serde::{Deserialize, Serialize}; +use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use tokio::task::JoinSet; +use tracing::info; + +/// One round, as the driver observed it. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RevertRound { + pub agent: String, + pub round: u32, + /// The counter value the round's last increment returned, so the value the + /// agent was worth immediately before the revert. `None` when an increment + /// in the round did not answer, which leaves the round unjudgeable rather + /// than failed. + pub before_revert: Option, + /// How many invocations this round's revert asked to take back. + pub asked_to_revert: u32, + /// What the revert call itself returned. + pub outcome: Outcome, + pub submitted_at: DateTime, + pub completed_at: Option>, + /// The counter value observed after the revert, taken from the next round's + /// first increment. `None` for the last round of the run, which the final + /// read-back answers instead. + pub observed_after: Option, +} + +/// A running revert workload. +pub struct RevertHandle { + stop: Arc, + tasks: JoinSet<()>, + submitted: Arc, + rounds: Arc>>, +} + +impl RevertHandle { + pub fn submitted(&self) -> u64 { + self.submitted.load(Ordering::Relaxed) + } + + /// Rounds recorded so far, in no particular order. + pub fn rounds(&self) -> Vec { + self.rounds.lock().map(|r| r.clone()).unwrap_or_default() + } + + /// Signals every emitter to stop and waits for operations in flight to + /// record themselves. + pub async fn stop(mut self) -> Vec { + self.stop.store(1, Ordering::Relaxed); + while self.tasks.join_next().await.is_some() {} + let rounds = self.rounds(); + info!( + "Chaos revert workload stopped after {} operations across {} rounds", + self.submitted(), + rounds.len() + ); + rounds + } +} + +/// The agents a run of `count` emitters drives, in index order. +pub fn agent_names(ctx: &WorkloadContext, count: u32) -> Vec { + (0..count) + .map(|index| ctx.agent_name(Stream::Revert, index)) + .collect() +} + +/// Starts one emitter per agent. +pub fn start(ctx: WorkloadContext, config: &RevertConfig) -> RevertHandle { + let stop = Arc::new(AtomicU8::new(0)); + let submitted = Arc::new(AtomicU64::new(0)); + let rounds: Arc>> = Arc::new(Mutex::new(Vec::new())); + let mut tasks = JoinSet::new(); + + info!( + "Chaos revert workload starting: {} emitters, {} increments then a revert of {} per \ + round, {:?} between rounds", + config.agents, + config.increments_per_round, + config.revert_invocations, + config.interval() + ); + + for index in 0..config.agents { + let ctx = ctx.clone(); + let stop = stop.clone(); + let submitted = submitted.clone(); + let rounds = rounds.clone(); + let config = config.clone(); + + tasks.spawn(async move { + let agent = ctx.agent_name(Stream::Revert, index); + let mut round: u32 = 0; + // Index into `rounds` of the round still waiting to be judged by + // the next increment this agent runs. + let mut pending: Option = None; + + while stop.load(Ordering::Relaxed) == 0 { + // ── Increments ────────────────────────────────────────────── + let mut last_value = None; + let mut all_answered = true; + for step in 0..config.increments_per_round { + if stop.load(Ordering::Relaxed) != 0 { + break; + } + let seq = round as u64 * config.increments_per_round as u64 + step as u64; + submitted.fetch_add(1, Ordering::Relaxed); + let value = increment(&ctx, &agent, seq).await; + + // The first increment of a round is also the probe that + // says what the previous round's revert left behind. + if let Some(slot) = pending.take() + && let Some(observed) = value + && let Ok(mut rounds) = rounds.lock() + && let Some(entry) = rounds.get_mut(slot) + { + entry.observed_after = Some(observed.saturating_sub(1)); + } + + match value { + Some(v) => last_value = Some(v), + None => all_answered = false, + } + } + if stop.load(Ordering::Relaxed) != 0 { + break; + } + + // ── Revert ────────────────────────────────────────────────── + let submitted_at = Utc::now(); + submitted.fetch_add(1, Ordering::Relaxed); + let outcome = revert_once(&ctx, &agent, round, &config).await; + + if let Ok(mut rounds) = rounds.lock() { + rounds.push(RevertRound { + agent: agent.clone(), + round, + before_revert: all_answered.then_some(last_value).flatten(), + asked_to_revert: config.revert_invocations, + outcome, + submitted_at, + completed_at: Some(Utc::now()), + observed_after: None, + }); + pending = Some(rounds.len() - 1); + } + + round += 1; + if stop.load(Ordering::Relaxed) == 0 { + tokio::time::sleep(config.interval()).await; + } + } + }); + } + + RevertHandle { + stop, + tasks, + submitted, + rounds, + } +} + +/// One increment, returning the counter value it reported. +async fn increment(ctx: &WorkloadContext, agent: &str, seq: u64) -> Option { + let key = ctx.idempotency_key(agent, seq); + workload::increment_counter(ctx, Stream::Revert, agent, key) + .await + .value + .map(u64::from) +} + +/// One revert, with retries switched off. See the module docs. +async fn revert_once( + ctx: &WorkloadContext, + agent: &str, + round: u32, + config: &RevertConfig, +) -> Outcome { + let mut once = ctx.clone(); + once.retry = RetryPolicy { + transport_only: true, + max_retries: 0, + delay_secs: 0, + }; + + let key = format!("{agent}-revert-{round:08}"); + let agent_id = workload::counter_agent_id(&once, agent); + let number_of_invocations = config.revert_invocations; + let ctx2 = once.clone(); + + workload::run_operation( + &once, + Stream::Revert, + agent.to_string(), + "revert", + key, + |_| { + let ctx = ctx2.clone(); + let agent_id = agent_id.clone(); + async move { + ctx.user + .revert( + &agent_id, + RevertWorkerTarget::RevertLastInvocations(RevertLastInvocations { + number_of_invocations: number_of_invocations as u64, + }), + ) + .await?; + Ok(None) + } + }, + ) + .await + .outcome +} + +/// The value a completed run of `rounds` rounds should leave on an agent whose +/// every round landed. +pub fn expected_after(config: &RevertConfig, rounds: u32) -> u64 { + config.net_per_round() as u64 * rounds as u64 +} + +#[cfg(test)] +mod tests { + use super::*; + use test_r::test; + + fn config() -> RevertConfig { + RevertConfig { + agents: 200, + increments_per_round: 4, + revert_invocations: 2, + interval_millis: 500, + recovery_budget_secs: 60, + } + } + + /// The arithmetic the whole oracle rests on: a round that lands is worth + /// exactly its increments less what the revert took back. + #[test] + fn a_completed_round_is_worth_its_increments_less_the_revert() { + assert_eq!(config().net_per_round(), 2); + assert_eq!(expected_after(&config(), 0), 0); + assert_eq!(expected_after(&config(), 10), 20); + } + + /// A revert that takes back everything it added is legal and leaves the + /// agent where it started. Nothing in the arithmetic may go negative. + #[test] + fn a_round_that_reverts_everything_it_added_is_worth_nothing() { + let config = RevertConfig { + increments_per_round: 3, + revert_invocations: 3, + ..config() + }; + assert_eq!(config.net_per_round(), 0); + assert_eq!(expected_after(&config, 100), 0); + } +} diff --git a/integration-tests/src/chaos/scenarios/mod.rs b/integration-tests/src/chaos/scenarios/mod.rs index 6c8a25dd83..ab2f361fad 100644 --- a/integration-tests/src/chaos/scenarios/mod.rs +++ b/integration-tests/src/chaos/scenarios/mod.rs @@ -32,6 +32,7 @@ pub mod s12; pub mod s13; pub mod s3; pub mod s5; +pub mod s7; pub mod s8; use crate::chaos::ScenarioConfig; @@ -89,6 +90,9 @@ pub struct ScenarioOutcome { /// Present only for S3, which divides its agents around the executor the /// partition cuts off rather than around one that dies. pub isolation_selection: Option, + /// Present only for S7, which divides the agents whose state is being + /// reverted around the executor the kill is aimed at. + pub revert_selection: Option, } /// Assembles the archived result. @@ -118,6 +122,8 @@ pub fn build_result(config: &ScenarioConfig, outcome: ScenarioOutcome) -> ChaosR promise_selection: outcome.promise_selection, isolation: config.isolation.clone(), isolation_selection: outcome.isolation_selection, + revert: config.revert.clone(), + revert_selection: outcome.revert_selection, retry_policy: config.retry_policy.clone(), scope: outcome.scope, summary: outcome.summary, diff --git a/integration-tests/src/chaos/scenarios/s1.rs b/integration-tests/src/chaos/scenarios/s1.rs index e6ffd7fef8..22692eb5c5 100644 --- a/integration-tests/src/chaos/scenarios/s1.rs +++ b/integration-tests/src/chaos/scenarios/s1.rs @@ -238,6 +238,7 @@ pub async fn run( scheduled_selection: None, promise_selection: None, isolation_selection: None, + revert_selection: None, }, ); write_outputs(&result, &history, outputs)?; diff --git a/integration-tests/src/chaos/scenarios/s10.rs b/integration-tests/src/chaos/scenarios/s10.rs index 2831ac67a0..e9ae75611a 100644 --- a/integration-tests/src/chaos/scenarios/s10.rs +++ b/integration-tests/src/chaos/scenarios/s10.rs @@ -224,6 +224,7 @@ pub async fn run( scheduled_selection: selection.clone(), promise_selection: None, isolation_selection: None, + revert_selection: None, }, ); write_outputs(&result, &history, outputs)?; diff --git a/integration-tests/src/chaos/scenarios/s11.rs b/integration-tests/src/chaos/scenarios/s11.rs index 641f8927eb..84a1c21006 100644 --- a/integration-tests/src/chaos/scenarios/s11.rs +++ b/integration-tests/src/chaos/scenarios/s11.rs @@ -198,6 +198,7 @@ pub async fn run( scheduled_selection: None, promise_selection: selection.clone(), isolation_selection: None, + revert_selection: None, }, ); write_outputs(&result, &history, outputs)?; diff --git a/integration-tests/src/chaos/scenarios/s12.rs b/integration-tests/src/chaos/scenarios/s12.rs index 3dd67fe1e7..bd95b6ec52 100644 --- a/integration-tests/src/chaos/scenarios/s12.rs +++ b/integration-tests/src/chaos/scenarios/s12.rs @@ -135,6 +135,7 @@ pub async fn run( scheduled_selection: None, promise_selection: None, isolation_selection: None, + revert_selection: None, }, ); write_outputs(&result, &history, outputs)?; diff --git a/integration-tests/src/chaos/scenarios/s13.rs b/integration-tests/src/chaos/scenarios/s13.rs index f749e52c3c..520bd1b021 100644 --- a/integration-tests/src/chaos/scenarios/s13.rs +++ b/integration-tests/src/chaos/scenarios/s13.rs @@ -206,6 +206,7 @@ pub async fn run( scheduled_selection: None, promise_selection: None, isolation_selection: None, + revert_selection: None, }, ); write_outputs(&result, &history, outputs)?; diff --git a/integration-tests/src/chaos/scenarios/s3.rs b/integration-tests/src/chaos/scenarios/s3.rs index 843a62fa3a..2537e7fe90 100644 --- a/integration-tests/src/chaos/scenarios/s3.rs +++ b/integration-tests/src/chaos/scenarios/s3.rs @@ -183,6 +183,7 @@ pub async fn run( scheduled_selection: None, promise_selection: None, isolation_selection: selection.clone(), + revert_selection: None, }, ); write_outputs(&result, &history, outputs)?; diff --git a/integration-tests/src/chaos/scenarios/s5.rs b/integration-tests/src/chaos/scenarios/s5.rs index 71eb018393..a08bd04def 100644 --- a/integration-tests/src/chaos/scenarios/s5.rs +++ b/integration-tests/src/chaos/scenarios/s5.rs @@ -178,6 +178,7 @@ pub async fn run( scheduled_selection: None, promise_selection: None, isolation_selection: None, + revert_selection: None, }, ); write_outputs(&result, &history, outputs)?; diff --git a/integration-tests/src/chaos/scenarios/s7.rs b/integration-tests/src/chaos/scenarios/s7.rs new file mode 100644 index 0000000000..1780b0ba88 --- /dev/null +++ b/integration-tests/src/chaos/scenarios/s7.rs @@ -0,0 +1,449 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! S7 — executor pod kill during agent state revert (GOL-371). +//! +//! Every other scenario in this suite disturbs work that is trying to happen. +//! S7 disturbs work that is trying to be **undone**: each agent builds its +//! counter up with a run of increments and then asks the platform to take some +//! of them back, over and over, while an executor is killed underneath. +//! +//! ### Why this one can assert +//! +//! Read-backs elsewhere compare a counter against a range, and the width of the +//! range is the operations whose fate the driver could not determine. Here +//! there is no range. The last increment of a round *returns* the counter's +//! value, so the driver knows exactly what the agent was worth immediately +//! before the revert, and it asked for an exact number of invocations back. So +//! afterwards there are two legal values, `V` and `V - N`, and nothing between +//! them. See [`crate::chaos::truncation`] for what each other answer means. +//! +//! That is why S7 is one of the few scenarios whose read-back can fail the run +//! outright rather than being reported for a human to weigh. +//! +//! ### What is actually being killed into +//! +//! The truncation itself cannot tear: `RevertLastInvocations` commits a single +//! `OplogEntry::revert` marking a region deleted. The window worth aiming at is +//! the one around it — reverting takes `lock_stopped_worker`, so the worker is +//! stopped, the entry is committed, and only then is the worker status +//! reattached. An executor that dies between the commit and the reattach has +//! changed durable state and lost the thing that tells anyone about it. +//! +//! ### The choreography +//! +//! Like S8, S10, S11 and S3 the driver names the pod: it picks the executor +//! owning the largest share of its agents and keeps driving the rest as a +//! control group. Unlike them, the last thing it does before read-back is a +//! single read per agent, which answers the one round per agent that no +//! following increment ever probed. + +use crate::chaos::history::{OperationHistory, Outcome, Phase, Stream}; +use crate::chaos::ownership::OwnershipSample; +use crate::chaos::prep::ChaosPrepManifest; +use crate::chaos::result::{ChaosResult, PhaseWindow, Phases, RunScope}; +use crate::chaos::reverts::{self, RevertRound}; +use crate::chaos::scenarios::{ + OutputPaths, ScenarioOutcome, WARMUP_SETTLE, build_result, sample_ownership, + signal_termination, snapshot_routing, wait_for_settled_routing, write_outputs, +}; +use crate::chaos::signal::{BaselineReady, FaultSignals, FaultTarget}; +use crate::chaos::split::{self, FaultWindow, PodSplit}; +use crate::chaos::summary::{ChaosSummary, Note, TerminationReason}; +use crate::chaos::truncation::TruncationReport; +use crate::chaos::workload::{self, PhaseMarker, WorkloadContext}; +use crate::chaos::{ScenarioCode, ScenarioConfig}; +use chrono::Utc; +use golem_test_framework::config::BenchmarkTestDependencies; +use golem_test_framework::dsl::TestDsl; +use std::time::Duration; +use tracing::{info, warn}; + +/// How long to wait after stopping the workload before the final read. +/// +/// Shorter than the other scenarios' settle: nothing here is queued or +/// scheduled, and `stop` already waits for every operation in flight to record +/// itself. This only covers a worker still coming back from the last revert. +const SETTLE_BEFORE_READBACK: Duration = Duration::from_secs(20); + +/// How far into the fault window the assignment is sampled. +const DURING_FAULT_SAMPLE_FRACTION: f64 = 0.6; +const DURING_FAULT_SAMPLE_CAP: Duration = Duration::from_secs(120); + +/// Runs S7 end to end. +pub async fn run( + config: &ScenarioConfig, + manifest: &ChaosPrepManifest, + deps: &BenchmarkTestDependencies, + signals: &FaultSignals, + outputs: &OutputPaths, +) -> anyhow::Result { + let started_at = Utc::now(); + let revert_config = config.require_revert()?; + let history = OperationHistory::new(ScenarioCode::S7.as_str()); + let key_prefix = crate::chaos::scenario_key_prefix(ScenarioCode::S7); + + let user = manifest.user_context(deps); + let counters = user + .get_latest_component_revision(&manifest.counters_component_id) + .await?; + let promise = user + .get_latest_component_revision(&manifest.promise_component_id) + .await?; + + let ctx = WorkloadContext { + user, + counters, + promise, + history: history.clone(), + retry: config.retry_policy.clone(), + phase: PhaseMarker::new(Phase::Baseline), + key_prefix: key_prefix.clone(), + }; + + let scope = RunScope { + environment_id: manifest.environment_id.0.to_string(), + component_ids: vec![manifest.counters_component_id.0.to_string()], + agent_id_prefix: key_prefix.clone(), + idempotency_key_prefix: format!("{key_prefix}-"), + }; + + let agents = reverts::agent_names(&ctx, revert_config.agents); + + let mut phases = Phases::default(); + let mut routing_snapshots = Vec::new(); + let mut ownership: Vec = Vec::new(); + let mut fault_injected_at = None; + let mut fault_recovered_at = None; + let mut fault_id = None; + let mut fault_target_observed = None; + let mut selection: Option = None; + let mut attention_extra: Vec = Vec::new(); + + macro_rules! finish { + ($reason:expr, $records:expr, $truncation:expr) => {{ + let mut summary = ChaosSummary::build( + $records, + Vec::new(), + routing_snapshots.clone(), + fault_injected_at, + ) + .with_ownership(ownership.clone()); + summary.absorb(attention_extra.clone()); + if let Some(report) = $truncation { + summary = summary.with_truncation(report); + } + let result = build_result( + config, + ScenarioOutcome { + started_at, + phases: phases.clone(), + fault_injected_at, + fault_recovered_at, + fault_id: fault_id.clone(), + fault_target_observed: fault_target_observed.clone(), + scope: scope.clone(), + summary, + termination_reason: $reason, + pinned_selection: None, + scheduled_selection: None, + promise_selection: None, + isolation_selection: None, + revert_selection: selection.clone(), + }, + ); + write_outputs(&result, &history, outputs)?; + return Ok(result); + }}; + } + + // ── Warm-up ───────────────────────────────────────────────────────────── + // + // Constructing an agent is itself recorded in its oplog, so doing it inside + // a measured round would put an entry between the increments and the revert + // that counts them. Reads here rather than increments, for the same reason + // every other scenario warms with reads: an increment would be invisible to + // the round arithmetic the whole oracle rests on. + routing_snapshots.push(snapshot_routing(deps, "before-warmup").await); + attention_extra.push(wait_for_settled_routing(deps, &mut routing_snapshots).await); + + info!("S7: warming up {} revert agents", agents.len()); + let mut warmed = 0usize; + for agent in &agents { + if workload::read_counter(&ctx, agent).await.is_ok() { + warmed += 1; + } + } + info!( + "S7: warmed {warmed} of {} agents, settling {WARMUP_SETTLE:?}", + agents.len() + ); + tokio::time::sleep(WARMUP_SETTLE).await; + + // ── Aim ───────────────────────────────────────────────────────────────── + let subject = split::revert_subject(&ctx); + let split = match split::select(subject, deps, &agents).await { + Ok(split) => split, + Err(e) => { + warn!("S7: cannot aim the kill: {e:#}"); + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!("{e:#}"), + }, + &records, + None + ); + } + }; + selection = Some(split.clone()); + + // ── Baseline ──────────────────────────────────────────────────────────── + info!( + "S7: baseline phase, running {} revert emitters for {:?}", + agents.len(), + config.phases.baseline() + ); + phases.baseline = Some(PhaseWindow::started(Utc::now())); + let handle = reverts::start(ctx.clone(), revert_config); + tokio::time::sleep(config.phases.baseline()).await; + routing_snapshots.push(snapshot_routing(deps, "before-fault").await); + ownership.push(sample_ownership(deps, "before-fault", ownership.last(), false).await); + if let Some(window) = phases.baseline.as_mut() { + window.end(Utc::now()); + } + + let baseline_operations = history.confirmed_in_phase(Phase::Baseline); + if baseline_operations == 0 { + warn!("S7: baseline produced no confirmed operations, aborting before injection"); + let rounds = handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::PlatformUnreachable { + detail: "no operation succeeded during the baseline phase".to_string(), + }, + &records, + Some(build_truncation(&rounds, &split, None, revert_config)) + ); + } + + if let Err(e) = split::verify_ownership(subject, deps, &split).await { + warn!("S7: ownership drifted between selection and injection: {e:#}"); + let rounds = handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!("{e:#}"), + }, + &records, + Some(build_truncation(&rounds, &split, None, revert_config)) + ); + } + + info!( + "S7: baseline complete ({baseline_operations} confirmed ops, {} rounds), naming {} and \ + signalling readiness", + handle.rounds().len(), + split.pod_address + ); + signals.write_baseline_ready(&BaselineReady { + scenario_code: ScenarioCode::S7.as_str().to_string(), + ready_at: Utc::now(), + baseline_operations, + fault_target: Some(FaultTarget { + pod_address: split.pod_address.clone(), + pod_ip: split.pod_ip.clone(), + owned_agents: split.on_pod.clone(), + }), + })?; + + // ── Fault ─────────────────────────────────────────────────────────────── + let injected = match signals.await_fault_injected(config.signal_timeout()).await { + Ok(injected) => injected, + Err(e) => { + warn!("S7: no fault-injected signal arrived: {e}"); + let rounds = handle.stop().await; + let records = history.snapshot(); + finish!( + signal_termination(&e), + &records, + Some(build_truncation(&rounds, &split, None, revert_config)) + ); + } + }; + info!( + "S7: fault {} ({} on {}) reported active at {}", + injected.fault_id, injected.kind, injected.target, injected.injected_at + ); + fault_injected_at = Some(injected.injected_at); + fault_id = Some(injected.fault_id.clone()); + fault_target_observed = Some(injected.target.clone()); + ctx.phase.set(Phase::Fault); + phases.fault = Some(PhaseWindow::started(injected.injected_at)); + + let observe_after = config + .phases + .fault() + .mul_f64(DURING_FAULT_SAMPLE_FRACTION) + .min(DURING_FAULT_SAMPLE_CAP); + tokio::time::sleep(observe_after).await; + ownership.push(sample_ownership(deps, "during-fault", ownership.last(), false).await); + + let recovered = match signals.await_fault_recovered(config.signal_timeout()).await { + Ok(recovered) => recovered, + Err(e) => { + warn!("S7: no fault-recovered signal arrived: {e}"); + let rounds = handle.stop().await; + let records = history.snapshot(); + finish!( + signal_termination(&e), + &records, + Some(build_truncation( + &rounds, + &split, + fault_window(fault_injected_at, None), + revert_config + )) + ); + } + }; + info!( + "S7: executor back at {} ({})", + recovered.recovered_at, recovered.termination_reason + ); + fault_recovered_at = Some(recovered.recovered_at); + if let Some(window) = phases.fault.as_mut() { + window.end(recovered.recovered_at); + } + + // ── Recovery ──────────────────────────────────────────────────────────── + ctx.phase.set(Phase::Recovery); + phases.recovery = Some(PhaseWindow::started(Utc::now())); + info!( + "S7: recovery phase, running for {:?}", + config.phases.recovery() + ); + tokio::time::sleep(config.phases.recovery()).await; + + let mut rounds = handle.stop().await; + if let Some(window) = phases.recovery.as_mut() { + window.end(Utc::now()); + } + routing_snapshots.push(snapshot_routing(deps, "after-recovery").await); + ownership.push(sample_ownership(deps, "after-recovery", ownership.last(), true).await); + + // ── Read-back ─────────────────────────────────────────────────────────── + info!("S7: settling {SETTLE_BEFORE_READBACK:?} before the final read"); + tokio::time::sleep(SETTLE_BEFORE_READBACK).await; + + // The one read per agent, which answers the last round it ran. Every other + // round was probed by the increment that followed it; this is the only one + // that has nothing after it. + close_last_rounds(&ctx, &agents, &mut rounds).await; + + let records = history.snapshot(); + let truncation = build_truncation( + &rounds, + &split, + fault_window(fault_injected_at, fault_recovered_at), + revert_config, + ); + info!( + "S7: truncation account — {} rounds, {} applied exactly, {} findings", + truncation.rounds_recorded, + truncation.applied_exactly, + truncation.findings.len() + ); + + let reason = if truncation.has_violations() { + let first = truncation + .findings + .first() + .map(|f| format!("{} round {}: {}", f.agent, f.round, f.detail)) + .unwrap_or_default(); + TerminationReason::RevertTruncationViolated { + findings: truncation.findings.len() as u64 + truncation.findings_omitted, + first, + } + } else if records.iter().all(|r| r.outcome != Outcome::Confirmed) { + TerminationReason::StreamNeverSucceeded { + stream: Stream::Revert.to_string(), + } + } else { + TerminationReason::Completed + }; + + finish!(reason, &records, Some(truncation)); +} + +fn fault_window( + injected_at: Option>, + recovered_at: Option>, +) -> Option { + injected_at.map(|injected_at| FaultWindow { + injected_at, + recovered_at, + }) +} + +fn build_truncation( + rounds: &[RevertRound], + split: &PodSplit, + fault: Option, + config: &crate::chaos::RevertConfig, +) -> TruncationReport { + TruncationReport::build( + rounds, + split, + fault, + config.increments_per_round, + config.revert_invocations, + ) +} + +/// Reads each agent once and uses the value to judge the last round it ran. +/// +/// A read is an invocation and would shift what "the last N invocations" means +/// for any revert after it — which is exactly why the workload never reads +/// mid-round. Here there is nothing after it, so it is safe, and it recovers a +/// round per agent that would otherwise be unjudgeable. +async fn close_last_rounds(ctx: &WorkloadContext, agents: &[String], rounds: &mut [RevertRound]) { + let mut last_of: std::collections::BTreeMap<&str, usize> = std::collections::BTreeMap::new(); + for (index, round) in rounds.iter().enumerate() { + if round.observed_after.is_none() { + let slot = last_of.entry(round.agent.as_str()).or_insert(index); + if rounds[*slot].round < round.round { + *slot = index; + } + } + } + let pending: Vec<(String, usize)> = last_of + .into_iter() + .map(|(agent, index)| (agent.to_string(), index)) + .collect(); + + info!( + "S7: closing {} unprobed rounds with a final read", + pending.len() + ); + for (agent, index) in pending { + if !agents.iter().any(|a| a == &agent) { + continue; + } + match workload::read_counter(ctx, &agent).await { + Ok(value) => rounds[index].observed_after = Some(value), + Err(e) => warn!("S7: could not read {agent} to close its last round: {e}"), + } + } +} diff --git a/integration-tests/src/chaos/scenarios/s8.rs b/integration-tests/src/chaos/scenarios/s8.rs index b9515ad16e..0c5170d436 100644 --- a/integration-tests/src/chaos/scenarios/s8.rs +++ b/integration-tests/src/chaos/scenarios/s8.rs @@ -155,6 +155,7 @@ pub async fn run( scheduled_selection: None, promise_selection: None, isolation_selection: None, + revert_selection: None, }, ); write_outputs(&result, &history, outputs)?; diff --git a/integration-tests/src/chaos/split.rs b/integration-tests/src/chaos/split.rs index a7ec87d8ff..ba34d30cf3 100644 --- a/integration-tests/src/chaos/split.rs +++ b/integration-tests/src/chaos/split.rs @@ -338,6 +338,16 @@ pub fn counter_subject<'a>(ctx: &'a WorkloadContext) -> Subject<'a> { } } +/// The counters component's revert agents, as S7 aims at them. +pub fn revert_subject<'a>(ctx: &'a WorkloadContext) -> Subject<'a> { + Subject { + scenario: "S7", + component: &ctx.counters, + agent_type: crate::chaos::workload::COUNTER_AGENT, + noun: "revert agents", + } +} + /// The promise component's waiters, as S11 aims at them. pub fn waiter_subject<'a>(ctx: &'a WorkloadContext) -> Subject<'a> { Subject { diff --git a/integration-tests/src/chaos/summary.rs b/integration-tests/src/chaos/summary.rs index 51d69c3aa6..d977d6f4b0 100644 --- a/integration-tests/src/chaos/summary.rs +++ b/integration-tests/src/chaos/summary.rs @@ -53,6 +53,7 @@ use crate::chaos::history::{Outcome, Phase, Stream}; use crate::chaos::ownership::OwnershipSample; use crate::chaos::probe::KeyProbe; use crate::chaos::reachability::ReachabilityReport; +use crate::chaos::truncation::TruncationReport; use crate::chaos::wakeups::WakeupReport; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; @@ -578,6 +579,10 @@ pub struct ChaosSummary { /// same reason as `scheduleFires`. #[serde(default, skip_serializing_if = "Option::is_none")] pub reachability: Option, + /// The truncation account, for scenarios that revert agent state. Absent + /// for scenarios that do not, for the same reason as `scheduleFires`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub truncation: Option, /// Shard-ownership samples, in the order they were taken. Empty for /// scenarios that do not sample executor assignments. /// @@ -710,6 +715,7 @@ impl ChaosSummary { schedule_fires: None, promise_wakeups: None, reachability: None, + truncation: None, ownership: Vec::new(), attention, notes: Vec::new(), @@ -783,6 +789,20 @@ impl ChaosSummary { self } + /// Attaches the truncation account and hoists everything it wants a human + /// to see into [`Self::attention`]. + /// + /// Same split as [`Self::with_schedule_fires`]. The line worth calling out + /// here is the inconclusive one: a kill that caught no revert in flight + /// proves nothing about crashing during a revert, and every clean number + /// underneath it describes reverts that completed either side of the fault. + pub fn with_truncation(mut self, report: TruncationReport) -> Self { + self.attention.extend(report.attention_lines()); + self.notes.extend(report.note_lines()); + self.truncation = Some(report); + self + } + /// Attaches the shard-ownership samples and hoists their findings into /// [`Self::attention`]. /// @@ -831,6 +851,12 @@ pub enum TerminationReason { /// owners is an agent whose state can fork, and there is no instant at /// which that is legitimate. ShardOwnershipViolated { findings: u64, first: String }, + /// A revert landed somewhere other than the two values it was allowed to. + /// Asserted rather than reported, and the only read-back in the suite that + /// earns that: the driver knows the counter's value before the revert and + /// exactly how many invocations it asked to take back, so there is no band + /// of doubt around the answer. See [`crate::chaos::truncation`]. + RevertTruncationViolated { findings: u64, first: String }, /// A scheduled action the platform accepted never fired, fired twice, or /// fired after being refused. Asserted rather than reported: unlike a /// count-based read-back, each of these is a statement about one named @@ -1127,7 +1153,18 @@ mod tests { let summary = ChaosSummary::build(&[], Vec::new(), Vec::new(), None); assert_eq!( summary.streams_without_readback, - vec![Stream::Ephemeral, Stream::Promise, Stream::PromiseWait] + // `Revert` is here for a different reason from the other three. + // Those keep no comparable durable state; a revert agent does, but + // some of its acknowledged work was deliberately taken back, so a + // generic counter comparison would report every reverted increment + // as lost. `crate::chaos::truncation` judges those agents exactly + // instead, which is strictly stronger. + vec![ + Stream::Ephemeral, + Stream::Promise, + Stream::PromiseWait, + Stream::Revert + ] ); } diff --git a/integration-tests/src/chaos/truncation.rs b/integration-tests/src/chaos/truncation.rs new file mode 100644 index 0000000000..d0e39c28dc --- /dev/null +++ b/integration-tests/src/chaos/truncation.rs @@ -0,0 +1,731 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Did every revert land on a boundary, or did one tear (GOL-371)? +//! +//! The sharpest oracle in the suite, because it has no band of doubt in it. +//! Every other read-back compares a counter against a *range* — the width of +//! the range is the operations whose fate the driver could not determine. Here +//! the driver knows the counter's value immediately before the revert, because +//! the last increment of the round returned it, and it knows exactly how many +//! invocations the revert was asked to take back. So afterwards there are +//! exactly **two** legal values and nothing in between: +//! +//! * `V` — the revert never committed +//! * `V - N` — the revert committed +//! +//! Anything else is a defect, and which kind it is says what went wrong. A +//! value strictly between the two is a truncation that tore. A value below +//! `V - N` took back more than it was asked for. A value above `V` means state +//! grew across a revert. +//! +//! ### Why a partial truncation should be impossible, and why that is worth +//! testing anyway +//! +//! Reading `Worker::revert` in the executor: `RevertLastInvocations` walks back +//! to the nth `AgentInvocationStarted` entry and then commits **one** +//! `OplogEntry::revert` marking the region deleted. One entry cannot tear, so +//! the truncation itself is atomic by construction. +//! +//! The window worth killing into is the one *around* it. Reverting takes +//! `lock_stopped_worker`, so the worker is stopped first; then the entry is +//! committed; then `reattach_worker_status` runs, because — in the executor's +//! own words — "this commit will detach the worker status, immediately reattach +//! it so we see the up to date status". An executor that dies between the +//! commit and the reattach has left durable state changed and in-memory state +//! stale, which is the same shape as the S11 promise defect: the durable half +//! landed and the half that tells anyone about it did not. +//! +//! So the two findings this account expects to be able to make, if the platform +//! has a bug here, are [`TruncationViolation::AcknowledgedButNotApplied`] and +//! its opposite — not a torn counter. + +use crate::chaos::history::Outcome; +use crate::chaos::reverts::RevertRound; +use crate::chaos::split::{FaultWindow, Group, PodSplit, Window}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +/// The most findings the report carries. +const MAX_FINDINGS: usize = 50; + +/// What went wrong with one revert. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum TruncationViolation { + /// The counter landed strictly between the pre-revert and post-revert + /// values: some of the invocations were taken back and some were not. + PartialTruncation, + /// More was taken back than the revert was asked for. + OverTruncation, + /// The counter is higher after the revert than it was before it. + Divergent, + /// The platform confirmed the revert and the state never moved. The same + /// shape as an accepted promise completion that never woke its waiter. + AcknowledgedButNotApplied, + /// The platform refused the revert and the state moved anyway. + RefusedButApplied, +} + +impl TruncationViolation { + pub fn as_str(self) -> &'static str { + match self { + TruncationViolation::PartialTruncation => "partial-truncation", + TruncationViolation::OverTruncation => "over-truncation", + TruncationViolation::Divergent => "divergent", + TruncationViolation::AcknowledgedButNotApplied => "acknowledged-but-not-applied", + TruncationViolation::RefusedButApplied => "refused-but-applied", + } + } +} + +/// One violation, against one named round. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TruncationFinding { + pub violation: TruncationViolation, + pub agent: String, + pub round: u32, + pub window: Window, + /// The counter before the revert, what it should have become, and what it + /// actually became. Carried so a finding can be read without the history. + pub before: u64, + pub expected_after_commit: u64, + pub observed: u64, + pub detail: String, +} + +/// Rounds and their verdicts for one (group, window) cell. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TruncationCell { + pub group: Group, + pub window: Window, + pub rounds: u64, + /// Reverts the platform confirmed that landed on the post-revert value. + pub applied: u64, + /// Reverts that left the agent exactly where it was. Legitimate only when + /// the platform never confirmed them. + pub not_applied: u64, + pub violations: u64, + /// Rounds the driver cannot judge: an increment that did not answer, so the + /// pre-revert value is unknown. + pub unjudgeable: u64, + /// Rounds no following increment ever probed. + pub unprobed: u64, +} + +/// Reverts the kill landed in the middle of. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RevertsCaught { + pub group: Group, + /// Reverts submitted before the kill that had not answered when it landed. + /// The population the scenario is actually about: a run that caught none of + /// them proves nothing about crash-during-revert, however clean it looks. + pub reverts: u64, + pub agents: usize, + pub confirmed: u64, + pub indeterminate: u64, + pub rejected: u64, +} + +/// The truncation account. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TruncationReport { + /// What each round was configured to do, so an archived finding can be read + /// without the suite YAML to hand. + pub increments_per_round: u32, + pub revert_invocations: u32, + pub rounds_recorded: u64, + pub reverts_confirmed: u64, + pub reverts_indeterminate: u64, + pub reverts_rejected: u64, + /// Confirmed reverts that landed exactly on the post-revert value. + pub applied_exactly: u64, + /// Reverts the driver never heard back about that applied anyway — doubt + /// the platform resolved in its own favour, not a defect. + pub indeterminate_that_applied: u64, + /// Reverts the driver never heard back about that did not apply. Also not a + /// defect: the call may never have landed. + pub indeterminate_that_did_not: u64, + pub unjudgeable: u64, + pub unprobed: u64, + pub cells: Vec, + pub caught_by_the_kill: Vec, + pub findings: Vec, + pub findings_omitted: u64, +} + +/// One round's verdict. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Verdict { + Applied, + NotApplied, + Violation(TruncationViolation), + Unjudgeable, + Unprobed, +} + +/// Judges one round against the two values it is allowed to have landed on. +fn judge(round: &RevertRound) -> Verdict { + let Some(before) = round.before_revert else { + return Verdict::Unjudgeable; + }; + let Some(observed) = round.observed_after else { + return Verdict::Unprobed; + }; + let expected = before.saturating_sub(round.asked_to_revert as u64); + + if observed == expected { + // Applied. Legitimate unless the platform said it refused. + if round.outcome == Outcome::Rejected { + return Verdict::Violation(TruncationViolation::RefusedButApplied); + } + return Verdict::Applied; + } + if observed == before { + // Not applied. Legitimate unless the platform said it had been. + if round.outcome == Outcome::Confirmed { + return Verdict::Violation(TruncationViolation::AcknowledgedButNotApplied); + } + return Verdict::NotApplied; + } + if observed > before { + return Verdict::Violation(TruncationViolation::Divergent); + } + if observed < expected { + return Verdict::Violation(TruncationViolation::OverTruncation); + } + Verdict::Violation(TruncationViolation::PartialTruncation) +} + +impl TruncationReport { + /// Builds the account from the rounds the workload recorded. + pub fn build( + rounds: &[RevertRound], + split: &PodSplit, + fault: Option, + increments_per_round: u32, + revert_invocations: u32, + ) -> Self { + let mut cells: BTreeMap<(Group, Window), TruncationCell> = BTreeMap::new(); + let mut findings: Vec = Vec::new(); + let mut report = TruncationReport { + increments_per_round, + revert_invocations, + rounds_recorded: rounds.len() as u64, + reverts_confirmed: 0, + reverts_indeterminate: 0, + reverts_rejected: 0, + applied_exactly: 0, + indeterminate_that_applied: 0, + indeterminate_that_did_not: 0, + unjudgeable: 0, + unprobed: 0, + cells: Vec::new(), + caught_by_the_kill: Vec::new(), + findings: Vec::new(), + findings_omitted: 0, + }; + + for round in rounds { + let group = split.group_of(&round.agent).unwrap_or(Group::Elsewhere); + let window = Window::of(round.submitted_at, fault); + let cell = cells + .entry((group, window)) + .or_insert_with(|| TruncationCell { + group, + window, + rounds: 0, + applied: 0, + not_applied: 0, + violations: 0, + unjudgeable: 0, + unprobed: 0, + }); + cell.rounds += 1; + + match round.outcome { + Outcome::Confirmed => report.reverts_confirmed += 1, + Outcome::Indeterminate => report.reverts_indeterminate += 1, + Outcome::Rejected => report.reverts_rejected += 1, + } + + match judge(round) { + Verdict::Applied => { + cell.applied += 1; + if round.outcome == Outcome::Confirmed { + report.applied_exactly += 1; + } else { + report.indeterminate_that_applied += 1; + } + } + Verdict::NotApplied => { + cell.not_applied += 1; + if round.outcome != Outcome::Confirmed { + report.indeterminate_that_did_not += 1; + } + } + Verdict::Unjudgeable => { + cell.unjudgeable += 1; + report.unjudgeable += 1; + } + Verdict::Unprobed => { + cell.unprobed += 1; + report.unprobed += 1; + } + Verdict::Violation(violation) => { + cell.violations += 1; + let before = round.before_revert.unwrap_or_default(); + let observed = round.observed_after.unwrap_or_default(); + let expected = before.saturating_sub(round.asked_to_revert as u64); + findings.push(TruncationFinding { + violation, + agent: round.agent.clone(), + round: round.round, + window, + before, + expected_after_commit: expected, + observed, + detail: detail_for(violation, round, before, expected, observed), + }); + } + } + } + + // ── What the kill landed in the middle of ─────────────────────────── + if let Some(window) = fault { + let mut by_group: BTreeMap> = BTreeMap::new(); + for round in rounds { + let Some(group) = split.group_of(&round.agent) else { + continue; + }; + let unresolved = round.completed_at.is_none_or(|at| at >= window.injected_at); + if round.submitted_at < window.injected_at && unresolved { + by_group.entry(group).or_default().push(round); + } + } + for (group, caught) in by_group { + let agents: std::collections::BTreeSet<&str> = + caught.iter().map(|r| r.agent.as_str()).collect(); + report.caught_by_the_kill.push(RevertsCaught { + group, + reverts: caught.len() as u64, + agents: agents.len(), + confirmed: count(&caught, Outcome::Confirmed), + indeterminate: count(&caught, Outcome::Indeterminate), + rejected: count(&caught, Outcome::Rejected), + }); + } + } + + report.cells = cells.into_values().collect(); + report.findings_omitted = findings.len().saturating_sub(MAX_FINDINGS) as u64; + findings.truncate(MAX_FINDINGS); + report.findings = findings; + report + } + + pub fn has_violations(&self) -> bool { + !self.findings.is_empty() || self.findings_omitted > 0 + } + + /// The lines that need a human. + pub fn attention_lines(&self) -> Vec { + let mut lines: Vec = self + .findings + .iter() + .map(|f| { + format!( + "S7 {}: {} round {} — {}", + f.violation.as_str(), + f.agent, + f.round, + f.detail + ) + }) + .collect(); + if self.findings_omitted > 0 { + lines.push(format!( + "S7: {} further truncation finding(s) were dropped from the report", + self.findings_omitted + )); + } + + // The S10 lesson: a run that caught none of the mechanism proves + // nothing about it, however clean every other number looks. + let caught: u64 = self + .caught_by_the_kill + .iter() + .filter(|c| c.group == Group::OnPod) + .map(|c| c.reverts) + .sum(); + if caught == 0 { + lines.push( + "S7: the kill caught no revert in flight on the targeted executor, so this run \ + says nothing about crashing during a revert. Every verdict below describes \ + reverts that completed either side of it." + .to_string(), + ); + } + lines + } + + /// Lines a reader needs in order to interpret the run. + pub fn note_lines(&self) -> Vec { + let mut lines = vec![format!( + "S7: {} rounds of {} increments then a revert of {}; {} confirmed, {} in doubt, {} \ + refused", + self.rounds_recorded, + self.increments_per_round, + self.revert_invocations, + self.reverts_confirmed, + self.reverts_indeterminate, + self.reverts_rejected + )]; + lines.push(format!( + "S7: {} confirmed reverts landed exactly on the post-revert value; {} in doubt \ + applied anyway, {} in doubt did not", + self.applied_exactly, self.indeterminate_that_applied, self.indeterminate_that_did_not + )); + if self.unjudgeable > 0 || self.unprobed > 0 { + lines.push(format!( + "S7: {} rounds could not be judged (an increment never answered) and {} were \ + never probed by a following increment", + self.unjudgeable, self.unprobed + )); + } + for caught in &self.caught_by_the_kill { + lines.push(format!( + "S7 {}: {} reverts across {} agents were unresolved when the kill landed — {} \ + confirmed, {} in doubt, {} refused", + caught.group.as_str(), + caught.reverts, + caught.agents, + caught.confirmed, + caught.indeterminate, + caught.rejected + )); + } + for cell in &self.cells { + lines.push(format!( + "S7 {} {}: {} rounds, {} applied, {} not applied, {} violations, {} unjudgeable, \ + {} unprobed", + cell.group.as_str(), + cell.window.as_str(), + cell.rounds, + cell.applied, + cell.not_applied, + cell.violations, + cell.unjudgeable, + cell.unprobed + )); + } + lines + } +} + +fn count(rounds: &[&RevertRound], outcome: Outcome) -> u64 { + rounds.iter().filter(|r| r.outcome == outcome).count() as u64 +} + +fn detail_for( + violation: TruncationViolation, + round: &RevertRound, + before: u64, + expected: u64, + observed: u64, +) -> String { + let asked = round.asked_to_revert; + match violation { + TruncationViolation::PartialTruncation => format!( + "the agent was worth {before}, a revert of {asked} invocations should have left it \ + at {expected}, and it came back at {observed} — between the two, so part of the \ + truncation landed and part did not" + ), + TruncationViolation::OverTruncation => format!( + "the agent was worth {before} and a revert of {asked} invocations left it at \ + {observed}, below the {expected} it asked for: more was taken back than requested" + ), + TruncationViolation::Divergent => format!( + "the agent was worth {before} before a revert of {asked} invocations and came back \ + at {observed}, higher than it started: state grew across a revert" + ), + TruncationViolation::AcknowledgedButNotApplied => format!( + "the platform confirmed a revert of {asked} invocations and the agent is still \ + worth {before}, not the {expected} it acknowledged" + ), + TruncationViolation::RefusedButApplied => format!( + "the platform refused a revert of {asked} invocations and the agent moved from \ + {before} to {observed} anyway" + ), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::{DateTime, TimeDelta, Utc}; + use std::collections::BTreeMap; + use test_r::test; + + const ON_POD: &str = "chaos-s7-revert-0000"; + const CONTROL: &str = "chaos-s7-revert-0001"; + + fn t0() -> DateTime { + DateTime::parse_from_rfc3339("2026-08-24T12:00:00Z") + .unwrap() + .with_timezone(&Utc) + } + + fn split() -> PodSplit { + PodSplit { + pod_address: "10.0.1.1:9000".to_string(), + pod_ip: "10.0.1.1".to_string(), + on_pod: vec![ON_POD.to_string()], + elsewhere: vec![CONTROL.to_string()], + targets_per_pod: BTreeMap::new(), + number_of_shards: 1024, + } + } + + fn fault() -> FaultWindow { + FaultWindow { + injected_at: t0(), + recovered_at: Some(t0() + TimeDelta::seconds(120)), + } + } + + /// One round: the agent was worth `before`, asked for 2 invocations back, + /// the platform answered `outcome`, and afterwards it read `observed`. + fn round( + agent: &str, + offset_secs: i64, + before: Option, + outcome: Outcome, + observed: Option, + ) -> RevertRound { + let submitted_at = t0() + TimeDelta::seconds(offset_secs); + RevertRound { + agent: agent.to_string(), + round: 0, + before_revert: before, + asked_to_revert: 2, + outcome, + submitted_at, + completed_at: Some(submitted_at + TimeDelta::milliseconds(80)), + observed_after: observed, + } + } + + fn build(rounds: &[RevertRound]) -> TruncationReport { + TruncationReport::build(rounds, &split(), Some(fault()), 4, 2) + } + + fn violations(report: &TruncationReport) -> Vec { + report.findings.iter().map(|f| f.violation).collect() + } + + /// The healthy shape: the platform said yes and the agent landed exactly on + /// the post-revert value. + #[test] + fn a_revert_that_landed_where_it_said_it_would_is_not_a_finding() { + let report = build(&[round(ON_POD, -10, Some(10), Outcome::Confirmed, Some(8))]); + assert!(violations(&report).is_empty(), "{:?}", report.findings); + assert_eq!(report.applied_exactly, 1); + assert!(!report.has_violations()); + } + + /// The headline finding this scenario exists to make: the counter landed + /// between the two values it was allowed to have. + #[test] + fn a_counter_between_the_two_legal_values_is_a_torn_truncation() { + // Worth 10, asked for 4 back so 6 was legal, came back at 8. + let mut r = round(ON_POD, -10, Some(10), Outcome::Confirmed, Some(8)); + r.asked_to_revert = 4; + let report = build(&[r]); + + assert_eq!( + violations(&report), + vec![TruncationViolation::PartialTruncation] + ); + let finding = &report.findings[0]; + assert_eq!(finding.before, 10); + assert_eq!(finding.expected_after_commit, 6); + assert_eq!(finding.observed, 8); + assert!(report.has_violations(), "this must be able to fail the run"); + } + + /// The same shape as an accepted promise completion that never woke its + /// waiter: the durable half was acknowledged and nothing moved. + #[test] + fn a_confirmed_revert_that_changed_nothing_is_a_finding() { + let report = build(&[round(ON_POD, -10, Some(10), Outcome::Confirmed, Some(10))]); + assert_eq!( + violations(&report), + vec![TruncationViolation::AcknowledgedButNotApplied] + ); + } + + /// Its opposite, and just as serious: refused, and applied regardless. + #[test] + fn a_refused_revert_that_applied_anyway_is_a_finding() { + let report = build(&[round(ON_POD, -10, Some(10), Outcome::Rejected, Some(8))]); + assert_eq!( + violations(&report), + vec![TruncationViolation::RefusedButApplied] + ); + } + + #[test] + fn taking_back_more_than_was_asked_for_is_a_finding() { + let report = build(&[round(ON_POD, -10, Some(10), Outcome::Confirmed, Some(5))]); + assert_eq!( + violations(&report), + vec![TruncationViolation::OverTruncation] + ); + } + + #[test] + fn state_growing_across_a_revert_is_a_finding() { + let report = build(&[round(ON_POD, -10, Some(10), Outcome::Confirmed, Some(11))]); + assert_eq!(violations(&report), vec![TruncationViolation::Divergent]); + } + + /// A revert the driver never heard back about is doubt, not damage. Both + /// answers are legitimate and neither is a finding. + #[test] + fn a_revert_in_doubt_may_land_either_way_without_being_a_finding() { + let report = build(&[ + round(ON_POD, -10, Some(10), Outcome::Indeterminate, Some(8)), + round(CONTROL, -10, Some(10), Outcome::Indeterminate, Some(10)), + ]); + assert!(violations(&report).is_empty(), "{:?}", report.findings); + assert_eq!(report.indeterminate_that_applied, 1); + assert_eq!(report.indeterminate_that_did_not, 1); + } + + /// A round whose increments never answered has no pre-revert value, so it + /// cannot be judged. Counted, never guessed at. + #[test] + fn a_round_whose_increments_never_answered_is_not_judged() { + let report = build(&[round(ON_POD, -10, None, Outcome::Confirmed, Some(8))]); + assert_eq!(report.unjudgeable, 1); + assert!(report.findings.is_empty()); + } + + /// The last round an agent ran has nothing after it to probe with. It is + /// counted rather than assumed clean. + #[test] + fn a_round_no_increment_ever_probed_is_counted_as_unprobed() { + let report = build(&[round(ON_POD, -10, Some(10), Outcome::Confirmed, None)]); + assert_eq!(report.unprobed, 1); + assert!(report.findings.is_empty()); + } + + /// The S10 lesson: a kill that caught none of the mechanism proves nothing + /// about it, and the run has to say so rather than read as clean. + #[test] + fn a_kill_that_caught_no_revert_says_the_run_proved_nothing() { + // Both rounds resolved well before the kill. + let report = build(&[ + round(ON_POD, -100, Some(10), Outcome::Confirmed, Some(8)), + round(CONTROL, -100, Some(10), Outcome::Confirmed, Some(8)), + ]); + assert!(report.caught_by_the_kill.is_empty()); + assert!( + report + .attention_lines() + .iter() + .any(|l| l.contains("caught no revert in flight")), + "attention was {:?}", + report.attention_lines() + ); + } + + /// A revert still unresolved when the pod died is the population the whole + /// scenario is about, and it has to be counted apart from the rest. + #[test] + fn reverts_unresolved_when_the_pod_died_are_reported_separately() { + let mut caught = round(ON_POD, -1, Some(10), Outcome::Indeterminate, Some(10)); + caught.completed_at = Some(t0() + TimeDelta::seconds(30)); + let report = build(&[caught]); + + let entry = report + .caught_by_the_kill + .iter() + .find(|c| c.group == Group::OnPod) + .expect("the kill caught a revert"); + assert_eq!(entry.reverts, 1); + assert_eq!(entry.agents, 1); + assert_eq!(entry.indeterminate, 1); + assert!( + !report + .attention_lines() + .iter() + .any(|l| l.contains("caught no revert in flight")) + ); + } + + /// Rounds are split by which executor owned the agent and which side of the + /// kill they fell on, so a control group cannot hide a disturbed one. + #[test] + fn rounds_are_split_by_group_and_window() { + let report = build(&[ + round(ON_POD, -10, Some(10), Outcome::Confirmed, Some(8)), + round(ON_POD, 10, Some(12), Outcome::Confirmed, Some(10)), + round(CONTROL, 10, Some(10), Outcome::Confirmed, Some(8)), + ]); + let cell = |g, w| { + report + .cells + .iter() + .find(|c| c.group == g && c.window == w) + .cloned() + }; + assert_eq!(cell(Group::OnPod, Window::BeforeFault).unwrap().rounds, 1); + assert_eq!(cell(Group::OnPod, Window::DuringFault).unwrap().rounds, 1); + assert_eq!( + cell(Group::Elsewhere, Window::DuringFault).unwrap().rounds, + 1 + ); + } + + #[test] + fn findings_beyond_the_cap_are_counted_rather_than_carried() { + let rounds: Vec = (0..MAX_FINDINGS + 7) + .map(|_| round(ON_POD, -10, Some(10), Outcome::Confirmed, Some(10))) + .collect(); + let report = build(&rounds); + assert_eq!(report.findings.len(), MAX_FINDINGS); + assert_eq!(report.findings_omitted, 7); + assert!(report.has_violations()); + } + + /// A finding is read by an operator mid-window, so it has to state the + /// three numbers that make it interpretable without the history to hand. + #[test] + fn a_finding_states_the_numbers_that_make_it_readable() { + let report = build(&[round(ON_POD, -10, Some(10), Outcome::Confirmed, Some(10))]); + let detail = &report.findings[0].detail; + assert!(detail.contains("10"), "{detail}"); + assert!(detail.contains('8'), "{detail}"); + assert!( + !detail.contains("Some("), + "Option formatting leaked: {detail}" + ); + } +} diff --git a/integration-tests/src/chaos/workload.rs b/integration-tests/src/chaos/workload.rs index b847a5548c..473b6974c4 100644 --- a/integration-tests/src/chaos/workload.rs +++ b/integration-tests/src/chaos/workload.rs @@ -202,6 +202,20 @@ struct AttemptResult { class: Option, } +/// What one operation ended up as. +/// +/// Returned rather than only recorded because one scenario has to chain on it: +/// [`crate::chaos::reverts`] judges a revert by the value the *next* increment +/// reports, so it needs each operation's answer in hand rather than having to +/// go looking for its own record in a history every other emitter is also +/// appending to. +#[derive(Debug, Clone, Copy)] +pub struct OperationOutcome { + pub outcome: Outcome, + /// The value the operation returned, for the methods that return one. + pub value: Option, +} + /// Runs one operation with the configured bounded, same-key retry, and records /// it in the history. /// @@ -217,7 +231,8 @@ pub(crate) async fn run_operation( method: &str, key: String, invoke: F, -) where +) -> OperationOutcome +where F: Fn(IdempotencyKey) -> Fut, Fut: std::future::Future>>, { @@ -337,6 +352,44 @@ pub(crate) async fn run_operation( error_class: last.class, attempt_log, }); + + OperationOutcome { + outcome, + value: last.value, + } +} + +/// One `Counter.increment`, recorded under `stream`. +/// +/// Split out of [`submit_one`] so [`crate::chaos::reverts`] can drive +/// increments on its own stream through exactly the same retry rule and +/// classification, rather than growing a second copy of them. +pub(crate) async fn increment_counter( + ctx: &WorkloadContext, + stream: Stream, + agent: &str, + key: String, +) -> OperationOutcome { + let parsed: ParsedAgentId = agent_id!(COUNTER_AGENT, agent.to_string()); + let ctx2 = ctx.clone(); + run_operation(ctx, stream, agent.to_string(), "increment", key, |k| { + let ctx = ctx2.clone(); + let parsed = parsed.clone(); + async move { + let value = ctx + .user + .invoke_and_await_agent_with_key( + &ctx.counters, + &parsed, + &k, + "increment", + data_value!(), + ) + .await?; + Ok(as_u32(value)) + } + }) + .await } /// Extracts a `u32` return value, if the agent returned one. Absent values are @@ -497,6 +550,13 @@ pub(crate) async fn submit_one(ctx: &WorkloadContext, stream: Stream, index: u32 Stream::PromiseWait => { warn!("Chaos mixed workload cannot drive the waiter stream; see chaos::waiters"); } + // Driven by `crate::chaos::reverts`: a round is a run of increments + // followed by a revert that takes some of them back, and the value the + // *next* round's first increment returns is what says whether that + // revert landed. A shared rate cannot express that ordering. + Stream::Revert => { + warn!("Chaos mixed workload cannot drive the revert stream; see chaos::reverts"); + } Stream::Durable => { let agent = ctx.agent_name(Stream::Durable, index); let key = ctx.idempotency_key(&agent, seq); From 638f09ce480c17cfe866791f523cdf5c2347866e Mon Sep 17 00:00:00 2001 From: kmatasfp <33095685+kmatas@users.noreply.github.com> Date: Mon, 24 Aug 2026 17:30:42 -0700 Subject: [PATCH 16/40] Add S6 chaos scenario driver and resurrection account --- golem-test-framework/src/benchmark/config.rs | 2 + .../chaos_suites/cloud-chaos.yaml | 67 ++ integration-tests/src/benchmarks/all.rs | 4 + integration-tests/src/chaos/deletions.rs | 306 ++++++++ integration-tests/src/chaos/history.rs | 13 +- integration-tests/src/chaos/mod.rs | 86 ++- integration-tests/src/chaos/result.rs | 17 +- integration-tests/src/chaos/resurrection.rs | 708 ++++++++++++++++++ integration-tests/src/chaos/scenarios/mod.rs | 6 + integration-tests/src/chaos/scenarios/s1.rs | 1 + integration-tests/src/chaos/scenarios/s10.rs | 1 + integration-tests/src/chaos/scenarios/s11.rs | 1 + integration-tests/src/chaos/scenarios/s12.rs | 1 + integration-tests/src/chaos/scenarios/s13.rs | 1 + integration-tests/src/chaos/scenarios/s3.rs | 1 + integration-tests/src/chaos/scenarios/s5.rs | 1 + integration-tests/src/chaos/scenarios/s6.rs | 475 ++++++++++++ integration-tests/src/chaos/scenarios/s7.rs | 1 + integration-tests/src/chaos/scenarios/s8.rs | 1 + integration-tests/src/chaos/split.rs | 10 + integration-tests/src/chaos/summary.rs | 28 +- integration-tests/src/chaos/workload.rs | 6 + 22 files changed, 1732 insertions(+), 5 deletions(-) create mode 100644 integration-tests/src/chaos/deletions.rs create mode 100644 integration-tests/src/chaos/resurrection.rs create mode 100644 integration-tests/src/chaos/scenarios/s6.rs diff --git a/golem-test-framework/src/benchmark/config.rs b/golem-test-framework/src/benchmark/config.rs index 24bdfd6471..a0c4044e89 100644 --- a/golem-test-framework/src/benchmark/config.rs +++ b/golem-test-framework/src/benchmark/config.rs @@ -250,6 +250,8 @@ pub enum ChaosScenarioArg { S3, /// Executor pod kill while agents are having their state reverted. S7, + /// Executor pod kill while agents are being deleted. + S6, } /// Density subcommand action. diff --git a/integration-tests/chaos_suites/cloud-chaos.yaml b/integration-tests/chaos_suites/cloud-chaos.yaml index 645fd7571a..6adf181901 100644 --- a/integration-tests/chaos_suites/cloud-chaos.yaml +++ b/integration-tests/chaos_suites/cloud-chaos.yaml @@ -700,3 +700,70 @@ scenarios: delaySecs: 5 signalTimeoutSecs: 1800 + + # S6 — executor pod kill during agent deletion (GOL-372). + # + # S7 asks the platform to forget some of an agent's work. S6 asks it to forget + # the agent, and kills the executor while it is doing so. + # + # The same two-value oracle, one step further: invoking a deleted id creates a + # *new* agent, so when a slot is used again it must report either 1 (the + # deletion took) or its old value plus one (it did not). Neither is a defect + # alone — a lost response leaves the question open — but "the platform + # confirmed the deletion and the agent is still worth what it was" is the + # resurrection this scenario is named for, and it fails the run. + # + # Worth knowing before reading a result: the happy path is already defended. + # `Worker::start_deleting` stops the background status flush and checkpointer + # first, specifically so neither can resurrect the cached status after the + # durable removal. The question is whether that defence survives the pod dying + # between the mark and the removal. + - code: S6 + name: executor-crash-during-deletion + enabled: true + + fault: + kind: pod-kill + target: worker-executor + # The workflow narrows the selector to the pod the driver named; `one` + # stays as the belt-and-braces bound, as in every other pinned scenario. + mode: one + durationSecs: 60 + + phases: + baselineSecs: 180 + faultSecs: 120 + # Rounds continue throughout, so every slot is reused several times after + # the kill and a round it disturbed is followed by ones it did not. + recoverySecs: 300 + + delete: + # 200 slots, split by shard ownership. Also the resolution of the report: + # a resurrection localises to one slot out of two hundred. + agents: 200 + # Three increments before each delete, so an agent is worth 3 when it + # goes. More than one is load-bearing rather than arbitrary: with a single + # increment a fresh agent and a resurrected one both read 1 and the oracle + # cannot tell them apart. The driver refuses to start below 2. + incrementsPerRound: 3 + # Between rounds. A round is four operations, so this keeps the delete a + # meaningful share of each slot's cycle and gives the kill something to + # land in. + intervalMillis: 500 + # What recovering a slot may cost once its executor is back. Recorded, not + # asserted. + recoveryBudgetSecs: 60 + + retryPolicy: + # Applies to the increments only. **The delete is never retried**, for a + # reason specific to this scenario: `delete_worker_internal` starts with a + # metadata lookup and returns `worker_not_found` when there is nothing + # there, so deleting twice does not succeed twice. A delete whose response + # was lost but which landed would come back on retry as a refusal, and the + # run would record "refused, and gone anyway" — one of the violations this + # scenario exists to detect. See `chaos/deletions.rs`. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 diff --git a/integration-tests/src/benchmarks/all.rs b/integration-tests/src/benchmarks/all.rs index 497189d5e3..d48ac081fd 100644 --- a/integration-tests/src/benchmarks/all.rs +++ b/integration-tests/src/benchmarks/all.rs @@ -596,6 +596,7 @@ async fn run_chaos( ChaosScenarioArg::S11 => chaos::ScenarioCode::S11, ChaosScenarioArg::S3 => chaos::ScenarioCode::S3, ChaosScenarioArg::S7 => chaos::ScenarioCode::S7, + ChaosScenarioArg::S6 => chaos::ScenarioCode::S6, }; let config = suite .scenario(code, allow_disabled) @@ -639,6 +640,9 @@ async fn run_chaos( chaos::ScenarioCode::S7 => { chaos::scenarios::s7::run(&config, &manifest, &deps, &signals, &outputs).await } + chaos::ScenarioCode::S6 => { + chaos::scenarios::s6::run(&config, &manifest, &deps, &signals, &outputs).await + } }; deps.kill_all().await; diff --git a/integration-tests/src/chaos/deletions.rs b/integration-tests/src/chaos/deletions.rs new file mode 100644 index 0000000000..3cb6cbfaf5 --- /dev/null +++ b/integration-tests/src/chaos/deletions.rs @@ -0,0 +1,306 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Agents that are built up and then deleted outright (GOL-372). +//! +//! One emitter per agent slot, running rounds. A round is +//! `increments_per_round` increments followed by one `delete`. Then the slot is +//! used again: invoking a deleted agent id creates a **new** agent, so the next +//! round's first increment returns `1` if the deletion took and `V + 1` if the +//! old agent is still there. +//! +//! That is the same probe [`crate::chaos::reverts`] uses and it works for the +//! same reason: the round's last increment reports the value, so there are +//! exactly two legal answers afterwards and nothing between them. +//! +//! ### The delete must never be retried +//! +//! `delete_worker_internal` in the executor starts with `get_latest_metadata` +//! and returns `worker_not_found` when there is nothing there. So deleting an +//! agent twice does **not** return success twice: the second call is an error. +//! +//! A delete whose response was lost but which actually landed would therefore, +//! on retry, come back as a refusal — and the driver would record "the platform +//! refused, and the agent is gone anyway", which is one of the violations this +//! scenario exists to detect. The retry is switched off for the delete alone, +//! exactly as it is for a revert, and left on for the increments around it. +//! +//! ### What the kill is aimed at +//! +//! Deleting is four steps: interrupt the running worker, `start_deleting`, +//! remove it from the worker service, remove it from the active set. Only the +//! third is durable. `start_deleting` exists to stop a background status flush +//! from — in the executor's own words — "resurrecting the cached status" after +//! the removal, so the happy path is already defended against exactly the thing +//! this scenario is named for. The question is whether that defence survives the +//! pod dying between the mark and the removal. + +use crate::chaos::history::{Outcome, Stream}; +use crate::chaos::workload::{self, WorkloadContext}; +use crate::chaos::{DeleteConfig, RetryPolicy}; +use chrono::{DateTime, Utc}; +use golem_test_framework::dsl::TestDsl; +use serde::{Deserialize, Serialize}; +use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use tokio::task::JoinSet; +use tracing::info; + +/// What a freshly created agent's first increment returns. +/// +/// Named rather than written as `1` at the comparison, because it is the whole +/// definition of "the deletion took": an agent id that was deleted and then +/// invoked again is a *new* agent, and a new counter starts from nothing. +pub const FIRST_VALUE_OF_A_NEW_AGENT: u64 = 1; + +/// One round, as the driver observed it. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DeleteRound { + pub agent: String, + pub round: u32, + /// The value the round's last increment returned, so what the agent was + /// worth immediately before it was deleted. `None` when an increment did + /// not answer, which leaves the round unjudgeable rather than failed. + pub before_delete: Option, + /// What the delete call itself returned. + pub outcome: Outcome, + pub submitted_at: DateTime, + pub completed_at: Option>, + /// What the next round's first increment returned. `1` means the id came + /// back as a new agent; `before_delete + 1` means the old one is still + /// there. + pub observed_after: Option, +} + +/// A running deletion workload. +pub struct DeleteHandle { + stop: Arc, + tasks: JoinSet<()>, + submitted: Arc, + rounds: Arc>>, +} + +impl DeleteHandle { + pub fn submitted(&self) -> u64 { + self.submitted.load(Ordering::Relaxed) + } + + pub fn rounds(&self) -> Vec { + self.rounds.lock().map(|r| r.clone()).unwrap_or_default() + } + + pub async fn stop(mut self) -> Vec { + self.stop.store(1, Ordering::Relaxed); + while self.tasks.join_next().await.is_some() {} + let rounds = self.rounds(); + info!( + "Chaos delete workload stopped after {} operations across {} rounds", + self.submitted(), + rounds.len() + ); + rounds + } +} + +/// The agent slots a run of `count` emitters drives, in index order. +pub fn agent_names(ctx: &WorkloadContext, count: u32) -> Vec { + (0..count) + .map(|index| ctx.agent_name(Stream::Delete, index)) + .collect() +} + +/// Builds one agent up, deletes it, and checks it came back new. +/// +/// Run once before the baseline, against a throwaway id. It exists because the +/// first S11 run did not have its equivalent: a scenario whose premise is wrong +/// spends its whole baseline before the numbers say so, and the maintenance +/// window is gone. If deletion does not behave the way this whole account +/// assumes, this fails in seconds with the platform's own error. +pub async fn smoke_round(ctx: &WorkloadContext, config: &DeleteConfig) -> anyhow::Result<()> { + let agent = format!("{}-delete-smoke", ctx.key_prefix); + let mut value = 0; + for step in 0..config.increments_per_round.max(1) { + value = workload::increment_counter( + ctx, + Stream::Delete, + &agent, + ctx.idempotency_key(&agent, step as u64), + ) + .await + .value + .map(u64::from) + .ok_or_else(|| { + anyhow::anyhow!("smoke round: increment {step} on {agent} did not answer") + })?; + } + + let agent_id = workload::counter_agent_id(ctx, &agent); + ctx.user + .delete_worker(&agent_id) + .await + .map_err(|e| anyhow::anyhow!("smoke round: deleting {agent} failed: {e:#}"))?; + + let after = workload::increment_counter( + ctx, + Stream::Delete, + &agent, + ctx.idempotency_key(&agent, u64::from(config.increments_per_round) + 1), + ) + .await + .value + .map(u64::from) + .ok_or_else(|| anyhow::anyhow!("smoke round: {agent} did not answer after being deleted"))?; + + if after != FIRST_VALUE_OF_A_NEW_AGENT { + anyhow::bail!( + "smoke round: {agent} was worth {value}, was deleted, and its next increment \ + returned {after} rather than {FIRST_VALUE_OF_A_NEW_AGENT}. Deleting an agent does \ + not behave the way this scenario's whole account assumes, so the run would report \ + a resurrection on every round." + ); + } + info!("S6: smoke round passed — a deleted agent came back as a new one"); + Ok(()) +} + +/// Starts one emitter per agent slot. +pub fn start(ctx: WorkloadContext, config: &DeleteConfig) -> DeleteHandle { + let stop = Arc::new(AtomicU8::new(0)); + let submitted = Arc::new(AtomicU64::new(0)); + let rounds: Arc>> = Arc::new(Mutex::new(Vec::new())); + let mut tasks = JoinSet::new(); + + info!( + "Chaos delete workload starting: {} emitters, {} increments then a delete per round, \ + {:?} between rounds", + config.agents, + config.increments_per_round, + config.interval() + ); + + for index in 0..config.agents { + let ctx = ctx.clone(); + let stop = stop.clone(); + let submitted = submitted.clone(); + let rounds = rounds.clone(); + let config = config.clone(); + + tasks.spawn(async move { + let agent = ctx.agent_name(Stream::Delete, index); + let mut round: u32 = 0; + let mut pending: Option = None; + + while stop.load(Ordering::Relaxed) == 0 { + let mut last_value = None; + let mut all_answered = true; + for step in 0..config.increments_per_round { + if stop.load(Ordering::Relaxed) != 0 { + break; + } + let seq = round as u64 * config.increments_per_round as u64 + step as u64; + submitted.fetch_add(1, Ordering::Relaxed); + let value = workload::increment_counter( + &ctx, + Stream::Delete, + &agent, + ctx.idempotency_key(&agent, seq), + ) + .await + .value + .map(u64::from); + + // The first increment of a round says what the previous + // round's delete left behind. + if let Some(slot) = pending.take() + && let Some(observed) = value + && let Ok(mut rounds) = rounds.lock() + && let Some(entry) = rounds.get_mut(slot) + { + entry.observed_after = Some(observed); + } + + match value { + Some(v) => last_value = Some(v), + None => all_answered = false, + } + } + if stop.load(Ordering::Relaxed) != 0 { + break; + } + + let submitted_at = Utc::now(); + submitted.fetch_add(1, Ordering::Relaxed); + let outcome = delete_once(&ctx, &agent, round).await; + + if let Ok(mut rounds) = rounds.lock() { + rounds.push(DeleteRound { + agent: agent.clone(), + round, + before_delete: all_answered.then_some(last_value).flatten(), + outcome, + submitted_at, + completed_at: Some(Utc::now()), + observed_after: None, + }); + pending = Some(rounds.len() - 1); + } + + round += 1; + if stop.load(Ordering::Relaxed) == 0 { + tokio::time::sleep(config.interval()).await; + } + } + }); + } + + DeleteHandle { + stop, + tasks, + submitted, + rounds, + } +} + +/// One delete, with retries switched off. See the module docs. +async fn delete_once(ctx: &WorkloadContext, agent: &str, round: u32) -> Outcome { + let mut once = ctx.clone(); + once.retry = RetryPolicy { + transport_only: true, + max_retries: 0, + delay_secs: 0, + }; + + let key = format!("{agent}-delete-{round:08}"); + let agent_id = workload::counter_agent_id(&once, agent); + let ctx2 = once.clone(); + + workload::run_operation( + &once, + Stream::Delete, + agent.to_string(), + "delete", + key, + |_| { + let ctx = ctx2.clone(); + let agent_id = agent_id.clone(); + async move { + ctx.user.delete_worker(&agent_id).await?; + Ok(None) + } + }, + ) + .await + .outcome +} diff --git a/integration-tests/src/chaos/history.rs b/integration-tests/src/chaos/history.rs index 302f661e68..fc1d380cf9 100644 --- a/integration-tests/src/chaos/history.rs +++ b/integration-tests/src/chaos/history.rs @@ -67,6 +67,15 @@ pub enum Stream { /// one known executor, so mixing them into the durable population would /// blur two different experiments. PinnedHttp, + /// `Counter.increment` rounds on agents that are then deleted outright + /// (GOL-372). + /// + /// Excluded from the generic read-back for the same reason as + /// [`Stream::Revert`], one step further: a deleted agent's counter is + /// *supposed* to be gone, so comparing it against everything confirmed + /// against it would report the whole agent as lost work. See + /// [`crate::chaos::resurrection`]. + Delete, /// `Counter.increment` rounds that are deliberately taken back again, and /// the `revert` calls that take them (GOL-371). /// @@ -96,6 +105,7 @@ impl Stream { Stream::Quota => "quota", Stream::PinnedHttp => "pinned-http", Stream::PromiseWait => "promise-wait", + Stream::Delete => "delete", Stream::Revert => "revert", } } @@ -122,7 +132,7 @@ impl Stream { ) } - pub const ALL: [Stream; 8] = [ + pub const ALL: [Stream; 9] = [ Stream::Durable, Stream::Ephemeral, Stream::Scheduled, @@ -131,6 +141,7 @@ impl Stream { Stream::PinnedHttp, Stream::PromiseWait, Stream::Revert, + Stream::Delete, ]; } diff --git a/integration-tests/src/chaos/mod.rs b/integration-tests/src/chaos/mod.rs index ef9e240cc8..32e97a4a88 100644 --- a/integration-tests/src/chaos/mod.rs +++ b/integration-tests/src/chaos/mod.rs @@ -34,6 +34,7 @@ //! engine here — see [`summary`] for what is measured and the narrow set of //! conditions that fail a run outright. +pub mod deletions; pub mod errors; pub mod fires; pub mod history; @@ -43,6 +44,7 @@ pub mod prep; pub mod probe; pub mod reachability; pub mod result; +pub mod resurrection; pub mod reverts; pub mod scenarios; pub mod scheduled; @@ -82,6 +84,8 @@ pub enum ScenarioCode { S3, /// Executor pod kill while agents are having their state reverted. S7, + /// Executor pod kill while agents are being deleted. + S6, } impl ScenarioCode { @@ -96,16 +100,18 @@ impl ScenarioCode { ScenarioCode::S11 => "S11", ScenarioCode::S3 => "S3", ScenarioCode::S7 => "S7", + ScenarioCode::S6 => "S6", } } /// Every scenario this driver implements. The suite YAML is checked against /// this list, so a scenario cannot be enabled in YAML without code behind /// it, nor implemented without an operational switch in front of it. - pub const ALL: [ScenarioCode; 9] = [ + pub const ALL: [ScenarioCode; 10] = [ ScenarioCode::S1, ScenarioCode::S3, ScenarioCode::S5, + ScenarioCode::S6, ScenarioCode::S7, ScenarioCode::S8, ScenarioCode::S10, @@ -535,6 +541,52 @@ impl RevertConfig { } } +/// Shape of the deletion workload (GOL-372). +/// +/// The seventh experiment shape, and one step past [`RevertConfig`]. A revert +/// asks the platform to forget some of an agent's work; this asks it to forget +/// the agent. Each slot builds a counter up, deletes it, and is used again — +/// invoking a deleted id creates a new agent, so the next round's first +/// increment says which of the two things happened. +/// +/// The failure mode it is named for has a defence in the executor already: +/// `start_deleting` stops a background status flush from "resurrecting the +/// cached status" after the durable removal. So the question is not whether +/// anyone thought about it, but whether the defence survives the pod dying +/// between the mark and the removal. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DeleteConfig { + /// Agent slots running rounds, split by shard ownership around the executor + /// the kill is aimed at. Also the resolution of the report: a resurrection + /// localises to one slot out of this many. + pub agents: u32, + /// Increments before each delete. + /// + /// More than one, so that a *partial* survival is observable at all. The + /// two legal answers are always distinguishable — a fresh agent reports 1 + /// and a survivor reports `before + 1`, which never collide — but at one + /// increment there is no value *between* them, so a slot that came back + /// carrying some of a state it should have lost has nowhere to land and + /// [`crate::chaos::resurrection::ResurrectionViolation::PartialState`] can + /// never fire. Three of them leaves room for it. + pub increments_per_round: u32, + /// Milliseconds a slot waits between rounds. + pub interval_millis: u64, + /// What recovering a deleted agent's slot may cost. Recorded rather than + /// asserted, like every other budget in the suite. + pub recovery_budget_secs: u64, +} + +impl DeleteConfig { + pub fn interval(&self) -> Duration { + Duration::from_millis(self.interval_millis) + } + pub fn recovery_budget(&self) -> Duration { + Duration::from_secs(self.recovery_budget_secs) + } +} + /// One step of the executor scale schedule the workflow runs during the fault. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -619,6 +671,9 @@ pub struct ScenarioConfig { /// The revert workload. Absent for scenarios that do not run one. #[serde(default)] pub revert: Option, + /// The deletion workload. Absent for scenarios that do not run one. + #[serde(default)] + pub delete: Option, /// Shard-ownership oracle settings. Absent for scenarios that do not sample /// executor assignments. #[serde(default)] @@ -732,6 +787,29 @@ impl ScenarioConfig { Ok(config) } + /// The deletion workload block. See [`Self::require_workload`]. + pub fn require_delete(&self) -> anyhow::Result<&DeleteConfig> { + let config = self.delete.as_ref().ok_or_else(|| { + anyhow::anyhow!( + "chaos scenario {} needs a `delete` block in the suite YAML", + self.code + ) + })?; + // At one increment the two legal answers are adjacent — 1 and 2 — so + // nothing can land between them and the partial-state violation is + // structurally unobservable. A third of the oracle would be blind, and + // every run would look clean on that axis by construction. + if config.increments_per_round < 2 { + anyhow::bail!( + "chaos scenario {}: incrementsPerRound is {}, which leaves no value between \ + a fresh agent and a survivor, so a partial state could never be observed", + self.code, + config.increments_per_round + ); + } + Ok(config) + } + /// The pinned workload block. See [`Self::require_workload`]. pub fn require_pinned(&self) -> anyhow::Result<&PinnedConfig> { self.pinned.as_ref().ok_or_else(|| { @@ -869,6 +947,7 @@ mod tests { promise: None, isolation: None, revert: None, + delete: None, ownership: None, scale_during_fault: None, retry_policy: RetryPolicy::default(), @@ -903,6 +982,7 @@ mod tests { scheduled: None, promise: None, isolation: None, + delete: None, revert: Some(RevertConfig { agents: 10, increments_per_round: increments, @@ -1009,6 +1089,7 @@ mod tests { assert_eq!(ScenarioCode::parse("s1"), Some(ScenarioCode::S1)); assert_eq!(ScenarioCode::parse("s3"), Some(ScenarioCode::S3)); assert_eq!(ScenarioCode::parse("s7"), Some(ScenarioCode::S7)); + assert_eq!(ScenarioCode::parse("s6"), Some(ScenarioCode::S6)); assert_eq!(ScenarioCode::parse("S99"), None); } @@ -1047,6 +1128,9 @@ mod tests { ScenarioCode::S7 => { entry.require_revert().unwrap(); } + ScenarioCode::S6 => { + entry.require_delete().unwrap(); + } } } } diff --git a/integration-tests/src/chaos/result.rs b/integration-tests/src/chaos/result.rs index 3f03fa3104..38e0761708 100644 --- a/integration-tests/src/chaos/result.rs +++ b/integration-tests/src/chaos/result.rs @@ -29,8 +29,8 @@ use crate::chaos::scheduled::ScheduledSelection; use crate::chaos::split::PodSplit; use crate::chaos::summary::{ChaosSummary, TerminationReason}; use crate::chaos::{ - FaultConfig, IsolationConfig, PinnedConfig, PromiseConfig, RetryPolicy, RevertConfig, - ScheduledConfig, WorkloadConfig, + DeleteConfig, FaultConfig, IsolationConfig, PinnedConfig, PromiseConfig, RetryPolicy, + RevertConfig, ScheduledConfig, WorkloadConfig, }; use chrono::{DateTime, Utc}; use golem_test_framework::benchmark::RunMetadata; @@ -169,6 +169,13 @@ pub struct ChaosResult { /// Present only for S7. #[serde(default, skip_serializing_if = "Option::is_none")] pub revert_selection: Option, + /// The deletion workload the run was configured with, if any. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub delete: Option, + /// How the agent slots divided around the executor the kill was aimed at. + /// Present only for S6. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub delete_selection: Option, pub retry_policy: RetryPolicy, pub scope: RunScope, pub summary: ChaosSummary, @@ -244,6 +251,8 @@ mod tests { isolation_selection: None, revert: None, revert_selection: None, + delete: None, + delete_selection: None, retry_policy: RetryPolicy::default(), scope: RunScope { environment_id: "env-1".to_string(), @@ -839,6 +848,8 @@ mod sample_artifact { isolation_selection: None, revert: None, revert_selection: None, + delete: None, + delete_selection: None, retry_policy: RetryPolicy::default(), scope: RunScope { environment_id: "0192f000-0000-7000-8000-000000000001".to_string(), @@ -1057,6 +1068,8 @@ mod sample_artifact { isolation_selection: None, revert: None, revert_selection: None, + delete: None, + delete_selection: None, retry_policy: RetryPolicy::default(), scope: RunScope { environment_id: "0192f000-0000-7000-8000-000000000001".to_string(), diff --git a/integration-tests/src/chaos/resurrection.rs b/integration-tests/src/chaos/resurrection.rs new file mode 100644 index 0000000000..06bffd6c91 --- /dev/null +++ b/integration-tests/src/chaos/resurrection.rs @@ -0,0 +1,708 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Did every deleted agent stay deleted (GOL-372)? +//! +//! The same two-value oracle [`crate::chaos::truncation`] uses, one step +//! further. There a round asked the platform to forget some of an agent's work; +//! here it asks it to forget the agent. Invoking a deleted id creates a **new** +//! agent, so the next round's first increment has exactly two legal answers: +//! +//! * `1` — the deletion took, and this is a fresh agent counting from nothing +//! * `V + 1` — the deletion did not take, and the old agent is still there +//! +//! Anything else means an agent came back carrying part of a state it was +//! supposed to have lost. +//! +//! ### Which of the two is the finding +//! +//! Neither, on its own. Both are legitimate outcomes of a delete the driver +//! never heard back about — a lost response leaves the question genuinely open. +//! What makes one a defect is the platform's own answer next to it: +//! +//! * confirmed, and the agent is still worth `V` → **resurrection**. The +//! platform said the agent was gone and it is not. +//! * refused, and the agent is gone → the opposite, and just as wrong. +//! +//! ### Why this is worth a scenario at all +//! +//! Because the happy path is already defended and the crash path is not +//! obviously so. `Worker::start_deleting` in the executor exists specifically to +//! stop a background status flush from "resurrecting the cached status" after +//! the durable removal — its own comment. Deleting is four steps (interrupt, +//! mark, remove from the worker service, remove from the active set) and only +//! the third is durable, so a pod that dies between the mark and the removal +//! leaves an agent marked for deletion that was never removed. Whoever picks up +//! its shard next decides what that means. + +use crate::chaos::deletions::{DeleteRound, FIRST_VALUE_OF_A_NEW_AGENT}; +use crate::chaos::history::Outcome; +use crate::chaos::split::{FaultWindow, Group, PodSplit, Window}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; + +/// The most findings the report carries. +const MAX_FINDINGS: usize = 50; + +/// What went wrong with one deletion. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum ResurrectionViolation { + /// The platform confirmed the deletion and the agent came back carrying the + /// value it had before. The failure this scenario is named for. + ResurrectedWithState, + /// The agent came back worth neither nothing nor what it had been: part of + /// a state it was supposed to have lost survived. + PartialState, + /// The platform refused the deletion and the agent is gone anyway. + RefusedButDeleted, +} + +impl ResurrectionViolation { + pub fn as_str(self) -> &'static str { + match self { + ResurrectionViolation::ResurrectedWithState => "resurrected-with-state", + ResurrectionViolation::PartialState => "partial-state", + ResurrectionViolation::RefusedButDeleted => "refused-but-deleted", + } + } +} + +/// One violation, against one named round. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ResurrectionFinding { + pub violation: ResurrectionViolation, + pub agent: String, + pub round: u32, + pub window: Window, + /// What the agent was worth before the delete, and what the slot reported + /// afterwards. Carried so a finding reads without the history. + pub before: u64, + pub observed: u64, + pub detail: String, +} + +/// Rounds and their verdicts for one (group, window) cell. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ResurrectionCell { + pub group: Group, + pub window: Window, + pub rounds: u64, + /// Deletions that took: the slot came back as a new agent. + pub deleted: u64, + /// Deletions that did not. Legitimate only when the platform never + /// confirmed them. + pub survived: u64, + pub violations: u64, + pub unjudgeable: u64, + pub unprobed: u64, +} + +/// Deletions the kill landed in the middle of. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DeletesCaught { + pub group: Group, + /// Deletes submitted before the kill that had not answered when it landed. + /// The population the scenario is actually about. + pub deletes: u64, + pub agents: usize, + pub confirmed: u64, + pub indeterminate: u64, + pub rejected: u64, +} + +/// The resurrection account. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ResurrectionReport { + /// What each round was configured to do, so an archived finding reads + /// without the suite YAML. + pub increments_per_round: u32, + pub rounds_recorded: u64, + pub deletes_confirmed: u64, + pub deletes_indeterminate: u64, + pub deletes_rejected: u64, + /// Confirmed deletes whose slot came back as a new agent. + pub deleted_exactly: u64, + /// Deletes the driver never heard back about that took anyway. + pub indeterminate_that_deleted: u64, + /// Deletes the driver never heard back about that did not. + pub indeterminate_that_did_not: u64, + pub unjudgeable: u64, + pub unprobed: u64, + pub cells: Vec, + pub caught_by_the_kill: Vec, + pub findings: Vec, + pub findings_omitted: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Verdict { + Deleted, + Survived, + Violation(ResurrectionViolation), + Unjudgeable, + Unprobed, +} + +/// Judges one round against the two values it is allowed to have landed on. +fn judge(round: &DeleteRound) -> Verdict { + let Some(before) = round.before_delete else { + return Verdict::Unjudgeable; + }; + let Some(observed) = round.observed_after else { + return Verdict::Unprobed; + }; + + if observed == FIRST_VALUE_OF_A_NEW_AGENT { + // Gone. Legitimate unless the platform said it refused. + if round.outcome == Outcome::Rejected { + return Verdict::Violation(ResurrectionViolation::RefusedButDeleted); + } + return Verdict::Deleted; + } + if observed == before + 1 { + // Still there. Legitimate unless the platform said it was gone. + if round.outcome == Outcome::Confirmed { + return Verdict::Violation(ResurrectionViolation::ResurrectedWithState); + } + return Verdict::Survived; + } + Verdict::Violation(ResurrectionViolation::PartialState) +} + +impl ResurrectionReport { + /// Builds the account from the rounds the workload recorded. + pub fn build( + rounds: &[DeleteRound], + split: &PodSplit, + fault: Option, + increments_per_round: u32, + ) -> Self { + let mut cells: BTreeMap<(Group, Window), ResurrectionCell> = BTreeMap::new(); + let mut findings: Vec = Vec::new(); + let mut report = ResurrectionReport { + increments_per_round, + rounds_recorded: rounds.len() as u64, + deletes_confirmed: 0, + deletes_indeterminate: 0, + deletes_rejected: 0, + deleted_exactly: 0, + indeterminate_that_deleted: 0, + indeterminate_that_did_not: 0, + unjudgeable: 0, + unprobed: 0, + cells: Vec::new(), + caught_by_the_kill: Vec::new(), + findings: Vec::new(), + findings_omitted: 0, + }; + + for round in rounds { + let group = split.group_of(&round.agent).unwrap_or(Group::Elsewhere); + let window = Window::of(round.submitted_at, fault); + let cell = cells + .entry((group, window)) + .or_insert_with(|| ResurrectionCell { + group, + window, + rounds: 0, + deleted: 0, + survived: 0, + violations: 0, + unjudgeable: 0, + unprobed: 0, + }); + cell.rounds += 1; + + match round.outcome { + Outcome::Confirmed => report.deletes_confirmed += 1, + Outcome::Indeterminate => report.deletes_indeterminate += 1, + Outcome::Rejected => report.deletes_rejected += 1, + } + + match judge(round) { + Verdict::Deleted => { + cell.deleted += 1; + if round.outcome == Outcome::Confirmed { + report.deleted_exactly += 1; + } else { + report.indeterminate_that_deleted += 1; + } + } + Verdict::Survived => { + cell.survived += 1; + if round.outcome != Outcome::Confirmed { + report.indeterminate_that_did_not += 1; + } + } + Verdict::Unjudgeable => { + cell.unjudgeable += 1; + report.unjudgeable += 1; + } + Verdict::Unprobed => { + cell.unprobed += 1; + report.unprobed += 1; + } + Verdict::Violation(violation) => { + cell.violations += 1; + let before = round.before_delete.unwrap_or_default(); + let observed = round.observed_after.unwrap_or_default(); + findings.push(ResurrectionFinding { + violation, + agent: round.agent.clone(), + round: round.round, + window, + before, + observed, + detail: detail_for(violation, before, observed), + }); + } + } + } + + if let Some(window) = fault { + let mut by_group: BTreeMap> = BTreeMap::new(); + for round in rounds { + let Some(group) = split.group_of(&round.agent) else { + continue; + }; + let unresolved = round.completed_at.is_none_or(|at| at >= window.injected_at); + if round.submitted_at < window.injected_at && unresolved { + by_group.entry(group).or_default().push(round); + } + } + for (group, caught) in by_group { + let agents: BTreeSet<&str> = caught.iter().map(|r| r.agent.as_str()).collect(); + report.caught_by_the_kill.push(DeletesCaught { + group, + deletes: caught.len() as u64, + agents: agents.len(), + confirmed: count(&caught, Outcome::Confirmed), + indeterminate: count(&caught, Outcome::Indeterminate), + rejected: count(&caught, Outcome::Rejected), + }); + } + } + + report.cells = cells.into_values().collect(); + report.findings_omitted = findings.len().saturating_sub(MAX_FINDINGS) as u64; + findings.truncate(MAX_FINDINGS); + report.findings = findings; + report + } + + pub fn has_violations(&self) -> bool { + !self.findings.is_empty() || self.findings_omitted > 0 + } + + /// The lines that need a human. + pub fn attention_lines(&self) -> Vec { + let mut lines: Vec = self + .findings + .iter() + .map(|f| { + format!( + "S6 {}: {} round {} — {}", + f.violation.as_str(), + f.agent, + f.round, + f.detail + ) + }) + .collect(); + if self.findings_omitted > 0 { + lines.push(format!( + "S6: {} further resurrection finding(s) were dropped from the report", + self.findings_omitted + )); + } + + let caught: u64 = self + .caught_by_the_kill + .iter() + .filter(|c| c.group == Group::OnPod) + .map(|c| c.deletes) + .sum(); + if caught == 0 { + lines.push( + "S6: the kill caught no delete in flight on the targeted executor, so this run \ + says nothing about crashing during a deletion. Every verdict below describes \ + deletes that completed either side of it." + .to_string(), + ); + } + lines + } + + /// Lines a reader needs in order to interpret the run. + pub fn note_lines(&self) -> Vec { + let mut lines = vec![format!( + "S6: {} rounds of {} increments then a delete; {} confirmed, {} in doubt, {} refused", + self.rounds_recorded, + self.increments_per_round, + self.deletes_confirmed, + self.deletes_indeterminate, + self.deletes_rejected + )]; + lines.push(format!( + "S6: {} confirmed deletes left a slot that came back as a new agent; {} in doubt \ + deleted anyway, {} in doubt did not", + self.deleted_exactly, self.indeterminate_that_deleted, self.indeterminate_that_did_not + )); + if self.unjudgeable > 0 || self.unprobed > 0 { + lines.push(format!( + "S6: {} rounds could not be judged (an increment never answered) and {} were \ + never probed by a following increment", + self.unjudgeable, self.unprobed + )); + } + for caught in &self.caught_by_the_kill { + lines.push(format!( + "S6 {}: {} deletes across {} agents were unresolved when the kill landed — {} \ + confirmed, {} in doubt, {} refused", + caught.group.as_str(), + caught.deletes, + caught.agents, + caught.confirmed, + caught.indeterminate, + caught.rejected + )); + } + for cell in &self.cells { + lines.push(format!( + "S6 {} {}: {} rounds, {} deleted, {} survived, {} violations, {} unjudgeable, \ + {} unprobed", + cell.group.as_str(), + cell.window.as_str(), + cell.rounds, + cell.deleted, + cell.survived, + cell.violations, + cell.unjudgeable, + cell.unprobed + )); + } + lines + } +} + +fn count(rounds: &[&DeleteRound], outcome: Outcome) -> u64 { + rounds.iter().filter(|r| r.outcome == outcome).count() as u64 +} + +fn detail_for(violation: ResurrectionViolation, before: u64, observed: u64) -> String { + match violation { + ResurrectionViolation::ResurrectedWithState => format!( + "the platform confirmed the deletion of an agent worth {before}, and the slot came \ + back worth {observed} — the old agent is still there with its state intact" + ), + ResurrectionViolation::PartialState => format!( + "the agent was worth {before} when it was deleted and the slot came back worth \ + {observed}, which is neither a new agent ({FIRST_VALUE_OF_A_NEW_AGENT}) nor the \ + old one ({}): part of a state it was supposed to have lost survived", + before + 1 + ), + ResurrectionViolation::RefusedButDeleted => format!( + "the platform refused to delete an agent worth {before} and it is gone anyway — the \ + slot came back worth {observed}" + ), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::{DateTime, TimeDelta, Utc}; + use test_r::test; + + const ON_POD: &str = "chaos-s6-delete-0000"; + const CONTROL: &str = "chaos-s6-delete-0001"; + /// What a round builds an agent up to before deleting it. + const BEFORE: u64 = 3; + + fn t0() -> DateTime { + DateTime::parse_from_rfc3339("2026-08-25T12:00:00Z") + .unwrap() + .with_timezone(&Utc) + } + + fn split() -> PodSplit { + PodSplit { + pod_address: "10.0.1.1:9000".to_string(), + pod_ip: "10.0.1.1".to_string(), + on_pod: vec![ON_POD.to_string()], + elsewhere: vec![CONTROL.to_string()], + targets_per_pod: BTreeMap::new(), + number_of_shards: 1024, + } + } + + fn fault() -> FaultWindow { + FaultWindow { + injected_at: t0(), + recovered_at: Some(t0() + TimeDelta::seconds(120)), + } + } + + fn round( + agent: &str, + offset_secs: i64, + before: Option, + outcome: Outcome, + observed: Option, + ) -> DeleteRound { + let submitted_at = t0() + TimeDelta::seconds(offset_secs); + DeleteRound { + agent: agent.to_string(), + round: 0, + before_delete: before, + outcome, + submitted_at, + completed_at: Some(submitted_at + TimeDelta::milliseconds(90)), + observed_after: observed, + } + } + + fn build(rounds: &[DeleteRound]) -> ResurrectionReport { + ResurrectionReport::build(rounds, &split(), Some(fault()), 3) + } + + fn violations(report: &ResurrectionReport) -> Vec { + report.findings.iter().map(|f| f.violation).collect() + } + + /// The healthy shape: the platform said the agent was gone, and the slot + /// came back counting from nothing. + #[test] + fn a_deletion_that_took_is_not_a_finding() { + let report = build(&[round( + ON_POD, + -10, + Some(BEFORE), + Outcome::Confirmed, + Some(1), + )]); + assert!(violations(&report).is_empty(), "{:?}", report.findings); + assert_eq!(report.deleted_exactly, 1); + assert!(!report.has_violations()); + } + + /// The failure this scenario is named for. + #[test] + fn a_confirmed_deletion_whose_agent_came_back_is_a_resurrection() { + // Worth 3, deleted, and the slot reported 4 — the old agent, incremented. + let report = build(&[round( + ON_POD, + -10, + Some(BEFORE), + Outcome::Confirmed, + Some(BEFORE + 1), + )]); + assert_eq!( + violations(&report), + vec![ResurrectionViolation::ResurrectedWithState] + ); + assert!(report.has_violations(), "this must be able to fail the run"); + let finding = &report.findings[0]; + assert_eq!(finding.before, BEFORE); + assert_eq!(finding.observed, BEFORE + 1); + } + + /// Its opposite: refused, and gone regardless. + #[test] + fn a_refused_deletion_that_happened_anyway_is_a_finding() { + let report = build(&[round(ON_POD, -10, Some(BEFORE), Outcome::Rejected, Some(1))]); + assert_eq!( + violations(&report), + vec![ResurrectionViolation::RefusedButDeleted] + ); + } + + /// Neither a new agent nor the old one: some of a state that was supposed + /// to be gone survived. + #[test] + fn a_slot_that_came_back_part_way_is_a_finding() { + // Worth 3, so 1 or 4 were legal. It reported 3. + let report = build(&[round( + ON_POD, + -10, + Some(BEFORE), + Outcome::Confirmed, + Some(3), + )]); + assert_eq!( + violations(&report), + vec![ResurrectionViolation::PartialState] + ); + assert!( + report.findings[0].detail.contains("neither a new agent"), + "{}", + report.findings[0].detail + ); + } + + /// A delete the driver never heard back about may land either way, and + /// neither answer is a defect. + #[test] + fn a_deletion_in_doubt_may_land_either_way_without_being_a_finding() { + let report = build(&[ + round(ON_POD, -10, Some(BEFORE), Outcome::Indeterminate, Some(1)), + round( + CONTROL, + -10, + Some(BEFORE), + Outcome::Indeterminate, + Some(BEFORE + 1), + ), + ]); + assert!(violations(&report).is_empty(), "{:?}", report.findings); + assert_eq!(report.indeterminate_that_deleted, 1); + assert_eq!(report.indeterminate_that_did_not, 1); + } + + /// Why `incrementsPerRound` must be at least two, stated exactly. + /// + /// The two legal answers never collide: a fresh agent reports 1 and a + /// survivor reports `before + 1`. What one increment removes is the *gap* + /// between them, and the gap is where a partial state would show up. At + /// `before = 1` the answers are 1 and 2 with nothing in between, so + /// `PartialState` cannot fire whatever the platform does. + #[test] + fn one_increment_leaves_no_room_for_a_partial_state_to_be_seen() { + // At before = 3 there is room, and a slot landing in it is caught. + let seen = build(&[round(ON_POD, -10, Some(3), Outcome::Confirmed, Some(2))]); + assert_eq!(violations(&seen), vec![ResurrectionViolation::PartialState]); + + // At before = 1 every value is one of the two legal answers, so no + // observation can ever produce this finding. `require_delete` refuses + // the configuration rather than shipping a blind third of the oracle. + for observed in [1, 2] { + let report = build(&[round( + ON_POD, + -10, + Some(1), + Outcome::Indeterminate, + Some(observed), + )]); + assert!( + !violations(&report).contains(&ResurrectionViolation::PartialState), + "observed {observed} should be legal at before=1" + ); + } + } + + #[test] + fn a_round_whose_increments_never_answered_is_not_judged() { + let report = build(&[round(ON_POD, -10, None, Outcome::Confirmed, Some(1))]); + assert_eq!(report.unjudgeable, 1); + assert!(report.findings.is_empty()); + } + + #[test] + fn a_round_no_increment_ever_probed_is_counted_as_unprobed() { + let report = build(&[round(ON_POD, -10, Some(BEFORE), Outcome::Confirmed, None)]); + assert_eq!(report.unprobed, 1); + assert!(report.findings.is_empty()); + } + + /// The S10 lesson: a kill that caught nothing proves nothing, and the run + /// has to say so rather than read as clean. + #[test] + fn a_kill_that_caught_no_delete_says_the_run_proved_nothing() { + let report = build(&[round( + ON_POD, + -100, + Some(BEFORE), + Outcome::Confirmed, + Some(1), + )]); + assert!(report.caught_by_the_kill.is_empty()); + assert!( + report + .attention_lines() + .iter() + .any(|l| l.contains("caught no delete in flight")), + "attention was {:?}", + report.attention_lines() + ); + } + + /// A delete still unresolved when the pod died is the population the whole + /// scenario is about. + #[test] + fn deletes_unresolved_when_the_pod_died_are_reported_separately() { + let mut caught = round(ON_POD, -1, Some(BEFORE), Outcome::Indeterminate, Some(1)); + caught.completed_at = Some(t0() + TimeDelta::seconds(30)); + let report = build(&[caught]); + + let entry = report + .caught_by_the_kill + .iter() + .find(|c| c.group == Group::OnPod) + .expect("the kill caught a delete"); + assert_eq!(entry.deletes, 1); + assert_eq!(entry.indeterminate, 1); + assert!( + !report + .attention_lines() + .iter() + .any(|l| l.contains("caught no delete in flight")) + ); + } + + #[test] + fn rounds_are_split_by_group_and_window() { + let report = build(&[ + round(ON_POD, -10, Some(BEFORE), Outcome::Confirmed, Some(1)), + round(ON_POD, 10, Some(BEFORE), Outcome::Confirmed, Some(1)), + round(CONTROL, 10, Some(BEFORE), Outcome::Confirmed, Some(1)), + ]); + let cell = |g, w| { + report + .cells + .iter() + .find(|c| c.group == g && c.window == w) + .cloned() + }; + assert_eq!(cell(Group::OnPod, Window::BeforeFault).unwrap().rounds, 1); + assert_eq!(cell(Group::OnPod, Window::DuringFault).unwrap().deleted, 1); + assert_eq!( + cell(Group::Elsewhere, Window::DuringFault).unwrap().rounds, + 1 + ); + } + + #[test] + fn findings_beyond_the_cap_are_counted_rather_than_carried() { + let rounds: Vec = (0..MAX_FINDINGS + 4) + .map(|_| { + round( + ON_POD, + -10, + Some(BEFORE), + Outcome::Confirmed, + Some(BEFORE + 1), + ) + }) + .collect(); + let report = build(&rounds); + assert_eq!(report.findings.len(), MAX_FINDINGS); + assert_eq!(report.findings_omitted, 4); + assert!(report.has_violations()); + } +} diff --git a/integration-tests/src/chaos/scenarios/mod.rs b/integration-tests/src/chaos/scenarios/mod.rs index ab2f361fad..c3b1e1254d 100644 --- a/integration-tests/src/chaos/scenarios/mod.rs +++ b/integration-tests/src/chaos/scenarios/mod.rs @@ -32,6 +32,7 @@ pub mod s12; pub mod s13; pub mod s3; pub mod s5; +pub mod s6; pub mod s7; pub mod s8; @@ -93,6 +94,9 @@ pub struct ScenarioOutcome { /// Present only for S7, which divides the agents whose state is being /// reverted around the executor the kill is aimed at. pub revert_selection: Option, + /// Present only for S6, which divides the agent slots being deleted around + /// the executor the kill is aimed at. + pub delete_selection: Option, } /// Assembles the archived result. @@ -124,6 +128,8 @@ pub fn build_result(config: &ScenarioConfig, outcome: ScenarioOutcome) -> ChaosR isolation_selection: outcome.isolation_selection, revert: config.revert.clone(), revert_selection: outcome.revert_selection, + delete: config.delete.clone(), + delete_selection: outcome.delete_selection, retry_policy: config.retry_policy.clone(), scope: outcome.scope, summary: outcome.summary, diff --git a/integration-tests/src/chaos/scenarios/s1.rs b/integration-tests/src/chaos/scenarios/s1.rs index 22692eb5c5..be382bcd9f 100644 --- a/integration-tests/src/chaos/scenarios/s1.rs +++ b/integration-tests/src/chaos/scenarios/s1.rs @@ -239,6 +239,7 @@ pub async fn run( promise_selection: None, isolation_selection: None, revert_selection: None, + delete_selection: None, }, ); write_outputs(&result, &history, outputs)?; diff --git a/integration-tests/src/chaos/scenarios/s10.rs b/integration-tests/src/chaos/scenarios/s10.rs index e9ae75611a..7008272c0d 100644 --- a/integration-tests/src/chaos/scenarios/s10.rs +++ b/integration-tests/src/chaos/scenarios/s10.rs @@ -225,6 +225,7 @@ pub async fn run( promise_selection: None, isolation_selection: None, revert_selection: None, + delete_selection: None, }, ); write_outputs(&result, &history, outputs)?; diff --git a/integration-tests/src/chaos/scenarios/s11.rs b/integration-tests/src/chaos/scenarios/s11.rs index 84a1c21006..f8ccbd8d1e 100644 --- a/integration-tests/src/chaos/scenarios/s11.rs +++ b/integration-tests/src/chaos/scenarios/s11.rs @@ -199,6 +199,7 @@ pub async fn run( promise_selection: selection.clone(), isolation_selection: None, revert_selection: None, + delete_selection: None, }, ); write_outputs(&result, &history, outputs)?; diff --git a/integration-tests/src/chaos/scenarios/s12.rs b/integration-tests/src/chaos/scenarios/s12.rs index bd95b6ec52..537aa3375b 100644 --- a/integration-tests/src/chaos/scenarios/s12.rs +++ b/integration-tests/src/chaos/scenarios/s12.rs @@ -136,6 +136,7 @@ pub async fn run( promise_selection: None, isolation_selection: None, revert_selection: None, + delete_selection: None, }, ); write_outputs(&result, &history, outputs)?; diff --git a/integration-tests/src/chaos/scenarios/s13.rs b/integration-tests/src/chaos/scenarios/s13.rs index 520bd1b021..f76d4b59a3 100644 --- a/integration-tests/src/chaos/scenarios/s13.rs +++ b/integration-tests/src/chaos/scenarios/s13.rs @@ -207,6 +207,7 @@ pub async fn run( promise_selection: None, isolation_selection: None, revert_selection: None, + delete_selection: None, }, ); write_outputs(&result, &history, outputs)?; diff --git a/integration-tests/src/chaos/scenarios/s3.rs b/integration-tests/src/chaos/scenarios/s3.rs index 2537e7fe90..63351408a6 100644 --- a/integration-tests/src/chaos/scenarios/s3.rs +++ b/integration-tests/src/chaos/scenarios/s3.rs @@ -184,6 +184,7 @@ pub async fn run( promise_selection: None, isolation_selection: selection.clone(), revert_selection: None, + delete_selection: None, }, ); write_outputs(&result, &history, outputs)?; diff --git a/integration-tests/src/chaos/scenarios/s5.rs b/integration-tests/src/chaos/scenarios/s5.rs index a08bd04def..d80e382490 100644 --- a/integration-tests/src/chaos/scenarios/s5.rs +++ b/integration-tests/src/chaos/scenarios/s5.rs @@ -179,6 +179,7 @@ pub async fn run( promise_selection: None, isolation_selection: None, revert_selection: None, + delete_selection: None, }, ); write_outputs(&result, &history, outputs)?; diff --git a/integration-tests/src/chaos/scenarios/s6.rs b/integration-tests/src/chaos/scenarios/s6.rs new file mode 100644 index 0000000000..7afbb8f14e --- /dev/null +++ b/integration-tests/src/chaos/scenarios/s6.rs @@ -0,0 +1,475 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! S6 — executor pod kill during agent deletion (GOL-372). +//! +//! S7 asks the platform to forget some of an agent's work. S6 asks it to forget +//! the agent, and then kills the executor while it is doing so. +//! +//! ### Why this one can assert +//! +//! Invoking a deleted agent id creates a **new** agent, and a new counter starts +//! from nothing. So a round — increments to a known `V`, then a delete — has +//! exactly two legal answers when the slot is next used: +//! +//! * `1`, a fresh agent, meaning the deletion took +//! * `V + 1`, meaning the old agent is still there +//! +//! Neither is a defect on its own; a delete whose response was lost leaves the +//! question genuinely open. What makes one a defect is the platform's own answer +//! beside it. Confirmed, and the agent is still worth `V`, is the resurrection +//! this scenario is named for. See [`crate::chaos::resurrection`]. +//! +//! ### What is actually being killed into +//! +//! Deleting is four steps in `delete_worker_internal`: interrupt the running +//! worker, `start_deleting`, remove it from the worker service, remove it from +//! the active set. Only the third is durable. +//! +//! The interesting part is that the happy path is **already defended**. +//! `Worker::start_deleting` stops the background status flush and the +//! checkpointer first, specifically so neither can — in the executor's own +//! comment — "resurrect the cached status" after the removal. So the question +//! S6 asks is not whether anyone thought about resurrection, but whether that +//! defence survives the pod dying between the mark and the removal, when +//! whoever picks up the shard next has to decide what a worker marked for +//! deletion but never removed means. +//! +//! ### A smoke round before anything else +//! +//! The whole account rests on "a deleted id comes back as a new agent". If that +//! is not true, every round in the run reports a resurrection and the report is +//! worthless. So one throwaway agent is built, deleted and re-invoked before the +//! baseline starts, and a run whose premise is wrong aborts in seconds instead +//! of spending the maintenance window discovering it. That lesson is S11's. +//! +//! ### The choreography +//! +//! Like S8, S10, S11, S3 and S7 the driver names the pod: it picks the executor +//! owning the largest share of its agent slots and keeps driving the rest as a +//! control group. As in S7 the last thing before read-back is one read per slot, +//! which answers the round no following increment ever probed. + +use crate::chaos::deletions::{self, DeleteRound}; +use crate::chaos::history::{OperationHistory, Outcome, Phase, Stream}; +use crate::chaos::ownership::OwnershipSample; +use crate::chaos::prep::ChaosPrepManifest; +use crate::chaos::result::{ChaosResult, PhaseWindow, Phases, RunScope}; +use crate::chaos::resurrection::ResurrectionReport; +use crate::chaos::scenarios::{ + OutputPaths, ScenarioOutcome, WARMUP_SETTLE, build_result, sample_ownership, + signal_termination, snapshot_routing, wait_for_settled_routing, write_outputs, +}; +use crate::chaos::signal::{BaselineReady, FaultSignals, FaultTarget}; +use crate::chaos::split::{self, FaultWindow, PodSplit}; +use crate::chaos::summary::{ChaosSummary, Note, TerminationReason}; +use crate::chaos::workload::{self, PhaseMarker, WorkloadContext}; +use crate::chaos::{ScenarioCode, ScenarioConfig}; +use chrono::Utc; +use golem_test_framework::config::BenchmarkTestDependencies; +use golem_test_framework::dsl::TestDsl; +use std::time::Duration; +use tracing::{info, warn}; + +/// How long to wait after stopping the workload before the final read. +/// +/// Shorter than the other scenarios' settle: nothing here is queued or +/// scheduled, and `stop` already waits for every operation in flight to record +/// itself. This only covers a slot still coming back from the last delete. +const SETTLE_BEFORE_READBACK: Duration = Duration::from_secs(20); + +/// How far into the fault window the assignment is sampled. +const DURING_FAULT_SAMPLE_FRACTION: f64 = 0.6; +const DURING_FAULT_SAMPLE_CAP: Duration = Duration::from_secs(120); + +/// Runs S6 end to end. +pub async fn run( + config: &ScenarioConfig, + manifest: &ChaosPrepManifest, + deps: &BenchmarkTestDependencies, + signals: &FaultSignals, + outputs: &OutputPaths, +) -> anyhow::Result { + let started_at = Utc::now(); + let delete_config = config.require_delete()?; + let history = OperationHistory::new(ScenarioCode::S6.as_str()); + let key_prefix = crate::chaos::scenario_key_prefix(ScenarioCode::S6); + + let user = manifest.user_context(deps); + let counters = user + .get_latest_component_revision(&manifest.counters_component_id) + .await?; + let promise = user + .get_latest_component_revision(&manifest.promise_component_id) + .await?; + + let ctx = WorkloadContext { + user, + counters, + promise, + history: history.clone(), + retry: config.retry_policy.clone(), + phase: PhaseMarker::new(Phase::Baseline), + key_prefix: key_prefix.clone(), + }; + + let scope = RunScope { + environment_id: manifest.environment_id.0.to_string(), + component_ids: vec![manifest.counters_component_id.0.to_string()], + agent_id_prefix: key_prefix.clone(), + idempotency_key_prefix: format!("{key_prefix}-"), + }; + + let agents = deletions::agent_names(&ctx, delete_config.agents); + + let mut phases = Phases::default(); + let mut routing_snapshots = Vec::new(); + let mut ownership: Vec = Vec::new(); + let mut fault_injected_at = None; + let mut fault_recovered_at = None; + let mut fault_id = None; + let mut fault_target_observed = None; + let mut selection: Option = None; + let mut attention_extra: Vec = Vec::new(); + + macro_rules! finish { + ($reason:expr, $records:expr, $resurrection:expr) => {{ + let mut summary = ChaosSummary::build( + $records, + Vec::new(), + routing_snapshots.clone(), + fault_injected_at, + ) + .with_ownership(ownership.clone()); + summary.absorb(attention_extra.clone()); + if let Some(report) = $resurrection { + summary = summary.with_resurrection(report); + } + let result = build_result( + config, + ScenarioOutcome { + started_at, + phases: phases.clone(), + fault_injected_at, + fault_recovered_at, + fault_id: fault_id.clone(), + fault_target_observed: fault_target_observed.clone(), + scope: scope.clone(), + summary, + termination_reason: $reason, + pinned_selection: None, + scheduled_selection: None, + promise_selection: None, + isolation_selection: None, + revert_selection: None, + delete_selection: selection.clone(), + }, + ); + write_outputs(&result, &history, outputs)?; + return Ok(result); + }}; + } + + // ── Warm-up ───────────────────────────────────────────────────────────── + // + // Constructing an agent is itself recorded in its oplog, so doing it inside + // a measured round would leave a slot holding an agent the round did not + // build. Reads here rather than increments, for the same reason every other + // scenario warms with reads: an increment would be invisible to the round + // arithmetic the whole oracle rests on. + routing_snapshots.push(snapshot_routing(deps, "before-warmup").await); + attention_extra.push(wait_for_settled_routing(deps, &mut routing_snapshots).await); + + info!("S6: warming up {} delete agents", agents.len()); + let mut warmed = 0usize; + for agent in &agents { + if workload::read_counter(&ctx, agent).await.is_ok() { + warmed += 1; + } + } + info!( + "S6: warmed {warmed} of {} agents, settling {WARMUP_SETTLE:?}", + agents.len() + ); + tokio::time::sleep(WARMUP_SETTLE).await; + + // ── Smoke round ───────────────────────────────────────────────────────── + // + // Everything below assumes a deleted id comes back as a new agent. If that + // is not true, every round reports a resurrection and the artifact is + // worthless — so one throwaway agent proves it before the baseline starts. + // The first S11 run is why this is here: a wrong premise otherwise costs + // the whole maintenance window before the numbers say so. + if let Err(e) = deletions::smoke_round(&ctx, delete_config).await { + warn!("S6: smoke round failed, aborting before the baseline: {e:#}"); + let records = history.snapshot(); + finish!( + TerminationReason::PlatformUnreachable { + detail: format!("{e:#}"), + }, + &records, + None + ); + } + + // ── Aim ───────────────────────────────────────────────────────────────── + let subject = split::delete_subject(&ctx); + let split = match split::select(subject, deps, &agents).await { + Ok(split) => split, + Err(e) => { + warn!("S6: cannot aim the kill: {e:#}"); + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!("{e:#}"), + }, + &records, + None + ); + } + }; + selection = Some(split.clone()); + + // ── Baseline ──────────────────────────────────────────────────────────── + info!( + "S6: baseline phase, running {} delete emitters for {:?}", + agents.len(), + config.phases.baseline() + ); + phases.baseline = Some(PhaseWindow::started(Utc::now())); + let handle = deletions::start(ctx.clone(), delete_config); + tokio::time::sleep(config.phases.baseline()).await; + routing_snapshots.push(snapshot_routing(deps, "before-fault").await); + ownership.push(sample_ownership(deps, "before-fault", ownership.last(), false).await); + if let Some(window) = phases.baseline.as_mut() { + window.end(Utc::now()); + } + + let baseline_operations = history.confirmed_in_phase(Phase::Baseline); + if baseline_operations == 0 { + warn!("S6: baseline produced no confirmed operations, aborting before injection"); + let rounds = handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::PlatformUnreachable { + detail: "no operation succeeded during the baseline phase".to_string(), + }, + &records, + Some(build_resurrection(&rounds, &split, None, delete_config)) + ); + } + + if let Err(e) = split::verify_ownership(subject, deps, &split).await { + warn!("S6: ownership drifted between selection and injection: {e:#}"); + let rounds = handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::FaultTargetUnverified { + detail: format!("{e:#}"), + }, + &records, + Some(build_resurrection(&rounds, &split, None, delete_config)) + ); + } + + info!( + "S6: baseline complete ({baseline_operations} confirmed ops, {} rounds), naming {} and \ + signalling readiness", + handle.rounds().len(), + split.pod_address + ); + signals.write_baseline_ready(&BaselineReady { + scenario_code: ScenarioCode::S6.as_str().to_string(), + ready_at: Utc::now(), + baseline_operations, + fault_target: Some(FaultTarget { + pod_address: split.pod_address.clone(), + pod_ip: split.pod_ip.clone(), + owned_agents: split.on_pod.clone(), + }), + })?; + + // ── Fault ─────────────────────────────────────────────────────────────── + let injected = match signals.await_fault_injected(config.signal_timeout()).await { + Ok(injected) => injected, + Err(e) => { + warn!("S6: no fault-injected signal arrived: {e}"); + let rounds = handle.stop().await; + let records = history.snapshot(); + finish!( + signal_termination(&e), + &records, + Some(build_resurrection(&rounds, &split, None, delete_config)) + ); + } + }; + info!( + "S6: fault {} ({} on {}) reported active at {}", + injected.fault_id, injected.kind, injected.target, injected.injected_at + ); + fault_injected_at = Some(injected.injected_at); + fault_id = Some(injected.fault_id.clone()); + fault_target_observed = Some(injected.target.clone()); + ctx.phase.set(Phase::Fault); + phases.fault = Some(PhaseWindow::started(injected.injected_at)); + + let observe_after = config + .phases + .fault() + .mul_f64(DURING_FAULT_SAMPLE_FRACTION) + .min(DURING_FAULT_SAMPLE_CAP); + tokio::time::sleep(observe_after).await; + ownership.push(sample_ownership(deps, "during-fault", ownership.last(), false).await); + + let recovered = match signals.await_fault_recovered(config.signal_timeout()).await { + Ok(recovered) => recovered, + Err(e) => { + warn!("S6: no fault-recovered signal arrived: {e}"); + let rounds = handle.stop().await; + let records = history.snapshot(); + finish!( + signal_termination(&e), + &records, + Some(build_resurrection( + &rounds, + &split, + fault_window(fault_injected_at, None), + delete_config + )) + ); + } + }; + info!( + "S6: executor back at {} ({})", + recovered.recovered_at, recovered.termination_reason + ); + fault_recovered_at = Some(recovered.recovered_at); + if let Some(window) = phases.fault.as_mut() { + window.end(recovered.recovered_at); + } + + // ── Recovery ──────────────────────────────────────────────────────────── + ctx.phase.set(Phase::Recovery); + phases.recovery = Some(PhaseWindow::started(Utc::now())); + info!( + "S6: recovery phase, running for {:?}", + config.phases.recovery() + ); + tokio::time::sleep(config.phases.recovery()).await; + + let mut rounds = handle.stop().await; + if let Some(window) = phases.recovery.as_mut() { + window.end(Utc::now()); + } + routing_snapshots.push(snapshot_routing(deps, "after-recovery").await); + ownership.push(sample_ownership(deps, "after-recovery", ownership.last(), true).await); + + // ── Read-back ─────────────────────────────────────────────────────────── + info!("S6: settling {SETTLE_BEFORE_READBACK:?} before the final read"); + tokio::time::sleep(SETTLE_BEFORE_READBACK).await; + + // The one read per agent, which answers the last round it ran. Every other + // round was probed by the increment that followed it; this is the only one + // that has nothing after it. + close_last_rounds(&ctx, &agents, &mut rounds).await; + + let records = history.snapshot(); + let resurrection = build_resurrection( + &rounds, + &split, + fault_window(fault_injected_at, fault_recovered_at), + delete_config, + ); + info!( + "S6: resurrection account — {} rounds, {} deleted exactly, {} findings", + resurrection.rounds_recorded, + resurrection.deleted_exactly, + resurrection.findings.len() + ); + + let reason = if resurrection.has_violations() { + let first = resurrection + .findings + .first() + .map(|f| format!("{} round {}: {}", f.agent, f.round, f.detail)) + .unwrap_or_default(); + TerminationReason::AgentResurrected { + findings: resurrection.findings.len() as u64 + resurrection.findings_omitted, + first, + } + } else if records.iter().all(|r| r.outcome != Outcome::Confirmed) { + TerminationReason::StreamNeverSucceeded { + stream: Stream::Delete.to_string(), + } + } else { + TerminationReason::Completed + }; + + finish!(reason, &records, Some(resurrection)); +} + +fn fault_window( + injected_at: Option>, + recovered_at: Option>, +) -> Option { + injected_at.map(|injected_at| FaultWindow { + injected_at, + recovered_at, + }) +} + +fn build_resurrection( + rounds: &[DeleteRound], + split: &PodSplit, + fault: Option, + config: &crate::chaos::DeleteConfig, +) -> ResurrectionReport { + ResurrectionReport::build(rounds, split, fault, config.increments_per_round) +} + +/// Reads each agent once and uses the value to judge the last round it ran. +/// +/// A read is an invocation and would shift what "the last N invocations" means +/// for any delete after it — which is exactly why the workload never reads +/// mid-round. Here there is nothing after it, so it is safe, and it recovers a +/// round per agent that would otherwise be unjudgeable. +async fn close_last_rounds(ctx: &WorkloadContext, agents: &[String], rounds: &mut [DeleteRound]) { + let mut last_of: std::collections::BTreeMap<&str, usize> = std::collections::BTreeMap::new(); + for (index, round) in rounds.iter().enumerate() { + if round.observed_after.is_none() { + let slot = last_of.entry(round.agent.as_str()).or_insert(index); + if rounds[*slot].round < round.round { + *slot = index; + } + } + } + let pending: Vec<(String, usize)> = last_of + .into_iter() + .map(|(agent, index)| (agent.to_string(), index)) + .collect(); + + info!( + "S6: closing {} unprobed rounds with a final read", + pending.len() + ); + for (agent, index) in pending { + if !agents.iter().any(|a| a == &agent) { + continue; + } + match workload::read_counter(ctx, &agent).await { + Ok(value) => rounds[index].observed_after = Some(value), + Err(e) => warn!("S6: could not read {agent} to close its last round: {e}"), + } + } +} diff --git a/integration-tests/src/chaos/scenarios/s7.rs b/integration-tests/src/chaos/scenarios/s7.rs index 1780b0ba88..db109fded3 100644 --- a/integration-tests/src/chaos/scenarios/s7.rs +++ b/integration-tests/src/chaos/scenarios/s7.rs @@ -161,6 +161,7 @@ pub async fn run( promise_selection: None, isolation_selection: None, revert_selection: selection.clone(), + delete_selection: None, }, ); write_outputs(&result, &history, outputs)?; diff --git a/integration-tests/src/chaos/scenarios/s8.rs b/integration-tests/src/chaos/scenarios/s8.rs index 0c5170d436..5fcfc893fa 100644 --- a/integration-tests/src/chaos/scenarios/s8.rs +++ b/integration-tests/src/chaos/scenarios/s8.rs @@ -156,6 +156,7 @@ pub async fn run( promise_selection: None, isolation_selection: None, revert_selection: None, + delete_selection: None, }, ); write_outputs(&result, &history, outputs)?; diff --git a/integration-tests/src/chaos/split.rs b/integration-tests/src/chaos/split.rs index ba34d30cf3..cd811fd1ef 100644 --- a/integration-tests/src/chaos/split.rs +++ b/integration-tests/src/chaos/split.rs @@ -348,6 +348,16 @@ pub fn revert_subject<'a>(ctx: &'a WorkloadContext) -> Subject<'a> { } } +/// The counters component's delete slots, as S6 aims at them. +pub fn delete_subject<'a>(ctx: &'a WorkloadContext) -> Subject<'a> { + Subject { + scenario: "S6", + component: &ctx.counters, + agent_type: crate::chaos::workload::COUNTER_AGENT, + noun: "delete agents", + } +} + /// The promise component's waiters, as S11 aims at them. pub fn waiter_subject<'a>(ctx: &'a WorkloadContext) -> Subject<'a> { Subject { diff --git a/integration-tests/src/chaos/summary.rs b/integration-tests/src/chaos/summary.rs index d977d6f4b0..b1a0d6c751 100644 --- a/integration-tests/src/chaos/summary.rs +++ b/integration-tests/src/chaos/summary.rs @@ -53,6 +53,7 @@ use crate::chaos::history::{Outcome, Phase, Stream}; use crate::chaos::ownership::OwnershipSample; use crate::chaos::probe::KeyProbe; use crate::chaos::reachability::ReachabilityReport; +use crate::chaos::resurrection::ResurrectionReport; use crate::chaos::truncation::TruncationReport; use crate::chaos::wakeups::WakeupReport; use serde::{Deserialize, Serialize}; @@ -583,6 +584,10 @@ pub struct ChaosSummary { /// for scenarios that do not, for the same reason as `scheduleFires`. #[serde(default, skip_serializing_if = "Option::is_none")] pub truncation: Option, + /// The resurrection account, for scenarios that delete agents. Absent for + /// scenarios that do not, for the same reason as `scheduleFires`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub resurrection: Option, /// Shard-ownership samples, in the order they were taken. Empty for /// scenarios that do not sample executor assignments. /// @@ -716,6 +721,7 @@ impl ChaosSummary { promise_wakeups: None, reachability: None, truncation: None, + resurrection: None, ownership: Vec::new(), attention, notes: Vec::new(), @@ -803,6 +809,19 @@ impl ChaosSummary { self } + /// Attaches the resurrection account and hoists everything it wants a human + /// to see into [`Self::attention`]. + /// + /// Same split as [`Self::with_truncation`], including the inconclusive line: + /// a kill that caught no delete in flight proves nothing about crashing + /// during a deletion. + pub fn with_resurrection(mut self, report: ResurrectionReport) -> Self { + self.attention.extend(report.attention_lines()); + self.notes.extend(report.note_lines()); + self.resurrection = Some(report); + self + } + /// Attaches the shard-ownership samples and hoists their findings into /// [`Self::attention`]. /// @@ -851,6 +870,12 @@ pub enum TerminationReason { /// owners is an agent whose state can fork, and there is no instant at /// which that is legitimate. ShardOwnershipViolated { findings: u64, first: String }, + /// An agent the platform said it had deleted came back with its state, or a + /// deletion landed somewhere other than the two answers it was allowed. + /// Asserted for the same reason as `RevertTruncationViolated`: invoking a + /// deleted id creates a new agent, so there are exactly two legal values and + /// no band of doubt. See [`crate::chaos::resurrection`]. + AgentResurrected { findings: u64, first: String }, /// A revert landed somewhere other than the two values it was allowed to. /// Asserted rather than reported, and the only read-back in the suite that /// earns that: the driver knows the counter's value before the revert and @@ -1163,7 +1188,8 @@ mod tests { Stream::Ephemeral, Stream::Promise, Stream::PromiseWait, - Stream::Revert + Stream::Revert, + Stream::Delete ] ); } diff --git a/integration-tests/src/chaos/workload.rs b/integration-tests/src/chaos/workload.rs index 473b6974c4..a1d49d97d4 100644 --- a/integration-tests/src/chaos/workload.rs +++ b/integration-tests/src/chaos/workload.rs @@ -550,6 +550,12 @@ pub(crate) async fn submit_one(ctx: &WorkloadContext, stream: Stream, index: u32 Stream::PromiseWait => { warn!("Chaos mixed workload cannot drive the waiter stream; see chaos::waiters"); } + // Driven by `crate::chaos::deletions`: a round builds an agent up and + // then deletes it outright, and the value the *next* round's first + // increment returns says whether it stayed deleted. + Stream::Delete => { + warn!("Chaos mixed workload cannot drive the delete stream; see chaos::deletions"); + } // Driven by `crate::chaos::reverts`: a round is a run of increments // followed by a revert that takes some of them back, and the value the // *next* round's first increment returns is what says whether that From a965ebd20eb88c5ec8748db0f31a47d30cd2b183 Mon Sep 17 00:00:00 2001 From: kmatasfp <33095685+kmatas@users.noreply.github.com> Date: Mon, 24 Aug 2026 17:34:00 -0700 Subject: [PATCH 17/40] Add the S6 result shape test --- integration-tests/src/chaos/result.rs | 79 +++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/integration-tests/src/chaos/result.rs b/integration-tests/src/chaos/result.rs index 38e0761708..61a4d958a0 100644 --- a/integration-tests/src/chaos/result.rs +++ b/integration-tests/src/chaos/result.rs @@ -353,6 +353,85 @@ mod tests { assert!(parsed.promise_selection.is_some()); } + /// The S6 shape. Same contract as the S3, S7, S10 and S11 tests: the + /// investigation report in golem-cloud reads these fields by name. + #[test] + fn an_s6_result_carries_the_resurrection_fields_the_investigation_report_reads() { + use crate::chaos::deletions::DeleteRound; + use crate::chaos::resurrection::ResurrectionReport; + use crate::chaos::split::{FaultWindow, PodSplit}; + + let now = Utc::now(); + let agent = "chaos-s6-delete-0000".to_string(); + let split = PodSplit { + pod_address: "10.0.1.1:9000".to_string(), + pod_ip: "10.0.1.1".to_string(), + on_pod: vec![agent.clone()], + elsewhere: Vec::new(), + targets_per_pod: std::collections::BTreeMap::new(), + number_of_shards: 1024, + }; + + let mut result = sample_result(TerminationReason::Completed); + result.scenario_code = "S6".to_string(); + result.delete = Some(crate::chaos::DeleteConfig { + agents: 200, + increments_per_round: 3, + interval_millis: 500, + recovery_budget_secs: 60, + }); + result.delete_selection = Some(split.clone()); + result.summary = ChaosSummary::build(&[], Vec::new(), Vec::new(), Some(now)) + .with_resurrection(ResurrectionReport::build( + &[DeleteRound { + agent, + round: 0, + before_delete: Some(3), + outcome: crate::chaos::history::Outcome::Confirmed, + submitted_at: now, + completed_at: Some(now), + observed_after: Some(1), + }], + &split, + Some(FaultWindow { + injected_at: now, + recovered_at: None, + }), + 3, + )); + + let json = serde_json::to_value(&result).unwrap(); + let resurrection = &json["summary"]["resurrection"]; + for key in [ + "incrementsPerRound", + "roundsRecorded", + "deletesConfirmed", + "deletesIndeterminate", + "deletesRejected", + "deletedExactly", + "indeterminateThatDeleted", + "indeterminateThatDidNot", + "unjudgeable", + "unprobed", + "cells", + "caughtByTheKill", + "findings", + "findingsOmitted", + ] { + assert!( + !resurrection[key].is_null(), + "summary.resurrection.{key} is what the investigation report reads" + ); + } + assert_eq!(json["delete"]["incrementsPerRound"], 3); + assert_eq!(json["deleteSelection"]["podIp"], "10.0.1.1"); + + let parsed: ChaosResult = serde_json::from_str(&json.to_string()).unwrap(); + assert_eq!(parsed.scenario_code, "S6"); + assert!(parsed.summary.resurrection.is_some()); + assert!(parsed.delete_selection.is_some()); + } + /// The S7 shape. Same contract as the S3, S10 and S11 tests: the /// investigation report in golem-cloud reads these fields by name. #[test] From 2c52da8a3b4830297a84ebca2fbc0afa7eb9b0d2 Mon Sep 17 00:00:00 2001 From: kmatasfp <33095685+kmatas@users.noreply.github.com> Date: Mon, 24 Aug 2026 18:37:20 -0700 Subject: [PATCH 18/40] Judge a deletion on the counter it left behind, and read a not-found refusal correctly --- integration-tests/src/chaos/deletions.rs | 110 ++++++++++-- integration-tests/src/chaos/result.rs | 3 +- integration-tests/src/chaos/resurrection.rs | 190 ++++++++++++++++---- integration-tests/src/chaos/reverts.rs | 22 ++- integration-tests/src/chaos/workload.rs | 14 +- 5 files changed, 284 insertions(+), 55 deletions(-) diff --git a/integration-tests/src/chaos/deletions.rs b/integration-tests/src/chaos/deletions.rs index 3cb6cbfaf5..ab5a974cdc 100644 --- a/integration-tests/src/chaos/deletions.rs +++ b/integration-tests/src/chaos/deletions.rs @@ -57,12 +57,12 @@ use std::sync::{Arc, Mutex}; use tokio::task::JoinSet; use tracing::info; -/// What a freshly created agent's first increment returns. +/// The counter a freshly created agent carries. /// -/// Named rather than written as `1` at the comparison, because it is the whole +/// Named rather than written as `0` at the comparison, because it is the whole /// definition of "the deletion took": an agent id that was deleted and then /// invoked again is a *new* agent, and a new counter starts from nothing. -pub const FIRST_VALUE_OF_A_NEW_AGENT: u64 = 1; +pub const COUNTER_OF_A_NEW_AGENT: u64 = 0; /// One round, as the driver observed it. #[derive(Debug, Clone, Serialize, Deserialize)] @@ -78,12 +78,43 @@ pub struct DeleteRound { pub outcome: Outcome, pub submitted_at: DateTime, pub completed_at: Option>, - /// What the next round's first increment returned. `1` means the id came - /// back as a new agent; `before_delete + 1` means the old one is still - /// there. + /// Whether the platform refused this delete by saying the agent was not + /// there, as opposed to refusing it for any other reason. + /// + /// Load-bearing rather than diagnostic. Deleting is not idempotent, and + /// worker-service retries a call whose executor became unreachable — so a + /// delete that *succeeded* on a pod that then died comes back to the caller + /// as not-found. Without this flag that reads as "the platform refused and + /// the agent is gone anyway", which is one of this scenario's violations. + #[serde(default)] + pub rejected_as_not_found: bool, + /// The counter the deletion left behind: `0` if the id came back as a new + /// agent, `before_delete` if the old one is still there. + /// + /// One meaning, whichever probe produced it, which is the point. A round is + /// normally probed by the *next* round's first increment, and an increment + /// reports the counter it just raised — so the value it leaves behind is + /// that minus one. The last round of a run has no increment after it and is + /// closed by a plain read instead, which reports the counter directly. + /// Storing the increment's own return would make those two probes disagree + /// by one, and only ever on the last round of each slot. That is exactly + /// the bug the first S6 run reported 125 times. pub observed_after: Option, } +/// The counter an operation left behind, given what the *next* increment +/// reported. +/// +/// One line, and it earns a name because getting it wrong is invisible. The +/// account judges `observed_after` as the counter a delete left behind, and the +/// two probes that produce it disagree by one: an increment reports the counter +/// it just raised, a plain read reports the counter itself. The first S6 run +/// stored the increment's own return and reported 125 partial-state findings — +/// exactly the slots whose last round was closed by a read. +pub fn counter_left_by(increment_returned: u64) -> u64 { + increment_returned.saturating_sub(1) +} + /// A running deletion workload. pub struct DeleteHandle { stop: Arc, @@ -163,12 +194,13 @@ pub async fn smoke_round(ctx: &WorkloadContext, config: &DeleteConfig) -> anyhow .map(u64::from) .ok_or_else(|| anyhow::anyhow!("smoke round: {agent} did not answer after being deleted"))?; - if after != FIRST_VALUE_OF_A_NEW_AGENT { + if after != COUNTER_OF_A_NEW_AGENT + 1 { anyhow::bail!( "smoke round: {agent} was worth {value}, was deleted, and its next increment \ - returned {after} rather than {FIRST_VALUE_OF_A_NEW_AGENT}. Deleting an agent does \ - not behave the way this scenario's whole account assumes, so the run would report \ - a resurrection on every round." + returned {after} rather than {}. Deleting an agent does not behave the way this \ + scenario's whole account assumes, so the run would report a resurrection on every \ + round.", + COUNTER_OF_A_NEW_AGENT + 1 ); } info!("S6: smoke round passed — a deleted agent came back as a new one"); @@ -228,7 +260,7 @@ pub fn start(ctx: WorkloadContext, config: &DeleteConfig) -> DeleteHandle { && let Ok(mut rounds) = rounds.lock() && let Some(entry) = rounds.get_mut(slot) { - entry.observed_after = Some(observed); + entry.observed_after = Some(counter_left_by(observed)); } match value { @@ -242,7 +274,7 @@ pub fn start(ctx: WorkloadContext, config: &DeleteConfig) -> DeleteHandle { let submitted_at = Utc::now(); submitted.fetch_add(1, Ordering::Relaxed); - let outcome = delete_once(&ctx, &agent, round).await; + let (outcome, rejected_as_not_found) = delete_once(&ctx, &agent, round).await; if let Ok(mut rounds) = rounds.lock() { rounds.push(DeleteRound { @@ -250,6 +282,7 @@ pub fn start(ctx: WorkloadContext, config: &DeleteConfig) -> DeleteHandle { round, before_delete: all_answered.then_some(last_value).flatten(), outcome, + rejected_as_not_found, submitted_at, completed_at: Some(Utc::now()), observed_after: None, @@ -273,8 +306,14 @@ pub fn start(ctx: WorkloadContext, config: &DeleteConfig) -> DeleteHandle { } } +/// The error code the platform returns when there is nothing to delete. +const AGENT_NOT_FOUND: &str = "AGENT_NOT_FOUND"; + /// One delete, with retries switched off. See the module docs. -async fn delete_once(ctx: &WorkloadContext, agent: &str, round: u32) -> Outcome { +/// +/// Returns the outcome and whether a refusal was specifically "no such agent", +/// which is the signature of a delete that had already taken effect. +async fn delete_once(ctx: &WorkloadContext, agent: &str, round: u32) -> (Outcome, bool) { let mut once = ctx.clone(); once.retry = RetryPolicy { transport_only: true, @@ -286,7 +325,7 @@ async fn delete_once(ctx: &WorkloadContext, agent: &str, round: u32) -> Outcome let agent_id = workload::counter_agent_id(&once, agent); let ctx2 = once.clone(); - workload::run_operation( + let result = workload::run_operation( &once, Stream::Delete, agent.to_string(), @@ -301,6 +340,45 @@ async fn delete_once(ctx: &WorkloadContext, agent: &str, round: u32) -> Outcome } }, ) - .await - .outcome + .await; + + let not_found = result + .error + .as_deref() + .is_some_and(|e| e.contains(AGENT_NOT_FOUND)); + (result.outcome, not_found) +} + +#[cfg(test)] +mod tests { + use super::*; + use test_r::test; + + /// The conversion the first S6 run got wrong, and which no test covered + /// because it lived in a spawned task rather than in the account. + /// + /// `observed_after` means the counter the delete left behind. An increment + /// reports the counter it just raised, so it is one more than that; a plain + /// read reports the counter directly and needs no conversion. Both probes + /// have to land on the same scale or every slot's last round — the only one + /// closed by a read — reads as a partial state. + #[test] + fn the_two_probes_agree_on_what_a_delete_left_behind() { + // A deleted agent: the next increment creates it and reports 1. + assert_eq!(counter_left_by(1), COUNTER_OF_A_NEW_AGENT); + // A survivor worth 3: the next increment reports 4. + assert_eq!(counter_left_by(4), 3); + // A plain read of the same two cases reports 0 and 3 with no + // conversion, which is what `close_last_rounds` stores. + for counter in [COUNTER_OF_A_NEW_AGENT, 3] { + assert_eq!(counter_left_by(counter + 1), counter); + } + } + + /// An increment that somehow reported nothing must not underflow into a + /// value the account would read as a survivor. + #[test] + fn a_zero_increment_does_not_wrap_around() { + assert_eq!(counter_left_by(0), 0); + } } diff --git a/integration-tests/src/chaos/result.rs b/integration-tests/src/chaos/result.rs index 61a4d958a0..1967a4c48f 100644 --- a/integration-tests/src/chaos/result.rs +++ b/integration-tests/src/chaos/result.rs @@ -388,9 +388,10 @@ mod tests { round: 0, before_delete: Some(3), outcome: crate::chaos::history::Outcome::Confirmed, + rejected_as_not_found: false, submitted_at: now, completed_at: Some(now), - observed_after: Some(1), + observed_after: Some(0), }], &split, Some(FaultWindow { diff --git a/integration-tests/src/chaos/resurrection.rs b/integration-tests/src/chaos/resurrection.rs index 06bffd6c91..f5209448fc 100644 --- a/integration-tests/src/chaos/resurrection.rs +++ b/integration-tests/src/chaos/resurrection.rs @@ -17,10 +17,10 @@ //! The same two-value oracle [`crate::chaos::truncation`] uses, one step //! further. There a round asked the platform to forget some of an agent's work; //! here it asks it to forget the agent. Invoking a deleted id creates a **new** -//! agent, so the next round's first increment has exactly two legal answers: +//! agent, so the counter a deletion leaves behind has exactly two legal values: //! -//! * `1` — the deletion took, and this is a fresh agent counting from nothing -//! * `V + 1` — the deletion did not take, and the old agent is still there +//! * `0` — the deletion took, and the id is a fresh agent counting from nothing +//! * `V` — the deletion did not take, and the old agent is still there //! //! Anything else means an agent came back carrying part of a state it was //! supposed to have lost. @@ -46,7 +46,7 @@ //! leaves an agent marked for deletion that was never removed. Whoever picks up //! its shard next decides what that means. -use crate::chaos::deletions::{DeleteRound, FIRST_VALUE_OF_A_NEW_AGENT}; +use crate::chaos::deletions::{COUNTER_OF_A_NEW_AGENT, DeleteRound}; use crate::chaos::history::Outcome; use crate::chaos::split::{FaultWindow, Group, PodSplit, Window}; use serde::{Deserialize, Serialize}; @@ -65,7 +65,11 @@ pub enum ResurrectionViolation { /// The agent came back worth neither nothing nor what it had been: part of /// a state it was supposed to have lost survived. PartialState, - /// The platform refused the deletion and the agent is gone anyway. + /// The platform refused the deletion and the agent is gone anyway, for a + /// reason other than not finding it. + /// + /// A refusal that says the agent was not there is **not** this: see + /// [`ResurrectionReport::deleted_despite_not_found`]. RefusedButDeleted, } @@ -142,6 +146,18 @@ pub struct ResurrectionReport { pub indeterminate_that_deleted: u64, /// Deletes the driver never heard back about that did not. pub indeterminate_that_did_not: u64, + /// Deletions the platform reported as `AGENT_NOT_FOUND` whose agent was + /// nonetheless gone afterwards. + /// + /// Not a violation, and the distinction matters. Deleting is not + /// idempotent — `delete_worker_internal` opens with a metadata lookup and + /// returns not-found when there is nothing there — and worker-service's + /// routing layer retries a call whose executor became unreachable. So a + /// delete that succeeded, on an executor that then died, is retried against + /// the new owner and comes back as not-found. The work happened; the answer + /// is misleading. That is worth an operator's attention and is not the + /// platform resurrecting anything. + pub deleted_despite_not_found: u64, pub unjudgeable: u64, pub unprobed: u64, pub cells: Vec, @@ -153,6 +169,8 @@ pub struct ResurrectionReport { #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Verdict { Deleted, + /// Gone, but reported to the caller as "no such agent". + DeletedDespiteNotFound, Survived, Violation(ResurrectionViolation), Unjudgeable, @@ -168,14 +186,18 @@ fn judge(round: &DeleteRound) -> Verdict { return Verdict::Unprobed; }; - if observed == FIRST_VALUE_OF_A_NEW_AGENT { - // Gone. Legitimate unless the platform said it refused. + if observed == COUNTER_OF_A_NEW_AGENT { + // Gone. Legitimate unless the platform said it refused — and even then, + // one refusal means the opposite of what it looks like. See below. if round.outcome == Outcome::Rejected { + if round.rejected_as_not_found { + return Verdict::DeletedDespiteNotFound; + } return Verdict::Violation(ResurrectionViolation::RefusedButDeleted); } return Verdict::Deleted; } - if observed == before + 1 { + if observed == before { // Still there. Legitimate unless the platform said it was gone. if round.outcome == Outcome::Confirmed { return Verdict::Violation(ResurrectionViolation::ResurrectedWithState); @@ -204,6 +226,7 @@ impl ResurrectionReport { deleted_exactly: 0, indeterminate_that_deleted: 0, indeterminate_that_did_not: 0, + deleted_despite_not_found: 0, unjudgeable: 0, unprobed: 0, cells: Vec::new(), @@ -244,6 +267,10 @@ impl ResurrectionReport { report.indeterminate_that_deleted += 1; } } + Verdict::DeletedDespiteNotFound => { + cell.deleted += 1; + report.deleted_despite_not_found += 1; + } Verdict::Survived => { cell.survived += 1; if round.outcome != Outcome::Confirmed { @@ -332,6 +359,17 @@ impl ResurrectionReport { )); } + if self.deleted_despite_not_found > 0 { + lines.push(format!( + "S6: {} deletion(s) came back as AGENT_NOT_FOUND and had in fact taken effect. \ + Deleting is not idempotent and worker-service retries a call whose executor \ + became unreachable, so a delete that succeeded on a dying pod is reported to \ + the caller as though the agent had never existed. The work happened; the \ + answer did not say so.", + self.deleted_despite_not_found + )); + } + let caught: u64 = self .caught_by_the_kill .iter() @@ -413,9 +451,8 @@ fn detail_for(violation: ResurrectionViolation, before: u64, observed: u64) -> S ), ResurrectionViolation::PartialState => format!( "the agent was worth {before} when it was deleted and the slot came back worth \ - {observed}, which is neither a new agent ({FIRST_VALUE_OF_A_NEW_AGENT}) nor the \ - old one ({}): part of a state it was supposed to have lost survived", - before + 1 + {observed}, which is neither a new agent ({COUNTER_OF_A_NEW_AGENT}) nor the old \ + one ({before}): part of a state it was supposed to have lost survived" ), ResurrectionViolation::RefusedButDeleted => format!( "the platform refused to delete an agent worth {before} and it is gone anyway — the \ @@ -472,6 +509,7 @@ mod tests { round: 0, before_delete: before, outcome, + rejected_as_not_found: false, submitted_at, completed_at: Some(submitted_at + TimeDelta::milliseconds(90)), observed_after: observed, @@ -495,7 +533,7 @@ mod tests { -10, Some(BEFORE), Outcome::Confirmed, - Some(1), + Some(0), )]); assert!(violations(&report).is_empty(), "{:?}", report.findings); assert_eq!(report.deleted_exactly, 1); @@ -511,7 +549,7 @@ mod tests { -10, Some(BEFORE), Outcome::Confirmed, - Some(BEFORE + 1), + Some(BEFORE), )]); assert_eq!( violations(&report), @@ -520,13 +558,13 @@ mod tests { assert!(report.has_violations(), "this must be able to fail the run"); let finding = &report.findings[0]; assert_eq!(finding.before, BEFORE); - assert_eq!(finding.observed, BEFORE + 1); + assert_eq!(finding.observed, BEFORE); } /// Its opposite: refused, and gone regardless. #[test] fn a_refused_deletion_that_happened_anyway_is_a_finding() { - let report = build(&[round(ON_POD, -10, Some(BEFORE), Outcome::Rejected, Some(1))]); + let report = build(&[round(ON_POD, -10, Some(BEFORE), Outcome::Rejected, Some(0))]); assert_eq!( violations(&report), vec![ResurrectionViolation::RefusedButDeleted] @@ -537,13 +575,13 @@ mod tests { /// to be gone survived. #[test] fn a_slot_that_came_back_part_way_is_a_finding() { - // Worth 3, so 1 or 4 were legal. It reported 3. + // Worth 3, so the delete was allowed to leave 0 or 3. It left 2. let report = build(&[round( ON_POD, -10, Some(BEFORE), Outcome::Confirmed, - Some(3), + Some(2), )]); assert_eq!( violations(&report), @@ -561,13 +599,13 @@ mod tests { #[test] fn a_deletion_in_doubt_may_land_either_way_without_being_a_finding() { let report = build(&[ - round(ON_POD, -10, Some(BEFORE), Outcome::Indeterminate, Some(1)), + round(ON_POD, -10, Some(BEFORE), Outcome::Indeterminate, Some(0)), round( CONTROL, -10, Some(BEFORE), Outcome::Indeterminate, - Some(BEFORE + 1), + Some(BEFORE), ), ]); assert!(violations(&report).is_empty(), "{:?}", report.findings); @@ -591,7 +629,7 @@ mod tests { // At before = 1 every value is one of the two legal answers, so no // observation can ever produce this finding. `require_delete` refuses // the configuration rather than shipping a blind third of the oracle. - for observed in [1, 2] { + for observed in [0, 1] { let report = build(&[round( ON_POD, -10, @@ -608,7 +646,7 @@ mod tests { #[test] fn a_round_whose_increments_never_answered_is_not_judged() { - let report = build(&[round(ON_POD, -10, None, Outcome::Confirmed, Some(1))]); + let report = build(&[round(ON_POD, -10, None, Outcome::Confirmed, Some(0))]); assert_eq!(report.unjudgeable, 1); assert!(report.findings.is_empty()); } @@ -629,7 +667,7 @@ mod tests { -100, Some(BEFORE), Outcome::Confirmed, - Some(1), + Some(0), )]); assert!(report.caught_by_the_kill.is_empty()); assert!( @@ -646,7 +684,7 @@ mod tests { /// scenario is about. #[test] fn deletes_unresolved_when_the_pod_died_are_reported_separately() { - let mut caught = round(ON_POD, -1, Some(BEFORE), Outcome::Indeterminate, Some(1)); + let mut caught = round(ON_POD, -1, Some(BEFORE), Outcome::Indeterminate, Some(0)); caught.completed_at = Some(t0() + TimeDelta::seconds(30)); let report = build(&[caught]); @@ -668,9 +706,9 @@ mod tests { #[test] fn rounds_are_split_by_group_and_window() { let report = build(&[ - round(ON_POD, -10, Some(BEFORE), Outcome::Confirmed, Some(1)), - round(ON_POD, 10, Some(BEFORE), Outcome::Confirmed, Some(1)), - round(CONTROL, 10, Some(BEFORE), Outcome::Confirmed, Some(1)), + round(ON_POD, -10, Some(BEFORE), Outcome::Confirmed, Some(0)), + round(ON_POD, 10, Some(BEFORE), Outcome::Confirmed, Some(0)), + round(CONTROL, 10, Some(BEFORE), Outcome::Confirmed, Some(0)), ]); let cell = |g, w| { report @@ -687,18 +725,100 @@ mod tests { ); } + /// The bug the first S6 run reported 125 times, pinned. + /// + /// A slot's last round has no increment after it and is closed by a plain + /// read instead. A read reports the counter directly; an increment reports + /// the counter it just raised. `observed_after` means the counter the + /// deletion left behind, so the *workload* subtracts one from the increment + /// and the final read stores its value as-is. Get that backwards and every + /// slot's last round reads as a partial state — 125 findings out of 138,275 + /// rounds, all in `after-fault`, all with the same shape. + #[test] + fn a_round_closed_by_the_final_read_is_judged_on_the_same_scale() { + // A deleted agent reads 0. That is the fresh value, not a partial one. + let closed_by_read = build(&[round( + ON_POD, + -10, + Some(BEFORE), + Outcome::Confirmed, + Some(COUNTER_OF_A_NEW_AGENT), + )]); + assert!( + violations(&closed_by_read).is_empty(), + "a final read of a deleted agent must not read as a partial state: {:?}", + closed_by_read.findings + ); + assert_eq!(closed_by_read.deleted_exactly, 1); + + // And a survivor read reports what it was worth, not one more. + let survivor = build(&[round( + ON_POD, + -10, + Some(BEFORE), + Outcome::Indeterminate, + Some(BEFORE), + )]); + assert!(violations(&survivor).is_empty(), "{:?}", survivor.findings); + assert_eq!(survivor.indeterminate_that_did_not, 1); + } + + /// The other thing the first run found, and the reason it is not a finding. + /// + /// Deleting is not idempotent, and worker-service retries a call whose + /// executor became unreachable. So a delete that succeeded on a pod that + /// then died is retried against the new owner and comes back + /// `AGENT_NOT_FOUND`. The agent really is gone; only the answer is wrong. + #[test] + fn a_not_found_refusal_whose_agent_is_gone_is_reported_not_failed() { + let mut r = round( + ON_POD, + -10, + Some(BEFORE), + Outcome::Rejected, + Some(COUNTER_OF_A_NEW_AGENT), + ); + r.rejected_as_not_found = true; + let report = build(&[r]); + + assert!(violations(&report).is_empty(), "{:?}", report.findings); + assert_eq!(report.deleted_despite_not_found, 1); + assert!( + report + .attention_lines() + .iter() + .any(|l| l.contains("AGENT_NOT_FOUND") && l.contains("had in fact taken effect")), + "the operator still has to be told: {:?}", + report.attention_lines() + ); + } + + /// Any *other* refusal with the agent gone is still a finding. The + /// not-found case is an exception with a mechanism behind it, not a blanket + /// excuse for refusals. + #[test] + fn a_refusal_that_is_not_about_finding_the_agent_still_fails_the_run() { + let mut r = round( + ON_POD, + -10, + Some(BEFORE), + Outcome::Rejected, + Some(COUNTER_OF_A_NEW_AGENT), + ); + r.rejected_as_not_found = false; + let report = build(&[r]); + + assert_eq!( + violations(&report), + vec![ResurrectionViolation::RefusedButDeleted] + ); + assert!(report.has_violations()); + } + #[test] fn findings_beyond_the_cap_are_counted_rather_than_carried() { let rounds: Vec = (0..MAX_FINDINGS + 4) - .map(|_| { - round( - ON_POD, - -10, - Some(BEFORE), - Outcome::Confirmed, - Some(BEFORE + 1), - ) - }) + .map(|_| round(ON_POD, -10, Some(BEFORE), Outcome::Confirmed, Some(BEFORE))) .collect(); let report = build(&rounds); assert_eq!(report.findings.len(), MAX_FINDINGS); diff --git a/integration-tests/src/chaos/reverts.rs b/integration-tests/src/chaos/reverts.rs index e609e2bb70..e21e15c351 100644 --- a/integration-tests/src/chaos/reverts.rs +++ b/integration-tests/src/chaos/reverts.rs @@ -79,6 +79,16 @@ pub struct RevertRound { pub observed_after: Option, } +/// The counter an operation left behind, given what the *next* increment +/// reported. +/// +/// Same conversion as [`crate::chaos::deletions::counter_left_by`] and named for +/// the same reason: the account judges `observed_after` as a counter, and the +/// increment probe reports one more than that. +pub fn counter_left_by(increment_returned: u64) -> u64 { + increment_returned.saturating_sub(1) +} + /// A running revert workload. pub struct RevertHandle { stop: Arc, @@ -168,7 +178,7 @@ pub fn start(ctx: WorkloadContext, config: &RevertConfig) -> RevertHandle { && let Ok(mut rounds) = rounds.lock() && let Some(entry) = rounds.get_mut(slot) { - entry.observed_after = Some(observed.saturating_sub(1)); + entry.observed_after = Some(counter_left_by(observed)); } match value { @@ -299,6 +309,16 @@ mod tests { assert_eq!(expected_after(&config(), 10), 20); } + /// The increment probe and the final read have to land on the same scale. + /// See [`crate::chaos::deletions::counter_left_by`] for the run that proved + /// what happens when they do not. + #[test] + fn the_two_probes_agree_on_what_a_revert_left_behind() { + assert_eq!(counter_left_by(9), 8); + assert_eq!(counter_left_by(1), 0); + assert_eq!(counter_left_by(0), 0); + } + /// A revert that takes back everything it added is legal and leaves the /// agent where it started. Nothing in the arithmetic may go negative. #[test] diff --git a/integration-tests/src/chaos/workload.rs b/integration-tests/src/chaos/workload.rs index a1d49d97d4..ebe6b8ae6b 100644 --- a/integration-tests/src/chaos/workload.rs +++ b/integration-tests/src/chaos/workload.rs @@ -209,11 +209,19 @@ struct AttemptResult { /// reports, so it needs each operation's answer in hand rather than having to /// go looking for its own record in a history every other emitter is also /// appending to. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone)] pub struct OperationOutcome { pub outcome: Outcome, /// The value the operation returned, for the methods that return one. pub value: Option, + /// The last error, verbatim, for callers that need to tell one refusal from + /// another. [`crate::chaos::deletions`] does: a delete refused because the + /// agent was not there means something quite different from any other + /// refusal. + /// + /// Only ever `Some` on a failure, so the clone costs nothing on the path + /// every scenario actually runs. + pub error: Option, } /// Runs one operation with the configured bounded, same-key retry, and records @@ -334,6 +342,7 @@ where ); } + let error = last.error.as_ref().map(|e| format!("{e:#}")); ctx.history.record(OperationRecord { op_id, stream, @@ -348,7 +357,7 @@ where duration_ms: started.elapsed().as_millis().min(u64::MAX as u128) as u64, returned_value: last.value, first_attempt_value, - error: last.error.as_ref().map(|e| format!("{e:#}")), + error: error.clone(), error_class: last.class, attempt_log, }); @@ -356,6 +365,7 @@ where OperationOutcome { outcome, value: last.value, + error, } } From cb208e54917352af9324a15549d684cb3cae58a9 Mon Sep 17 00:00:00 2001 From: kmatasfp <33095685+kmatas@users.noreply.github.com> Date: Mon, 24 Aug 2026 20:10:44 -0700 Subject: [PATCH 19/40] Add S9 chaos scenario driver and rollback account --- golem-test-framework/src/benchmark/config.rs | 2 + .../chaos_suites/cloud-chaos.yaml | 86 +++ integration-tests/src/benchmarks/all.rs | 4 + integration-tests/src/chaos/mod.rs | 81 ++- integration-tests/src/chaos/result.rs | 80 ++- integration-tests/src/chaos/rollback.rs | 399 +++++++++++ integration-tests/src/chaos/scenarios/mod.rs | 2 + integration-tests/src/chaos/scenarios/s9.rs | 644 ++++++++++++++++++ integration-tests/src/chaos/summary.rs | 19 + 9 files changed, 1315 insertions(+), 2 deletions(-) create mode 100644 integration-tests/src/chaos/rollback.rs create mode 100644 integration-tests/src/chaos/scenarios/s9.rs diff --git a/golem-test-framework/src/benchmark/config.rs b/golem-test-framework/src/benchmark/config.rs index a0c4044e89..a0a90c0f02 100644 --- a/golem-test-framework/src/benchmark/config.rs +++ b/golem-test-framework/src/benchmark/config.rs @@ -252,6 +252,8 @@ pub enum ChaosScenarioArg { S7, /// Executor pod kill while agents are being deleted. S6, + /// Executor pod kill while a component rollback is in flight. + S9, } /// Density subcommand action. diff --git a/integration-tests/chaos_suites/cloud-chaos.yaml b/integration-tests/chaos_suites/cloud-chaos.yaml index 6adf181901..eec3558738 100644 --- a/integration-tests/chaos_suites/cloud-chaos.yaml +++ b/integration-tests/chaos_suites/cloud-chaos.yaml @@ -767,3 +767,89 @@ scenarios: delaySecs: 5 signalTimeoutSecs: 1800 + + # S9 — executor pod kill during a component rollback (GOL-369). + # + # S5 moves agents forward onto a new build and kills an executor while that is + # happening. S9 moves them forward, waits for that to land, and then moves them + # back — killing an executor during the return leg. + # + # The return leg is the one that matters operationally. A rollback is what you + # reach for when the new build is already going wrong, so a rollback happening + # under a dying executor is the situation you would actually be in. + # + # The rollback is a redeploy: `agent-counters` is uploaded again as a new + # revision and every agent is asked to move to it. `Counter::component_version` + # is compiled into each build, so an agent that has genuinely returned reports + # 1 from the code that is running rather than from metadata about what the + # platform believes. + - code: S9 + name: executor-crash-during-rollback + enabled: true + + fault: + kind: pod-kill + target: worker-executor + # `one` plus the namespace and opt-in label selectors in podchaos-s9.yaml + # bound the blast radius. Unlike S8 the driver does not name the pod: + # agents are mid-rollback across both executors, so either one interrupts + # returns in flight and which one carries no information. + mode: one + durationSecs: 60 + + phases: + # Long enough for cold starts and route warm-up to settle, so the roll + # forward is applied to a population that is running rather than arriving. + baselineSecs: 300 + # Has to cover the kill, the reschedule, and the rollback finishing on + # both the surviving executor and the replacement. + faultSecs: 120 + # Returns continue after the kill and agents have to converge before the + # census decides which build they are on. + recoverySecs: 300 + + workload: + # Durable agents are the population being rolled back, so the pool is + # large: a stuck agent localises to one out of 200. + durableAgents: 200 + # Not rolled back — they exist to show whether acceptance degraded during + # the kill, separately from the rollback. + ephemeralAgents: 50 + # Durable state the rollback does not touch, so they are the control: + # their read-back should be unaffected. + scheduledAgents: 50 + promiseAgents: 0 + # No quota stream, for the same reason as S5: a lease that cannot be + # renewed would add a second failure mode to a scenario that has one. + quotaAgents: 0 + ratePerSec: 100 + + rollback: + # How long to let the roll-forward land before measuring it. Too short and + # the census reads a population still in transit, and the gate below + # refuses a rollback that would have been fine. + settleSecs: 90 + # The share of agents that must actually report the new build before the + # rollback is attempted at all. Below it the driver stops: rolling agents + # back to a build they never left would pass every check and prove + # nothing, which is the worst artifact this suite can produce. + rolledForwardFloorPercent: 90 + # Retries for the rollback *control plane*, counted apart from the + # workload's own. Different question: a request refused because its + # agent's executor just died says nothing about correctness, but an agent + # nobody successfully asked to come back explains a stale agent later + # without excusing one. + controlRetries: 2 + controlRetryDelaySecs: 5 + # How far into the rollback to ask for the kill. As in S5 this is when the + # driver *asks*; applying the PodChaos and confirming it takes a few + # seconds more, and the result records both so how far in it landed is + # readable rather than assumed. + killDelaySecs: 2 + + retryPolicy: + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 diff --git a/integration-tests/src/benchmarks/all.rs b/integration-tests/src/benchmarks/all.rs index d48ac081fd..7a9b9876f7 100644 --- a/integration-tests/src/benchmarks/all.rs +++ b/integration-tests/src/benchmarks/all.rs @@ -597,6 +597,7 @@ async fn run_chaos( ChaosScenarioArg::S3 => chaos::ScenarioCode::S3, ChaosScenarioArg::S7 => chaos::ScenarioCode::S7, ChaosScenarioArg::S6 => chaos::ScenarioCode::S6, + ChaosScenarioArg::S9 => chaos::ScenarioCode::S9, }; let config = suite .scenario(code, allow_disabled) @@ -643,6 +644,9 @@ async fn run_chaos( chaos::ScenarioCode::S6 => { chaos::scenarios::s6::run(&config, &manifest, &deps, &signals, &outputs).await } + chaos::ScenarioCode::S9 => { + chaos::scenarios::s9::run(&config, &manifest, &deps, &signals, &outputs).await + } }; deps.kill_all().await; diff --git a/integration-tests/src/chaos/mod.rs b/integration-tests/src/chaos/mod.rs index 32e97a4a88..a0d11e55a1 100644 --- a/integration-tests/src/chaos/mod.rs +++ b/integration-tests/src/chaos/mod.rs @@ -46,6 +46,7 @@ pub mod reachability; pub mod result; pub mod resurrection; pub mod reverts; +pub mod rollback; pub mod scenarios; pub mod scheduled; pub mod signal; @@ -86,6 +87,8 @@ pub enum ScenarioCode { S7, /// Executor pod kill while agents are being deleted. S6, + /// Executor pod kill while a component rollback is in flight. + S9, } impl ScenarioCode { @@ -101,19 +104,21 @@ impl ScenarioCode { ScenarioCode::S3 => "S3", ScenarioCode::S7 => "S7", ScenarioCode::S6 => "S6", + ScenarioCode::S9 => "S9", } } /// Every scenario this driver implements. The suite YAML is checked against /// this list, so a scenario cannot be enabled in YAML without code behind /// it, nor implemented without an operational switch in front of it. - pub const ALL: [ScenarioCode; 10] = [ + pub const ALL: [ScenarioCode; 11] = [ ScenarioCode::S1, ScenarioCode::S3, ScenarioCode::S5, ScenarioCode::S6, ScenarioCode::S7, ScenarioCode::S8, + ScenarioCode::S9, ScenarioCode::S10, ScenarioCode::S11, ScenarioCode::S12, @@ -587,6 +592,60 @@ impl DeleteConfig { } } +/// Shape of the component rollback (GOL-369). +/// +/// S5 moves agents forward onto a new build and kills an executor while that is +/// happening. S9 moves them forward, waits for that to land, and then moves them +/// **back**, killing an executor during the return leg. +/// +/// The rollback is a redeploy: the original artifact is uploaded again as a new +/// revision, and every agent is asked to move to it. That is what "rollback" +/// means operationally, and it is what makes the evidence unambiguous — +/// `Counter::component_version` is compiled into each build, so an agent that +/// has genuinely returned reports `1` from the code that is actually running, +/// not from metadata about what the platform believes. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RollbackConfig { + /// How long to let the roll-forward settle before rolling back. + /// + /// Load-bearing. If the agents never reached the new build, the rollback + /// returns them to a build they never left and the run proves nothing, so + /// this has to be long enough for the forward leg to finish on a healthy + /// cluster. + pub settle_secs: u64, + /// The share of agents that must actually report the new build before the + /// rollback is worth attempting, as a percentage. + /// + /// Below it the driver stops rather than spending the maintenance window on + /// a return journey nobody made. The same instinct as S6's smoke round. + pub rolled_forward_floor_percent: f64, + /// Retries for the rollback *control plane* — the per-agent update + /// requests — recorded apart from the workload's own retries. + /// + /// Separate because they answer different questions. The workload's retry + /// exists to expose duplicate execution and is deliberately one attempt. + /// This one exists so that a refused rollback request is distinguishable + /// from a rollback that was never asked for, which matters when the + /// executor owning an agent is about to be killed. + pub control_retries: u32, + pub control_retry_delay_secs: u64, + /// How far into the rollback to ask for the kill. + pub kill_delay_secs: u64, +} + +impl RollbackConfig { + pub fn settle(&self) -> Duration { + Duration::from_secs(self.settle_secs) + } + pub fn control_retry_delay(&self) -> Duration { + Duration::from_secs(self.control_retry_delay_secs) + } + pub fn kill_delay(&self) -> Duration { + Duration::from_secs(self.kill_delay_secs) + } +} + /// One step of the executor scale schedule the workflow runs during the fault. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -674,6 +733,9 @@ pub struct ScenarioConfig { /// The deletion workload. Absent for scenarios that do not run one. #[serde(default)] pub delete: Option, + /// The component rollback. Absent for scenarios that do not run one. + #[serde(default)] + pub rollback: Option, /// Shard-ownership oracle settings. Absent for scenarios that do not sample /// executor assignments. #[serde(default)] @@ -810,6 +872,16 @@ impl ScenarioConfig { Ok(config) } + /// The rollback block. See [`Self::require_workload`]. + pub fn require_rollback(&self) -> anyhow::Result<&RollbackConfig> { + self.rollback.as_ref().ok_or_else(|| { + anyhow::anyhow!( + "chaos scenario {} needs a `rollback` block in the suite YAML", + self.code + ) + }) + } + /// The pinned workload block. See [`Self::require_workload`]. pub fn require_pinned(&self) -> anyhow::Result<&PinnedConfig> { self.pinned.as_ref().ok_or_else(|| { @@ -948,6 +1020,7 @@ mod tests { isolation: None, revert: None, delete: None, + rollback: None, ownership: None, scale_during_fault: None, retry_policy: RetryPolicy::default(), @@ -983,6 +1056,7 @@ mod tests { promise: None, isolation: None, delete: None, + rollback: None, revert: Some(RevertConfig { agents: 10, increments_per_round: increments, @@ -1090,6 +1164,7 @@ mod tests { assert_eq!(ScenarioCode::parse("s3"), Some(ScenarioCode::S3)); assert_eq!(ScenarioCode::parse("s7"), Some(ScenarioCode::S7)); assert_eq!(ScenarioCode::parse("s6"), Some(ScenarioCode::S6)); + assert_eq!(ScenarioCode::parse("s9"), Some(ScenarioCode::S9)); assert_eq!(ScenarioCode::parse("S99"), None); } @@ -1131,6 +1206,10 @@ mod tests { ScenarioCode::S6 => { entry.require_delete().unwrap(); } + ScenarioCode::S9 => { + entry.require_workload().unwrap(); + entry.require_rollback().unwrap(); + } } } } diff --git a/integration-tests/src/chaos/result.rs b/integration-tests/src/chaos/result.rs index 1967a4c48f..9f058c5c3c 100644 --- a/integration-tests/src/chaos/result.rs +++ b/integration-tests/src/chaos/result.rs @@ -30,7 +30,7 @@ use crate::chaos::split::PodSplit; use crate::chaos::summary::{ChaosSummary, TerminationReason}; use crate::chaos::{ DeleteConfig, FaultConfig, IsolationConfig, PinnedConfig, PromiseConfig, RetryPolicy, - RevertConfig, ScheduledConfig, WorkloadConfig, + RevertConfig, RollbackConfig, ScheduledConfig, WorkloadConfig, }; use chrono::{DateTime, Utc}; use golem_test_framework::benchmark::RunMetadata; @@ -176,6 +176,9 @@ pub struct ChaosResult { /// Present only for S6. #[serde(default, skip_serializing_if = "Option::is_none")] pub delete_selection: Option, + /// The component rollback the run was configured with, if any. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rollback: Option, pub retry_policy: RetryPolicy, pub scope: RunScope, pub summary: ChaosSummary, @@ -253,6 +256,7 @@ mod tests { revert_selection: None, delete: None, delete_selection: None, + rollback: None, retry_policy: RetryPolicy::default(), scope: RunScope { environment_id: "env-1".to_string(), @@ -353,6 +357,78 @@ mod tests { assert!(parsed.promise_selection.is_some()); } + /// The S9 shape. Same contract as the others: the investigation report in + /// golem-cloud reads these fields by name. + #[test] + fn an_s9_result_carries_the_rollback_fields_the_investigation_report_reads() { + use crate::chaos::rollback::{ControlPlaneAttempts, RollbackReport, VersionCensus}; + + let now = Utc::now(); + let mut forward = std::collections::BTreeMap::new(); + forward.insert("chaos-s9-durable-0000".to_string(), Some(2u32)); + let mut back = std::collections::BTreeMap::new(); + back.insert("chaos-s9-durable-0000".to_string(), Some(1u32)); + + let mut result = sample_result(TerminationReason::Completed); + result.scenario_code = "S9".to_string(); + result.rollback = Some(crate::chaos::RollbackConfig { + settle_secs: 90, + rolled_forward_floor_percent: 90.0, + control_retries: 2, + control_retry_delay_secs: 5, + kill_delay_secs: 2, + }); + result.summary = ChaosSummary::build(&[], Vec::new(), Vec::new(), Some(now)).with_rollback( + RollbackReport { + forward_revision: 2, + rollback_revision: 3, + forward_version: 2, + rollback_version: 1, + rolled_forward: VersionCensus::build("before-rollback", 2, &forward), + rolled_back: Some(VersionCensus::build("after-recovery", 1, &back)), + control: ControlPlaneAttempts { + requested: 200, + accepted_first_try: 198, + accepted_after_retry: 2, + refused: 0, + max_retries: 2, + }, + rolled_forward_floor_percent: 90.0, + }, + ); + + let json = serde_json::to_value(&result).unwrap(); + let rollback = &json["summary"]["rollback"]; + for key in [ + "forwardRevision", + "rollbackRevision", + "forwardVersion", + "rollbackVersion", + "rolledForward", + "rolledBack", + "control", + "rolledForwardFloorPercent", + ] { + assert!( + !rollback[key].is_null(), + "summary.rollback.{key} is what the investigation report reads" + ); + } + for key in [ + "requested", + "acceptedFirstTry", + "acceptedAfterRetry", + "refused", + ] { + assert!(!rollback["control"][key].is_null(), "control.{key}"); + } + assert_eq!(json["rollback"]["rolledForwardFloorPercent"], 90.0); + + let parsed: ChaosResult = serde_json::from_str(&json.to_string()).unwrap(); + assert_eq!(parsed.scenario_code, "S9"); + assert!(parsed.summary.rollback.is_some()); + } + /// The S6 shape. Same contract as the S3, S7, S10 and S11 tests: the /// investigation report in golem-cloud reads these fields by name. #[test] @@ -930,6 +1006,7 @@ mod sample_artifact { revert_selection: None, delete: None, delete_selection: None, + rollback: None, retry_policy: RetryPolicy::default(), scope: RunScope { environment_id: "0192f000-0000-7000-8000-000000000001".to_string(), @@ -1150,6 +1227,7 @@ mod sample_artifact { revert_selection: None, delete: None, delete_selection: None, + rollback: None, retry_policy: RetryPolicy::default(), scope: RunScope { environment_id: "0192f000-0000-7000-8000-000000000001".to_string(), diff --git a/integration-tests/src/chaos/rollback.rs b/integration-tests/src/chaos/rollback.rs new file mode 100644 index 0000000000..49752eb2a3 --- /dev/null +++ b/integration-tests/src/chaos/rollback.rs @@ -0,0 +1,399 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Did every agent come back to the build it was rolled back to (GOL-369)? +//! +//! S5 asks whether agents reach a *new* build when an executor dies mid-update. +//! S9 asks the return question, and the return leg is the one that matters +//! operationally: a rollback is what you reach for when the new build is +//! already going wrong, so it happening under a dying executor is exactly the +//! situation you would be in. +//! +//! ### Why the evidence is the running code, not the metadata +//! +//! `Counter::component_version` is compiled into each build — `1` in +//! `agent-counters`, `2` in `agent-counters-v2`, and nothing else differs +//! between them. Component metadata says which revision the platform *believes* +//! an agent is on; invoking `component_version` says what the code actually +//! executing reports. Only the second can distinguish a rollback that landed +//! from one the platform merely recorded. +//! +//! ### Why the forward leg is verified before the backward one +//! +//! If the agents never reached the new build, rolling them back returns them to +//! a build they never left, every check passes, and the run proves nothing. So +//! the forward leg is measured and the rollback is refused outright if too few +//! agents made it. The same instinct as S6's smoke round and S10's +//! how-much-did-the-kill-catch line: a clean report from a scenario that never +//! happened is the worst artifact this suite can produce. +//! +//! ### Why the control-plane retries are counted apart +//! +//! The workload's retry is one attempt, transport-only, under the original +//! idempotency key, and it exists to *expose* duplicate execution. The +//! rollback's per-agent update requests are a different thing entirely: they are +//! control-plane calls aimed at agents whose executor is about to be killed, and +//! a request refused because its owner just died says nothing about the +//! platform's correctness. Counting them together would let control-plane noise +//! read as workload trouble. + +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +/// What the agents were running when asked. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct VersionCensus { + /// When the census was taken, for the report to label it. + pub at: String, + /// The version every agent was expected to report. + pub expected: u32, + pub agents: usize, + pub on_expected: usize, + /// Agents on some other build, by the version they reported. Non-empty is + /// the finding; the key says which build they are stuck on. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub on_other: BTreeMap, + /// Agents that answered nothing. Neither passed nor failed: an agent that + /// cannot be read says nothing either way about which build it is on. + pub unreadable: usize, +} + +impl VersionCensus { + /// Builds a census from what each agent reported. + pub fn build(at: &str, expected: u32, observed: &BTreeMap>) -> Self { + let mut on_expected = 0; + let mut unreadable = 0; + let mut on_other: BTreeMap = BTreeMap::new(); + for version in observed.values() { + match version { + Some(v) if *v == expected => on_expected += 1, + Some(v) => *on_other.entry(*v).or_default() += 1, + None => unreadable += 1, + } + } + VersionCensus { + at: at.to_string(), + expected, + agents: observed.len(), + on_expected, + on_other, + unreadable, + } + } + + /// The share of agents on the expected build, out of those that answered. + /// + /// Unreadable agents are excluded from both halves rather than counted as + /// failures: they are reported separately, and treating silence as a wrong + /// answer would let a flaky read block a rollback that was fine. + pub fn share_of_answered_percent(&self) -> Option { + let answered = self.agents - self.unreadable; + (answered > 0).then(|| self.on_expected as f64 * 100.0 / answered as f64) + } +} + +/// Rollback requests, counted apart from the workload's own retries. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ControlPlaneAttempts { + pub requested: u64, + pub accepted_first_try: u64, + pub accepted_after_retry: u64, + /// Requests that never got through, even after the configured retries. Each + /// one is an agent nobody asked to come back, so it explains a stale agent + /// without excusing one. + pub refused: u64, + pub max_retries: u32, +} + +impl ControlPlaneAttempts { + pub fn accepted(&self) -> u64 { + self.accepted_first_try + self.accepted_after_retry + } +} + +/// The rollback account. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RollbackReport { + /// The revision the agents were moved forward to, and the one they were + /// rolled back to. The second carries the original build's code. + pub forward_revision: u64, + pub rollback_revision: u64, + /// What the running code reports on each of those builds. + pub forward_version: u32, + pub rollback_version: u32, + /// The forward leg, measured before the rollback was attempted. + pub rolled_forward: VersionCensus, + /// After recovery. `None` if the run aborted before it got there. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rolled_back: Option, + pub control: ControlPlaneAttempts, + /// The floor the forward leg had to clear for the rollback to be worth + /// attempting, from the suite YAML. + pub rolled_forward_floor_percent: f64, +} + +impl RollbackReport { + /// Whether enough agents reached the new build for a rollback to mean + /// anything. + pub fn forward_leg_landed(&self) -> bool { + self.rolled_forward + .share_of_answered_percent() + .is_some_and(|share| share >= self.rolled_forward_floor_percent) + } + + /// Agents still on the build they were supposed to leave, after recovery. + pub fn stuck_on_the_new_build(&self) -> usize { + self.rolled_back + .as_ref() + .and_then(|census| census.on_other.get(&self.forward_version).copied()) + .unwrap_or(0) + } + + /// The lines that need a human. + pub fn attention_lines(&self) -> Vec { + let mut lines = Vec::new(); + + if !self.forward_leg_landed() { + lines.push(format!( + "S9: only {} of {} agents reached revision {} before the rollback ({}% of those \ + that answered, against a {:.0}% floor). Rolling agents back to a build they \ + never left proves nothing, so this run does not test rollback.", + self.rolled_forward.on_expected, + self.rolled_forward.agents, + self.forward_revision, + self.rolled_forward + .share_of_answered_percent() + .map(|s| format!("{s:.1}")) + .unwrap_or_else(|| "n/a".to_string()), + self.rolled_forward_floor_percent + )); + } + + let stuck = self.stuck_on_the_new_build(); + if stuck > 0 { + lines.push(format!( + "S9: {stuck} agent(s) still report component version {} after recovery, not the \ + {} they were rolled back to", + self.forward_version, self.rollback_version + )); + } + + if self.control.refused > 0 { + lines.push(format!( + "S9: {} of {} rollback requests were refused even after {} control-plane \ + retries. Those agents were never asked to come back, which explains a stale \ + agent without excusing one.", + self.control.refused, self.control.requested, self.control.max_retries + )); + } + + if let Some(census) = &self.rolled_back + && census.unreadable > 0 + { + lines.push(format!( + "S9: {} agent(s) could not be read after recovery, so the run cannot say which \ + build they are on", + census.unreadable + )); + } + lines + } + + /// Lines a reader needs in order to interpret the run. + pub fn note_lines(&self) -> Vec { + let mut lines = vec![format!( + "S9: revision {} carries build v{}, revision {} carries build v{} again", + self.forward_revision, + self.forward_version, + self.rollback_revision, + self.rollback_version + )]; + lines.push(format!( + "S9 forward leg: {} of {} agents on version {} before the rollback ({} unreadable)", + self.rolled_forward.on_expected, + self.rolled_forward.agents, + self.forward_version, + self.rolled_forward.unreadable + )); + lines.push(format!( + "S9 rollback requests: {} asked, {} accepted first try, {} after a retry, {} refused \ + (up to {} control-plane retries)", + self.control.requested, + self.control.accepted_first_try, + self.control.accepted_after_retry, + self.control.refused, + self.control.max_retries + )); + if let Some(census) = &self.rolled_back { + lines.push(format!( + "S9 return leg: {} of {} agents on version {} after recovery ({} unreadable, \ + {:?} elsewhere)", + census.on_expected, + census.agents, + census.expected, + census.unreadable, + census.on_other + )); + } + lines + } +} + +#[cfg(test)] +mod tests { + use super::*; + use test_r::test; + + fn census( + expected: u32, + on_expected: usize, + on_other: &[(u32, usize)], + unreadable: usize, + ) -> VersionCensus { + let mut observed: BTreeMap> = BTreeMap::new(); + let mut n = 0; + for _ in 0..on_expected { + observed.insert(format!("agent-{n:04}"), Some(expected)); + n += 1; + } + for (version, count) in on_other { + for _ in 0..*count { + observed.insert(format!("agent-{n:04}"), Some(*version)); + n += 1; + } + } + for _ in 0..unreadable { + observed.insert(format!("agent-{n:04}"), None); + n += 1; + } + VersionCensus::build("test", expected, &observed) + } + + fn report(forward: VersionCensus, back: Option) -> RollbackReport { + RollbackReport { + forward_revision: 2, + rollback_revision: 3, + forward_version: 2, + rollback_version: 1, + rolled_forward: forward, + rolled_back: back, + control: ControlPlaneAttempts { + requested: 200, + accepted_first_try: 200, + max_retries: 2, + ..Default::default() + }, + rolled_forward_floor_percent: 90.0, + } + } + + /// An unreadable agent is not a wrong answer. Counting it as one would let a + /// flaky read block a rollback that was perfectly fine. + #[test] + fn silence_is_excluded_from_the_share_rather_than_counted_against_it() { + // 90 on the expected build, 10 silent, none actually wrong. + let c = census(2, 90, &[], 10); + assert_eq!(c.agents, 100); + assert_eq!(c.unreadable, 10); + assert_eq!(c.share_of_answered_percent(), Some(100.0)); + assert!(report(c, None).forward_leg_landed()); + } + + /// The gate the whole scenario rests on: rolling agents back to a build + /// they never left would pass every check downstream. + #[test] + fn a_forward_leg_that_did_not_land_refuses_the_rollback() { + // 50 of 100 answered agents made it, against a 90% floor. + let r = report(census(2, 50, &[(1, 50)], 0), None); + assert!(!r.forward_leg_landed()); + assert!( + r.attention_lines() + .iter() + .any(|l| l.contains("does not test rollback")), + "the operator has to be told the run proved nothing: {:?}", + r.attention_lines() + ); + } + + /// A census with nothing readable cannot clear the gate, rather than + /// clearing it vacuously on an empty average. + #[test] + fn a_census_nobody_answered_does_not_clear_the_gate() { + let c = census(2, 0, &[], 40); + assert_eq!(c.share_of_answered_percent(), None); + assert!(!report(c, None).forward_leg_landed()); + } + + /// The finding: agents still on the build they were rolled back from. + #[test] + fn agents_still_on_the_old_build_after_recovery_are_raised() { + let r = report(census(2, 200, &[], 0), Some(census(1, 197, &[(2, 3)], 0))); + assert_eq!(r.stuck_on_the_new_build(), 3); + assert!( + r.attention_lines() + .iter() + .any(|l| l.contains("still report component version 2")), + "{:?}", + r.attention_lines() + ); + } + + /// A clean return raises nothing. + #[test] + fn a_rollback_that_landed_everywhere_raises_nothing() { + let r = report(census(2, 200, &[], 0), Some(census(1, 200, &[], 0))); + assert_eq!(r.stuck_on_the_new_build(), 0); + assert!(r.attention_lines().is_empty(), "{:?}", r.attention_lines()); + } + + /// A refused control-plane request explains a stale agent without excusing + /// one, so it is raised even when the return leg otherwise looks clean. + #[test] + fn refused_rollback_requests_are_raised_even_on_a_clean_return() { + let mut r = report(census(2, 200, &[], 0), Some(census(1, 200, &[], 0))); + r.control = ControlPlaneAttempts { + requested: 200, + accepted_first_try: 190, + accepted_after_retry: 6, + refused: 4, + max_retries: 2, + }; + assert_eq!(r.control.accepted(), 196); + assert!( + r.attention_lines() + .iter() + .any(|l| l.contains("were refused even after")), + "{:?}", + r.attention_lines() + ); + } + + /// Retries are counted apart from first-try acceptances, because the two + /// say different things about how the control plane behaved under a kill. + #[test] + fn first_try_and_retried_acceptances_are_counted_apart() { + let control = ControlPlaneAttempts { + requested: 10, + accepted_first_try: 7, + accepted_after_retry: 2, + refused: 1, + max_retries: 2, + }; + assert_eq!(control.accepted(), 9); + assert_eq!(control.accepted() + control.refused, control.requested); + } +} diff --git a/integration-tests/src/chaos/scenarios/mod.rs b/integration-tests/src/chaos/scenarios/mod.rs index c3b1e1254d..b0232d777d 100644 --- a/integration-tests/src/chaos/scenarios/mod.rs +++ b/integration-tests/src/chaos/scenarios/mod.rs @@ -35,6 +35,7 @@ pub mod s5; pub mod s6; pub mod s7; pub mod s8; +pub mod s9; use crate::chaos::ScenarioConfig; use crate::chaos::history::{OperationHistory, OperationRecord, Stream}; @@ -130,6 +131,7 @@ pub fn build_result(config: &ScenarioConfig, outcome: ScenarioOutcome) -> ChaosR revert_selection: outcome.revert_selection, delete: config.delete.clone(), delete_selection: outcome.delete_selection, + rollback: config.rollback.clone(), retry_policy: config.retry_policy.clone(), scope: outcome.scope, summary: outcome.summary, diff --git a/integration-tests/src/chaos/scenarios/s9.rs b/integration-tests/src/chaos/scenarios/s9.rs new file mode 100644 index 0000000000..83b25642db --- /dev/null +++ b/integration-tests/src/chaos/scenarios/s9.rs @@ -0,0 +1,644 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! S9 — executor pod kill during a component rollback (GOL-369). +//! +//! S5 moves agents forward onto a new build and kills an executor while that is +//! happening. S9 moves them forward, waits for that to land, and then moves them +//! **back**, killing an executor during the return leg. +//! +//! The return leg is the one that matters operationally. A rollback is what you +//! reach for when the new build is already going wrong, so a rollback happening +//! under a dying executor is not a contrived situation — it is the situation you +//! would actually be in. +//! +//! ## What a rollback is here +//! +//! A redeploy. The original artifact is uploaded again as a **new** revision and +//! every agent is asked to move to it. That is what rollback means in practice, +//! and it is what makes the evidence unambiguous: `Counter::component_version` +//! is compiled into each build — `1` in `agent-counters`, `2` in +//! `agent-counters-v2`, nothing else different — so an agent that has genuinely +//! returned reports `1` from the code that is running, not from metadata about +//! what the platform believes. +//! +//! ## Why the forward leg is verified first +//! +//! If the agents never reached the new build, rolling them back returns them to +//! a build they never left, every check passes, and the run proves nothing. So +//! the forward leg is measured and the rollback is refused outright if too few +//! agents made it. Same instinct as S6's smoke round: a clean report from a +//! scenario that never happened is the worst artifact this suite can produce. +//! +//! ## What fails the run +//! +//! The same two things S5 asserts, reused rather than re-invented because they +//! are the same facts: +//! +//! - an agent whose durable state fell below what the driver was told succeeded, +//! or rose above what it could possibly have asked for; +//! - an agent still reporting the build it was rolled back *from*. +//! +//! An agent that cannot be read at all is reported rather than assumed either +//! way, and control-plane refusals are counted apart from workload retries — +//! see [`crate::chaos::rollback`] for why that separation is load-bearing. + +use crate::chaos::history::{OperationHistory, OperationRecord, Phase, Stream}; +use crate::chaos::prep::{COUNTERS_V2_WASM, COUNTERS_WASM, ChaosPrepManifest}; +use crate::chaos::result::{ChaosResult, PhaseWindow, Phases, RunScope}; +use crate::chaos::rollback::{ControlPlaneAttempts, RollbackReport, VersionCensus}; +use crate::chaos::scenarios::{ + OutputPaths, ScenarioOutcome, WARMUP_SETTLE, build_result, readback_for, signal_termination, + snapshot_routing, wait_for_settled_routing, warm_up, write_outputs, +}; +use crate::chaos::signal::{BaselineReady, FaultSignals}; +use crate::chaos::summary::{ + AgentReadback, ChaosSummary, Note, ReadbackVerdict, TerminationReason, + stream_that_never_succeeded, +}; +use crate::chaos::workload::{self, PhaseMarker, WorkloadContext}; +use crate::chaos::{ScenarioCode, ScenarioConfig}; +use chrono::Utc; +use golem_test_framework::config::BenchmarkTestDependencies; +use golem_test_framework::dsl::TestDsl; +use std::time::Duration; +use tracing::{info, warn}; + +/// How long to wait after stopping the workload before reading durable state. +const SETTLE_BEFORE_READBACK: Duration = Duration::from_secs(30); + +/// How many agents to update concurrently. +/// +/// Update requests are cheap to issue and the point is that many are in flight +/// when the executor dies, so this is wide rather than polite. +const UPDATE_CONCURRENCY: usize = 32; + +/// What the running code reports on each of the two builds. +/// +/// Compiled into the WASM rather than read from metadata, which is the whole +/// reason these numbers can be trusted: metadata says what the platform +/// believes, `component_version` says what is executing. +const VERSION_ON_THE_NEW_BUILD: u32 = 2; +const VERSION_AFTER_ROLLBACK: u32 = 1; + +pub async fn run( + config: &ScenarioConfig, + manifest: &ChaosPrepManifest, + deps: &BenchmarkTestDependencies, + signals: &FaultSignals, + outputs: &OutputPaths, +) -> anyhow::Result { + let started_at = Utc::now(); + let workload_config = config.require_workload()?; + let rollback_config = config.require_rollback()?; + let history = OperationHistory::new(ScenarioCode::S9.as_str()); + let key_prefix = crate::chaos::scenario_key_prefix(ScenarioCode::S9); + + let user = manifest.user_context(deps); + let counters = user + .get_latest_component_revision(&manifest.counters_component_id) + .await?; + let promise = user + .get_latest_component_revision(&manifest.promise_component_id) + .await?; + + let ctx = WorkloadContext { + user, + counters, + promise, + history: history.clone(), + retry: config.retry_policy.clone(), + phase: PhaseMarker::new(Phase::Baseline), + key_prefix: key_prefix.clone(), + }; + + let scope = RunScope { + environment_id: manifest.environment_id.0.to_string(), + component_ids: vec![ + manifest.counters_component_id.0.to_string(), + manifest.promise_component_id.0.to_string(), + ], + agent_id_prefix: key_prefix.clone(), + idempotency_key_prefix: format!("{key_prefix}-"), + }; + + let mut phases = Phases::default(); + let mut routing_snapshots = Vec::new(); + let mut fault_injected_at = None; + let mut fault_recovered_at = None; + let mut fault_id = None; + let mut fault_target_observed = None; + let mut attention_extra: Vec = Vec::new(); + let mut rollback_report: Option = None; + + macro_rules! finish { + ($reason:expr, $records:expr, $readback:expr) => {{ + let mut summary = ChaosSummary::build( + $records, + $readback, + routing_snapshots.clone(), + fault_injected_at, + ); + summary.absorb(attention_extra.clone()); + if let Some(report) = rollback_report.clone() { + summary = summary.with_rollback(report); + } + let result = build_result( + config, + ScenarioOutcome { + started_at, + phases: phases.clone(), + fault_injected_at, + fault_recovered_at, + fault_id: fault_id.clone(), + fault_target_observed: fault_target_observed.clone(), + scope: scope.clone(), + summary, + termination_reason: $reason, + pinned_selection: None, + scheduled_selection: None, + promise_selection: None, + isolation_selection: None, + revert_selection: None, + delete_selection: None, + }, + ); + write_outputs(&result, &history, outputs)?; + return Ok(result); + }}; + } + + // ── Warm-up ───────────────────────────────────────────────────────────── + // + // Same reason as S1: an agent's first invocation costs far more than its + // later ones, and a population still cold-starting is a different thing to + // update than a population that is running. + routing_snapshots.push(snapshot_routing(deps, "before-warmup").await); + attention_extra.push(wait_for_settled_routing(deps, &mut routing_snapshots).await); + + info!("S9: warming up agents before the baseline"); + let warmed = warm_up(&ctx, workload_config).await; + info!("S9: warmed {warmed} agents, settling {:?}", WARMUP_SETTLE); + tokio::time::sleep(WARMUP_SETTLE).await; + + // ── Baseline ──────────────────────────────────────────────────────────── + info!( + "S9: baseline phase, running mixed workload for {:?}", + config.phases.baseline() + ); + phases.baseline = Some(PhaseWindow::started(Utc::now())); + let handle = workload::start(ctx.clone(), workload_config); + tokio::time::sleep(config.phases.baseline()).await; + routing_snapshots.push(snapshot_routing(deps, "before-fault").await); + if let Some(window) = phases.baseline.as_mut() { + window.end(Utc::now()); + } + + let baseline_operations = history.confirmed_in_phase(Phase::Baseline); + if baseline_operations == 0 { + warn!("S9: baseline produced no confirmed operations, aborting before injection"); + handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::PlatformUnreachable { + detail: "no operation succeeded during the baseline phase".to_string(), + }, + &records, + Vec::new() + ); + } + + // ── Roll forward ──────────────────────────────────────────────────────── + // + // The workload keeps running throughout. This leg is not the experiment: it + // exists to put the agents somewhere they can be brought back *from*. + info!("S9: rolling forward — updating the counters component to {COUNTERS_V2_WASM}"); + let forward = match ctx + .user + .update_component(&manifest.counters_component_id, COUNTERS_V2_WASM) + .await + { + Ok(updated) => updated, + Err(e) => { + warn!("S9: roll-forward component update failed: {e:#}"); + handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::Aborted { + detail: format!("component update to {COUNTERS_V2_WASM} failed: {e:#}"), + }, + &records, + Vec::new() + ); + } + }; + let forward_revision = forward.revision; + info!( + "S9: component now at revision {forward_revision}, moving {} durable agents onto it", + workload_config.durable_agents + ); + let _ = request_updates(&ctx, workload_config, forward_revision, 0, Duration::ZERO).await; + + // Let the forward leg land before measuring it. Without this the census + // below reads a population still in transit, and the gate would refuse a + // rollback that would have been perfectly good. + info!( + "S9: letting the roll-forward settle for {:?}", + rollback_config.settle() + ); + tokio::time::sleep(rollback_config.settle()).await; + + let rolled_forward = VersionCensus::build( + "before-rollback", + VERSION_ON_THE_NEW_BUILD, + &read_versions(&ctx, workload_config).await, + ); + info!( + "S9: {} of {} agents are on version {} ({} unreadable)", + rolled_forward.on_expected, + rolled_forward.agents, + VERSION_ON_THE_NEW_BUILD, + rolled_forward.unreadable + ); + + // ── Roll back ─────────────────────────────────────────────────────────── + // + // A redeploy of the original artifact as a new revision. Uploading it again + // rather than pointing agents back at the old revision is what "rollback" + // means operationally, and it keeps the evidence in the running code. + info!("S9: rolling back — re-uploading {COUNTERS_WASM} as a new revision"); + let back = match ctx + .user + .update_component(&manifest.counters_component_id, COUNTERS_WASM) + .await + { + Ok(updated) => updated, + Err(e) => { + warn!("S9: rollback component update failed: {e:#}"); + handle.stop().await; + let records = history.snapshot(); + finish!( + TerminationReason::Aborted { + detail: format!("rollback upload of {COUNTERS_WASM} failed: {e:#}"), + }, + &records, + Vec::new() + ); + } + }; + let rollback_revision = back.revision; + + let mut report = RollbackReport { + forward_revision: forward_revision.get(), + rollback_revision: rollback_revision.get(), + forward_version: VERSION_ON_THE_NEW_BUILD, + rollback_version: VERSION_AFTER_ROLLBACK, + rolled_forward, + rolled_back: None, + control: ControlPlaneAttempts::default(), + rolled_forward_floor_percent: rollback_config.rolled_forward_floor_percent, + }; + + // Refuse to spend the maintenance window rolling agents back to a build + // they never left. Every check would pass and the run would prove nothing. + if !report.forward_leg_landed() { + warn!("S9: the roll-forward did not land, refusing to roll back"); + handle.stop().await; + let records = history.snapshot(); + let detail = report + .attention_lines() + .first() + .cloned() + .unwrap_or_else(|| "the roll-forward did not land".to_string()); + rollback_report = Some(report); + finish!( + TerminationReason::FaultTargetUnverified { detail }, + &records, + Vec::new() + ); + } + + info!( + "S9: rollback revision is {rollback_revision}, asking {} durable agents to return", + workload_config.durable_agents + ); + let rollback_started_at = Utc::now(); + report.control = request_updates( + &ctx, + workload_config, + rollback_revision, + rollback_config.control_retries, + rollback_config.control_retry_delay(), + ) + .await; + attention_extra.push(Note::leveled( + report.control.refused > 0, + format!( + "rollback to revision {rollback_revision} accepted for {} of {} durable agents at \ + {rollback_started_at}", + report.control.accepted(), + workload_config.durable_agents + ), + )); + rollback_report = Some(report); + + // ── Signal: ready for the fault ───────────────────────────────────────── + tokio::time::sleep(rollback_config.kill_delay()).await; + info!( + "S9: {:?} into the rollback, signalling readiness for the kill", + rollback_config.kill_delay() + ); + signals.write_baseline_ready(&BaselineReady { + scenario_code: ScenarioCode::S9.as_str().to_string(), + ready_at: Utc::now(), + baseline_operations, + // Agents are mid-rollback across both executors, so killing either one + // interrupts a return in flight. Which one carries no information. + fault_target: None, + })?; + + // ── Fault ─────────────────────────────────────────────────────────────── + let injected = match signals.await_fault_injected(config.signal_timeout()).await { + Ok(injected) => injected, + Err(e) => { + warn!("S9: no fault-injected signal arrived: {e}"); + handle.stop().await; + let records = history.snapshot(); + finish!(signal_termination(&e), &records, Vec::new()); + } + }; + let into_update = (injected.injected_at - rollback_started_at).num_milliseconds(); + info!( + "S9: fault {} ({} on {}) reported active at {}, {into_update}ms into the rollback", + injected.fault_id, injected.kind, injected.target, injected.injected_at + ); + attention_extra.push(Note::context(format!( + "the executor kill landed {into_update}ms into the rollback" + ))); + fault_injected_at = Some(injected.injected_at); + fault_id = Some(injected.fault_id.clone()); + fault_target_observed = Some(injected.target.clone()); + ctx.phase.set(Phase::Fault); + phases.fault = Some(PhaseWindow::started(injected.injected_at)); + + let recovered = match signals.await_fault_recovered(config.signal_timeout()).await { + Ok(recovered) => recovered, + Err(e) => { + warn!("S9: no fault-recovered signal arrived: {e}"); + handle.stop().await; + let records = history.snapshot(); + finish!(signal_termination(&e), &records, Vec::new()); + } + }; + info!("S9: fault cleared at {}", recovered.recovered_at); + fault_recovered_at = Some(recovered.recovered_at); + if let Some(window) = phases.fault.as_mut() { + window.end(recovered.recovered_at); + } + + // ── Recovery ──────────────────────────────────────────────────────────── + info!( + "S9: recovery phase, running for a further {:?}", + config.phases.recovery() + ); + ctx.phase.set(Phase::Recovery); + phases.recovery = Some(PhaseWindow::started(Utc::now())); + tokio::time::sleep(config.phases.recovery()).await; + handle.stop().await; + if let Some(window) = phases.recovery.as_mut() { + window.end(Utc::now()); + } + routing_snapshots.push(snapshot_routing(deps, "after-recovery").await); + + // ── Read-back ─────────────────────────────────────────────────────────── + info!("S9: settling {SETTLE_BEFORE_READBACK:?} before read-back"); + tokio::time::sleep(SETTLE_BEFORE_READBACK).await; + + let records = history.snapshot(); + let readback = read_back(&ctx, &records, workload_config).await; + + let rolled_back = VersionCensus::build( + "after-recovery", + VERSION_AFTER_ROLLBACK, + &read_versions(&ctx, workload_config).await, + ); + let stale: Vec = read_versions(&ctx, workload_config) + .await + .into_iter() + .filter(|(_, v)| *v == Some(VERSION_ON_THE_NEW_BUILD)) + .map(|(agent, _)| agent) + .collect(); + attention_extra.push(Note::leveled( + rolled_back.on_expected < rolled_back.agents, + format!( + "after recovery {} of {} durable agents report component version {}; {} could not \ + be read", + rolled_back.on_expected, + rolled_back.agents, + VERSION_AFTER_ROLLBACK, + rolled_back.unreadable + ), + )); + if let Some(report) = rollback_report.as_mut() { + report.rolled_back = Some(rolled_back); + } + + // ── Verdict ───────────────────────────────────────────────────────────── + let reason = if let Some(bad) = readback.iter().find(|r| { + matches!( + r.verdict, + ReadbackVerdict::LostWork | ReadbackVerdict::DuplicateExecution + ) + }) { + TerminationReason::UpdateStateInconsistent { + agent: bad.agent.clone(), + detail: format!( + "{:?}: observed {:?} against an expected range of {}..={}", + bad.verdict, bad.observed, bad.expected_min, bad.expected_max + ), + } + } else if let Some(agent) = stale.first() { + // An agent still answering with the build it was rolled back *from* is + // a real failure. One that could not be read at all is not: an + // unreadable agent is reported above and says nothing either way. + // + // `UpdateNotApplied` rather than a parallel rollback variant, because + // it is the same fact — an update that did not land — and inventing a + // second name for it would only make the two harder to search for. + TerminationReason::UpdateNotApplied { + agent: agent.clone(), + observed: Some(VERSION_ON_THE_NEW_BUILD), + expected: VERSION_AFTER_ROLLBACK, + } + } else if let Some(stream) = stream_that_never_succeeded(&ChaosSummary::build( + &records, + readback.clone(), + routing_snapshots.clone(), + fault_injected_at, + )) { + TerminationReason::StreamNeverSucceeded { + stream: stream.to_string(), + } + } else { + TerminationReason::Completed + }; + + finish!(reason, &records, readback); +} + +/// Asks every durable agent to move to `target_revision`, concurrently. +/// +/// Returns how many requests the platform accepted. A refused request is +/// recorded and the run continues: the scenario is about what happens to the +/// updates that *did* start when the executor died. +/// Asks every durable agent to move to `target_revision`, retrying refusals. +/// +/// The retries here are the **control plane's**, counted apart from the +/// workload's. They answer a different question: a request refused because its +/// agent's executor just died says nothing about the platform's correctness, +/// but an agent nobody successfully asked to come back explains a stale agent +/// later without excusing one. Passing `0` retries makes this the same +/// fire-once call S5 does. +async fn request_updates( + ctx: &WorkloadContext, + config: &crate::chaos::WorkloadConfig, + target_revision: golem_common::model::component::ComponentRevision, + retries: u32, + delay: Duration, +) -> ControlPlaneAttempts { + let mut pending: Vec = (0..config.durable_agents) + .map(|index| ctx.agent_name(Stream::Durable, index)) + .collect(); + + let mut account = ControlPlaneAttempts { + requested: pending.len() as u64, + max_retries: retries, + ..Default::default() + }; + + for attempt in 0..=retries { + if pending.is_empty() { + break; + } + if attempt > 0 && !delay.is_zero() { + tokio::time::sleep(delay).await; + } + + let mut refused = Vec::new(); + for chunk in pending.chunks(UPDATE_CONCURRENCY) { + let mut batch = tokio::task::JoinSet::new(); + for agent in chunk.iter().cloned() { + let ctx = ctx.clone(); + batch.spawn(async move { + let id = workload::counter_agent_id(&ctx, &agent); + // `disable_wakeup: false` — the agent should be woken to + // process the update rather than waiting for its next + // invocation, because the kill is timed against the + // rollback starting, not against the next caller happening + // along. + ctx.user + .auto_update_worker(&id, target_revision, false) + .await + .map_err(|e| (agent, e)) + }); + } + while let Some(joined) = batch.join_next().await { + match joined { + Ok(Ok(())) => { + if attempt == 0 { + account.accepted_first_try += 1; + } else { + account.accepted_after_retry += 1; + } + } + Ok(Err((agent, e))) => { + warn!("S9: update request for {agent} refused: {e:#}"); + refused.push(agent); + } + Err(e) => warn!("S9: an update request task panicked: {e}"), + } + } + } + pending = refused; + } + + account.refused = pending.len() as u64; + account +} + +/// Asks every durable agent which build it is running. +/// +/// `None` means the agent could not be read, which is reported rather than +/// counted against the update. +async fn read_versions( + ctx: &WorkloadContext, + config: &crate::chaos::WorkloadConfig, +) -> std::collections::BTreeMap> { + let names: Vec = (0..config.durable_agents) + .map(|index| ctx.agent_name(Stream::Durable, index)) + .collect(); + + let mut out = std::collections::BTreeMap::new(); + for chunk in names.chunks(UPDATE_CONCURRENCY) { + let mut batch = tokio::task::JoinSet::new(); + for agent in chunk.iter().cloned() { + let ctx = ctx.clone(); + batch.spawn(async move { + let observed = workload::read_component_version(&ctx, &agent).await.ok(); + (agent, observed) + }); + } + while let Some(joined) = batch.join_next().await { + if let Ok((agent, observed)) = joined { + out.insert(agent, observed); + } + } + } + out +} + +/// Durable and scheduled state, compared against what the driver submitted. +async fn read_back( + ctx: &WorkloadContext, + records: &[OperationRecord], + config: &crate::chaos::WorkloadConfig, +) -> Vec { + let mut readback = Vec::new(); + + for index in 0..config.durable_agents { + let agent = ctx.agent_name(Stream::Durable, index); + let scoped = records + .iter() + .filter(|r| r.stream == Stream::Durable && r.agent == agent); + if scoped.clone().next().is_none() { + continue; + } + let observed = workload::read_counter(ctx, &agent).await; + readback.extend(readback_for(Stream::Durable, &agent, scoped, observed)); + } + + for index in 0..config.scheduled_agents { + let target = ctx.schedule_target_name(index); + let scoped = records + .iter() + .filter(|r| r.stream == Stream::Scheduled && r.agent == target); + if scoped.clone().next().is_none() { + continue; + } + let observed = workload::read_polls(ctx, &target).await; + readback.extend(readback_for(Stream::Scheduled, &target, scoped, observed)); + } + + readback +} diff --git a/integration-tests/src/chaos/summary.rs b/integration-tests/src/chaos/summary.rs index b1a0d6c751..e1a6a8021d 100644 --- a/integration-tests/src/chaos/summary.rs +++ b/integration-tests/src/chaos/summary.rs @@ -54,6 +54,7 @@ use crate::chaos::ownership::OwnershipSample; use crate::chaos::probe::KeyProbe; use crate::chaos::reachability::ReachabilityReport; use crate::chaos::resurrection::ResurrectionReport; +use crate::chaos::rollback::RollbackReport; use crate::chaos::truncation::TruncationReport; use crate::chaos::wakeups::WakeupReport; use serde::{Deserialize, Serialize}; @@ -588,6 +589,10 @@ pub struct ChaosSummary { /// scenarios that do not, for the same reason as `scheduleFires`. #[serde(default, skip_serializing_if = "Option::is_none")] pub resurrection: Option, + /// The rollback account, for scenarios that move agents between builds and + /// back. Absent for scenarios that do not. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rollback: Option, /// Shard-ownership samples, in the order they were taken. Empty for /// scenarios that do not sample executor assignments. /// @@ -722,6 +727,7 @@ impl ChaosSummary { reachability: None, truncation: None, resurrection: None, + rollback: None, ownership: Vec::new(), attention, notes: Vec::new(), @@ -822,6 +828,19 @@ impl ChaosSummary { self } + /// Attaches the rollback account and hoists everything it wants a human to + /// see into [`Self::attention`]. + /// + /// Same split as the others. The line worth calling out is the forward-leg + /// one: a rollback of agents that never left the old build proves nothing, + /// and that has to read as inconclusive rather than as a pass. + pub fn with_rollback(mut self, report: RollbackReport) -> Self { + self.attention.extend(report.attention_lines()); + self.notes.extend(report.note_lines()); + self.rollback = Some(report); + self + } + /// Attaches the shard-ownership samples and hoists their findings into /// [`Self::attention`]. /// From f7b7a599d885633c6a2cbe093ac61201be626d4b Mon Sep 17 00:00:00 2001 From: kmatasfp <33095685+kmatas@users.noreply.github.com> Date: Mon, 24 Aug 2026 21:13:58 -0700 Subject: [PATCH 20/40] Read the S9 forward leg from metadata so it does not abort the rollback it measures --- golem-worker-executor/tests/hot_update.rs | 117 ++++++++++++++++++++ golem-worker-executor/tests/lib.rs | 10 ++ integration-tests/src/chaos/scenarios/s9.rs | 88 ++++++++++++++- 3 files changed, 210 insertions(+), 5 deletions(-) diff --git a/golem-worker-executor/tests/hot_update.rs b/golem-worker-executor/tests/hot_update.rs index a096b557b8..feddbf16c8 100644 --- a/golem-worker-executor/tests/hot_update.rs +++ b/golem-worker-executor/tests/hot_update.rs @@ -46,6 +46,14 @@ inherit_test_dep!( #[tagged_as("agent_update_v2")] PrecompiledComponent ); +inherit_test_dep!( + #[tagged_as("agent_counters")] + PrecompiledComponent +); +inherit_test_dep!( + #[tagged_as("agent_counters_v2")] + PrecompiledComponent +); inherit_test_dep!(Tracing); pub struct F1Blocker { @@ -1245,3 +1253,112 @@ async fn agent_can_be_invoked_after_manual_snapshot_update_and_restart( Ok(()) } + +/// Can an automatic update move an agent *back* to an earlier build? +/// +/// Written to settle what the first S9 chaos run found on golem-dev: 200 of 200 +/// agents were accepted for an automatic update to a revision carrying the +/// original build, and none of them moved. +/// +/// The answer turns out to be about what an automatic update *is*. It replays +/// the agent's oplog against the new build and aborts if any recorded +/// invocation produces a different result — so it can only cross a build +/// boundary that no recorded invocation can tell apart. The first version of +/// this test failed on its *forward* leg for exactly that reason: it probed +/// with `Counter::component_version`, whose whole purpose is to differ between +/// builds, and thereby poisoned the update it was trying to observe. +/// +/// So the agent here does nothing but `increment`, which returns the same +/// values under both builds, and `component_version` is invoked only at the +/// very end. That is the fair question: with an oplog no replay can object to, +/// does an automatic update roll an agent back? +#[test] +#[tracing::instrument] +async fn an_automatic_update_rolls_an_agent_back_to_an_earlier_build( + last_unique_id: &LastUniqueId, + deps: &WorkerExecutorTestDependencies, + #[tagged_as("agent_counters")] agent_counters: &PrecompiledComponent, + #[tagged_as("agent_counters_v2")] _agent_counters_v2: &PrecompiledComponent, + _tracing: &Tracing, +) -> anyhow::Result<()> { + let context = TestContext::new(last_unique_id); + let executor = start(deps, &context).await?; + + let component = executor + .component_dep(&context.default_environment_id, agent_counters) + .store() + .await?; + let counter_id = agent_id!("Counter", "rollback-probe"); + let worker_id = executor + .start_agent(&component.id, counter_id.clone()) + .await?; + + // Behaviour-neutral history: `increment` returns the same values on both + // builds, so nothing here can make a replay disagree. + for _ in 0..3 { + executor + .invoke_and_await_agent(&component, &counter_id, "increment", data_value!()) + .await?; + } + + let report = async |label: &str| -> anyhow::Result<((usize, usize, usize), ComponentRevision)> { + let metadata = executor.get_worker_metadata(&worker_id).await?; + let counts = update_counts(&metadata); + info!( + "{label}: revision {}, (pending, successful, failed) = {counts:?}", + metadata.component_revision + ); + Ok((counts, metadata.component_revision)) + }; + + // ── Forward ───────────────────────────────────────────────────────────── + let forward = executor + .update_component(&component.id, "it_agent_counters_v2_release") + .await?; + executor + .auto_update_worker(&worker_id, forward.revision, false) + .await?; + executor + .invoke_and_await_agent(&component, &counter_id, "increment", data_value!()) + .await?; + let (forward_counts, forward_at) = report("after the forward update").await?; + assert_eq!( + forward_counts, + (0, 1, 0), + "the forward update must be recorded as successful" + ); + assert_eq!(forward_at, forward.revision); + + // ── Back ──────────────────────────────────────────────────────────────── + let back = executor + .update_component(&component.id, "it_agent_counters_release") + .await?; + assert_ne!(back.revision, forward.revision); + executor + .auto_update_worker(&worker_id, back.revision, false) + .await?; + executor + .invoke_and_await_agent(&component, &counter_id, "increment", data_value!()) + .await?; + let (back_counts, back_at) = report("after the rollback").await?; + + // Only now, when no further update depends on it, ask which code is running. + let running = executor + .invoke_and_await_agent(&component, &counter_id, "component_version", data_value!()) + .await?; + info!("running code after the rollback: {running:?}"); + + assert_eq!( + back_counts, + (0, 2, 0), + "the rollback must be recorded as a second successful update; it was at revision \ + {back_at}, and the running code reported {running:?}" + ); + assert_eq!( + running, + data_value!(1u32), + "after rolling back to a revision carrying the original build, the agent must run it" + ); + + Ok(()) +} diff --git a/golem-worker-executor/tests/lib.rs b/golem-worker-executor/tests/lib.rs index 36540218f4..c7ceae0bba 100644 --- a/golem-worker-executor/tests/lib.rs +++ b/golem-worker-executor/tests/lib.rs @@ -153,6 +153,16 @@ test_component!( "it_agent_counters_release", "it:agent-counters" ); +// The same component with `Counter::component_version` returning 2 instead of +// 1, and nothing else different. Declared under the same component name on +// purpose: an update can only swap one build for the other underneath running +// agents if the agent types match exactly. +test_component!( + agent_counters_v2, + "agent_counters_v2", + "it_agent_counters_v2_release", + "it:agent-counters" +); test_component!( http_tests, "http_tests", diff --git a/integration-tests/src/chaos/scenarios/s9.rs b/integration-tests/src/chaos/scenarios/s9.rs index 83b25642db..676fa66663 100644 --- a/integration-tests/src/chaos/scenarios/s9.rs +++ b/integration-tests/src/chaos/scenarios/s9.rs @@ -33,6 +33,29 @@ //! returned reports `1` from the code that is running, not from metadata about //! what the platform believes. //! +//! ## The constraint that shapes this scenario +//! +//! An automatic update **replays the agent's oplog against the new build** and +//! aborts if any recorded invocation produces a different result. It can only +//! cross a build boundary that no recorded invocation can tell apart. +//! +//! That has a sharp operational consequence, and it is arguably S9's most +//! useful finding: **a behaviour-changing build cannot be rolled back +//! automatically.** If the new build ever returned a different answer for an +//! invocation still in an agent's oplog — which is usually *why* you are +//! rolling back — the automatic update is refused. A rollback in that situation +//! needs a snapshot-based update instead. +//! +//! It also dictates how this scenario may look at its own agents. The first S9 +//! run verified the forward leg by invoking `component_version`, which exists +//! precisely to differ between builds, and thereby wrote an entry into all 200 +//! oplogs that the rollback's replay could never reproduce. Every rollback was +//! refused with `Unexpected oplog entry: expected component_version => 1, got +//! 2`. The forward leg is now read from metadata, which leaves no trace; the +//! running code is asked only at the very end, where nothing depends on it. +//! See `an_automatic_update_rolls_an_agent_back_to_an_earlier_build` in +//! `golem-worker-executor/tests/hot_update.rs` for the minimal reproduction. +//! //! ## Why the forward leg is verified first //! //! If the agents never reached the new build, rolling them back returns them to @@ -259,16 +282,32 @@ pub async fn run( ); tokio::time::sleep(rollback_config.settle()).await; + // Read from metadata, NOT by invoking `component_version`. + // + // This is the correction the first S9 run forced, and it is not a + // downgrade of the evidence — it is the only way to gather it without + // destroying what comes next. An automatic update replays the agent's + // oplog against the new build and aborts if any recorded invocation + // produces a different result. `component_version` exists precisely to + // differ between builds, so invoking it here writes an entry into every + // agent's oplog that the rollback's replay can never reproduce. The first + // run did exactly that and all 200 rollbacks were refused with + // "Unexpected oplog entry: expected component_version => 1, got 2". + // + // The forward leg only has to establish that the agents moved, so that the + // rollback has something to undo. Which revision the platform has them on + // answers that, and leaves no trace. The end state is still judged on the + // running code, at the very end, where nothing depends on it. let rolled_forward = VersionCensus::build( "before-rollback", - VERSION_ON_THE_NEW_BUILD, - &read_versions(&ctx, workload_config).await, + forward_revision.get() as u32, + &read_revisions(&ctx, workload_config).await, ); info!( - "S9: {} of {} agents are on version {} ({} unreadable)", + "S9: {} of {} agents are on revision {} ({} unreadable)", rolled_forward.on_expected, rolled_forward.agents, - VERSION_ON_THE_NEW_BUILD, + forward_revision, rolled_forward.unreadable ); @@ -302,7 +341,7 @@ pub async fn run( let mut report = RollbackReport { forward_revision: forward_revision.get(), rollback_revision: rollback_revision.get(), - forward_version: VERSION_ON_THE_NEW_BUILD, + forward_version: forward_revision.get() as u32, rollback_version: VERSION_AFTER_ROLLBACK, rolled_forward, rolled_back: None, @@ -577,6 +616,45 @@ async fn request_updates( account } +/// Asks the platform which component revision each durable agent is on. +/// +/// Deliberately metadata rather than an invocation. See the comment at the +/// forward-leg census: invoking a method whose result differs between builds +/// writes an oplog entry that the next automatic update's replay cannot +/// reproduce, which aborts that update. Reading metadata leaves no trace. +async fn read_revisions( + ctx: &WorkloadContext, + config: &crate::chaos::WorkloadConfig, +) -> std::collections::BTreeMap> { + let names: Vec = (0..config.durable_agents) + .map(|index| ctx.agent_name(Stream::Durable, index)) + .collect(); + + let mut out = std::collections::BTreeMap::new(); + for chunk in names.chunks(UPDATE_CONCURRENCY) { + let mut batch = tokio::task::JoinSet::new(); + for agent in chunk.iter().cloned() { + let ctx = ctx.clone(); + batch.spawn(async move { + let id = workload::counter_agent_id(&ctx, &agent); + let observed = ctx + .user + .get_worker_metadata(&id) + .await + .ok() + .map(|m| m.component_revision.get() as u32); + (agent, observed) + }); + } + while let Some(joined) = batch.join_next().await { + if let Ok((agent, observed)) = joined { + out.insert(agent, observed); + } + } + } + out +} + /// Asks every durable agent which build it is running. /// /// `None` means the agent could not be read, which is reported rather than From 0ed1fb6eed0619a62d2f7982a04671759ca0d664 Mon Sep 17 00:00:00 2001 From: kmatasfp <33095685+kmatas@users.noreply.github.com> Date: Mon, 24 Aug 2026 21:20:34 -0700 Subject: [PATCH 21/40] Add local reproductions for automatic and snapshot rollback across builds --- golem-worker-executor/tests/hot_update.rs | 173 ++++++++++++++++++++++ 1 file changed, 173 insertions(+) diff --git a/golem-worker-executor/tests/hot_update.rs b/golem-worker-executor/tests/hot_update.rs index feddbf16c8..a76a3638c5 100644 --- a/golem-worker-executor/tests/hot_update.rs +++ b/golem-worker-executor/tests/hot_update.rs @@ -1362,3 +1362,176 @@ async fn an_automatic_update_rolls_an_agent_back_to_an_earlier_build( Ok(()) } + +/// Does a **snapshot-based** update roll an agent back where an automatic one +/// cannot? +/// +/// The companion to the test above, and the reason it matters. An automatic +/// update replays the oplog and so can only cross a build boundary no recorded +/// invocation can tell apart — which excludes exactly the case you roll back +/// for, a build that behaved differently. A snapshot update does not replay: it +/// saves the agent's state, loads it into the target build, and resumes. +/// +/// Observed through metadata rather than a discriminating invocation, because +/// `SnapshotCounter` is byte-identical between the two builds and has nothing +/// to discriminate with. That is the gap a real S9 snapshot leg would have to +/// close. +// Reproduces a platform bug rather than asserting current behaviour: the +// SECOND snapshot-based update on an agent fails with `snapshot-based pending +// update expected replay state to already be live`, whichever direction it +// goes. See `two_consecutive_snapshot_updates_in_the_same_direction` below, +// which isolates it from rollback entirely. Un-ignore once that is fixed. +#[test] +#[ignore] +#[tracing::instrument] +async fn a_snapshot_update_rolls_an_agent_back_to_an_earlier_build( + last_unique_id: &LastUniqueId, + deps: &WorkerExecutorTestDependencies, + #[tagged_as("agent_counters")] agent_counters: &PrecompiledComponent, + #[tagged_as("agent_counters_v2")] _agent_counters_v2: &PrecompiledComponent, + _tracing: &Tracing, +) -> anyhow::Result<()> { + let context = TestContext::new(last_unique_id); + let executor = start(deps, &context).await?; + + let component = executor + .component_dep(&context.default_environment_id, agent_counters) + .store() + .await?; + let counter_id = agent_id!("SnapshotCounter", "snapshot-rollback-probe"); + let worker_id = executor + .start_agent(&component.id, counter_id.clone()) + .await?; + + for _ in 0..3 { + executor + .invoke_and_await_agent(&component, &counter_id, "increment", data_value!()) + .await?; + } + + let report = async |label: &str| -> anyhow::Result<((usize, usize, usize), ComponentRevision)> { + let metadata = executor.get_worker_metadata(&worker_id).await?; + let counts = update_counts(&metadata); + info!( + "{label}: revision {}, (pending, successful, failed) = {counts:?}", + metadata.component_revision + ); + Ok((counts, metadata.component_revision)) + }; + + let forward = executor + .update_component(&component.id, "it_agent_counters_v2_release") + .await?; + executor + .manual_update_worker(&worker_id, forward.revision, false) + .await?; + let value_after_forward = executor + .invoke_and_await_agent(&component, &counter_id, "get", data_value!()) + .await?; + let (forward_counts, _) = report("after the forward snapshot update").await?; + assert_eq!(forward_counts, (0, 1, 0)); + assert_eq!( + value_after_forward, + data_value!(3u32), + "the snapshot must carry the agent's state across the build change" + ); + + let back = executor + .update_component(&component.id, "it_agent_counters_release") + .await?; + executor + .manual_update_worker(&worker_id, back.revision, false) + .await?; + let value_after_rollback = executor + .invoke_and_await_agent(&component, &counter_id, "get", data_value!()) + .await?; + let (back_counts, back_at) = report("after the snapshot rollback").await?; + + assert_eq!( + back_counts, + (0, 2, 0), + "a snapshot rollback must be recorded as a second successful update; it was at \ + revision {back_at}" + ); + assert_eq!(back_at, back.revision); + assert_eq!( + value_after_rollback, + data_value!(3u32), + "the agent's state must survive the rollback" + ); + + Ok(()) +} + +/// Control for the test above: are **two consecutive snapshot updates** the +/// problem, or is it specifically going *backwards*? +/// +/// Both updates here move forward, to two revisions carrying the same build, so +/// nothing about direction or behaviour differs. If this fails the same way, +/// the error belongs to consecutive snapshot updates and has nothing to do with +/// rollback. +// Reproduces the same platform bug, with direction ruled out: both updates +// here move forward to revisions carrying the same build, and the second one +// still leaves the agent unable to start. Un-ignore once it is fixed. +#[test] +#[ignore] +#[tracing::instrument] +async fn two_consecutive_snapshot_updates_in_the_same_direction( + last_unique_id: &LastUniqueId, + deps: &WorkerExecutorTestDependencies, + #[tagged_as("agent_counters")] agent_counters: &PrecompiledComponent, + #[tagged_as("agent_counters_v2")] _agent_counters_v2: &PrecompiledComponent, + _tracing: &Tracing, +) -> anyhow::Result<()> { + let context = TestContext::new(last_unique_id); + let executor = start(deps, &context).await?; + + let component = executor + .component_dep(&context.default_environment_id, agent_counters) + .store() + .await?; + let counter_id = agent_id!("SnapshotCounter", "twice-forward-probe"); + let worker_id = executor + .start_agent(&component.id, counter_id.clone()) + .await?; + + for _ in 0..3 { + executor + .invoke_and_await_agent(&component, &counter_id, "increment", data_value!()) + .await?; + } + + let first = executor + .update_component(&component.id, "it_agent_counters_v2_release") + .await?; + executor + .manual_update_worker(&worker_id, first.revision, false) + .await?; + let after_first = executor + .invoke_and_await_agent(&component, &counter_id, "get", data_value!()) + .await?; + assert_eq!(after_first, data_value!(3u32)); + info!( + "after the first snapshot update: {:?}", + update_counts(&executor.get_worker_metadata(&worker_id).await?) + ); + + // Same build again, so this is unambiguously a forward step. + let second = executor + .update_component(&component.id, "it_agent_counters_v2_release") + .await?; + assert_ne!(second.revision, first.revision); + executor + .manual_update_worker(&worker_id, second.revision, false) + .await?; + let after_second = executor + .invoke_and_await_agent(&component, &counter_id, "get", data_value!()) + .await?; + let counts = update_counts(&executor.get_worker_metadata(&worker_id).await?); + info!("after the second snapshot update: {counts:?}"); + + assert_eq!(after_second, data_value!(3u32)); + assert_eq!(counts, (0, 2, 0), "both snapshot updates must succeed"); + + Ok(()) +} From fd222286845613f332fc40781c291201f2736628 Mon Sep 17 00:00:00 2001 From: kmatasfp <33095685+kmatas@users.noreply.github.com> Date: Mon, 24 Aug 2026 21:40:51 -0700 Subject: [PATCH 22/40] Revert "Add local reproductions for automatic and snapshot rollback across builds" This reverts commit 0ed1fb6eed0619a62d2f7982a04671759ca0d664. --- golem-worker-executor/tests/hot_update.rs | 173 ---------------------- 1 file changed, 173 deletions(-) diff --git a/golem-worker-executor/tests/hot_update.rs b/golem-worker-executor/tests/hot_update.rs index a76a3638c5..feddbf16c8 100644 --- a/golem-worker-executor/tests/hot_update.rs +++ b/golem-worker-executor/tests/hot_update.rs @@ -1362,176 +1362,3 @@ async fn an_automatic_update_rolls_an_agent_back_to_an_earlier_build( Ok(()) } - -/// Does a **snapshot-based** update roll an agent back where an automatic one -/// cannot? -/// -/// The companion to the test above, and the reason it matters. An automatic -/// update replays the oplog and so can only cross a build boundary no recorded -/// invocation can tell apart — which excludes exactly the case you roll back -/// for, a build that behaved differently. A snapshot update does not replay: it -/// saves the agent's state, loads it into the target build, and resumes. -/// -/// Observed through metadata rather than a discriminating invocation, because -/// `SnapshotCounter` is byte-identical between the two builds and has nothing -/// to discriminate with. That is the gap a real S9 snapshot leg would have to -/// close. -// Reproduces a platform bug rather than asserting current behaviour: the -// SECOND snapshot-based update on an agent fails with `snapshot-based pending -// update expected replay state to already be live`, whichever direction it -// goes. See `two_consecutive_snapshot_updates_in_the_same_direction` below, -// which isolates it from rollback entirely. Un-ignore once that is fixed. -#[test] -#[ignore] -#[tracing::instrument] -async fn a_snapshot_update_rolls_an_agent_back_to_an_earlier_build( - last_unique_id: &LastUniqueId, - deps: &WorkerExecutorTestDependencies, - #[tagged_as("agent_counters")] agent_counters: &PrecompiledComponent, - #[tagged_as("agent_counters_v2")] _agent_counters_v2: &PrecompiledComponent, - _tracing: &Tracing, -) -> anyhow::Result<()> { - let context = TestContext::new(last_unique_id); - let executor = start(deps, &context).await?; - - let component = executor - .component_dep(&context.default_environment_id, agent_counters) - .store() - .await?; - let counter_id = agent_id!("SnapshotCounter", "snapshot-rollback-probe"); - let worker_id = executor - .start_agent(&component.id, counter_id.clone()) - .await?; - - for _ in 0..3 { - executor - .invoke_and_await_agent(&component, &counter_id, "increment", data_value!()) - .await?; - } - - let report = async |label: &str| -> anyhow::Result<((usize, usize, usize), ComponentRevision)> { - let metadata = executor.get_worker_metadata(&worker_id).await?; - let counts = update_counts(&metadata); - info!( - "{label}: revision {}, (pending, successful, failed) = {counts:?}", - metadata.component_revision - ); - Ok((counts, metadata.component_revision)) - }; - - let forward = executor - .update_component(&component.id, "it_agent_counters_v2_release") - .await?; - executor - .manual_update_worker(&worker_id, forward.revision, false) - .await?; - let value_after_forward = executor - .invoke_and_await_agent(&component, &counter_id, "get", data_value!()) - .await?; - let (forward_counts, _) = report("after the forward snapshot update").await?; - assert_eq!(forward_counts, (0, 1, 0)); - assert_eq!( - value_after_forward, - data_value!(3u32), - "the snapshot must carry the agent's state across the build change" - ); - - let back = executor - .update_component(&component.id, "it_agent_counters_release") - .await?; - executor - .manual_update_worker(&worker_id, back.revision, false) - .await?; - let value_after_rollback = executor - .invoke_and_await_agent(&component, &counter_id, "get", data_value!()) - .await?; - let (back_counts, back_at) = report("after the snapshot rollback").await?; - - assert_eq!( - back_counts, - (0, 2, 0), - "a snapshot rollback must be recorded as a second successful update; it was at \ - revision {back_at}" - ); - assert_eq!(back_at, back.revision); - assert_eq!( - value_after_rollback, - data_value!(3u32), - "the agent's state must survive the rollback" - ); - - Ok(()) -} - -/// Control for the test above: are **two consecutive snapshot updates** the -/// problem, or is it specifically going *backwards*? -/// -/// Both updates here move forward, to two revisions carrying the same build, so -/// nothing about direction or behaviour differs. If this fails the same way, -/// the error belongs to consecutive snapshot updates and has nothing to do with -/// rollback. -// Reproduces the same platform bug, with direction ruled out: both updates -// here move forward to revisions carrying the same build, and the second one -// still leaves the agent unable to start. Un-ignore once it is fixed. -#[test] -#[ignore] -#[tracing::instrument] -async fn two_consecutive_snapshot_updates_in_the_same_direction( - last_unique_id: &LastUniqueId, - deps: &WorkerExecutorTestDependencies, - #[tagged_as("agent_counters")] agent_counters: &PrecompiledComponent, - #[tagged_as("agent_counters_v2")] _agent_counters_v2: &PrecompiledComponent, - _tracing: &Tracing, -) -> anyhow::Result<()> { - let context = TestContext::new(last_unique_id); - let executor = start(deps, &context).await?; - - let component = executor - .component_dep(&context.default_environment_id, agent_counters) - .store() - .await?; - let counter_id = agent_id!("SnapshotCounter", "twice-forward-probe"); - let worker_id = executor - .start_agent(&component.id, counter_id.clone()) - .await?; - - for _ in 0..3 { - executor - .invoke_and_await_agent(&component, &counter_id, "increment", data_value!()) - .await?; - } - - let first = executor - .update_component(&component.id, "it_agent_counters_v2_release") - .await?; - executor - .manual_update_worker(&worker_id, first.revision, false) - .await?; - let after_first = executor - .invoke_and_await_agent(&component, &counter_id, "get", data_value!()) - .await?; - assert_eq!(after_first, data_value!(3u32)); - info!( - "after the first snapshot update: {:?}", - update_counts(&executor.get_worker_metadata(&worker_id).await?) - ); - - // Same build again, so this is unambiguously a forward step. - let second = executor - .update_component(&component.id, "it_agent_counters_v2_release") - .await?; - assert_ne!(second.revision, first.revision); - executor - .manual_update_worker(&worker_id, second.revision, false) - .await?; - let after_second = executor - .invoke_and_await_agent(&component, &counter_id, "get", data_value!()) - .await?; - let counts = update_counts(&executor.get_worker_metadata(&worker_id).await?); - info!("after the second snapshot update: {counts:?}"); - - assert_eq!(after_second, data_value!(3u32)); - assert_eq!(counts, (0, 2, 0), "both snapshot updates must succeed"); - - Ok(()) -} From b7b9cc7a5ae15f3aaa51dcd59ef6e43a1c35604f Mon Sep 17 00:00:00 2001 From: kmatasfp <33095685+kmatas@users.noreply.github.com> Date: Mon, 24 Aug 2026 21:41:17 -0700 Subject: [PATCH 23/40] Move the snapshot-update reproductions onto their own branch --- golem-worker-executor/tests/hot_update.rs | 117 ---------------------- golem-worker-executor/tests/lib.rs | 10 -- 2 files changed, 127 deletions(-) diff --git a/golem-worker-executor/tests/hot_update.rs b/golem-worker-executor/tests/hot_update.rs index feddbf16c8..a096b557b8 100644 --- a/golem-worker-executor/tests/hot_update.rs +++ b/golem-worker-executor/tests/hot_update.rs @@ -46,14 +46,6 @@ inherit_test_dep!( #[tagged_as("agent_update_v2")] PrecompiledComponent ); -inherit_test_dep!( - #[tagged_as("agent_counters")] - PrecompiledComponent -); -inherit_test_dep!( - #[tagged_as("agent_counters_v2")] - PrecompiledComponent -); inherit_test_dep!(Tracing); pub struct F1Blocker { @@ -1253,112 +1245,3 @@ async fn agent_can_be_invoked_after_manual_snapshot_update_and_restart( Ok(()) } - -/// Can an automatic update move an agent *back* to an earlier build? -/// -/// Written to settle what the first S9 chaos run found on golem-dev: 200 of 200 -/// agents were accepted for an automatic update to a revision carrying the -/// original build, and none of them moved. -/// -/// The answer turns out to be about what an automatic update *is*. It replays -/// the agent's oplog against the new build and aborts if any recorded -/// invocation produces a different result — so it can only cross a build -/// boundary that no recorded invocation can tell apart. The first version of -/// this test failed on its *forward* leg for exactly that reason: it probed -/// with `Counter::component_version`, whose whole purpose is to differ between -/// builds, and thereby poisoned the update it was trying to observe. -/// -/// So the agent here does nothing but `increment`, which returns the same -/// values under both builds, and `component_version` is invoked only at the -/// very end. That is the fair question: with an oplog no replay can object to, -/// does an automatic update roll an agent back? -#[test] -#[tracing::instrument] -async fn an_automatic_update_rolls_an_agent_back_to_an_earlier_build( - last_unique_id: &LastUniqueId, - deps: &WorkerExecutorTestDependencies, - #[tagged_as("agent_counters")] agent_counters: &PrecompiledComponent, - #[tagged_as("agent_counters_v2")] _agent_counters_v2: &PrecompiledComponent, - _tracing: &Tracing, -) -> anyhow::Result<()> { - let context = TestContext::new(last_unique_id); - let executor = start(deps, &context).await?; - - let component = executor - .component_dep(&context.default_environment_id, agent_counters) - .store() - .await?; - let counter_id = agent_id!("Counter", "rollback-probe"); - let worker_id = executor - .start_agent(&component.id, counter_id.clone()) - .await?; - - // Behaviour-neutral history: `increment` returns the same values on both - // builds, so nothing here can make a replay disagree. - for _ in 0..3 { - executor - .invoke_and_await_agent(&component, &counter_id, "increment", data_value!()) - .await?; - } - - let report = async |label: &str| -> anyhow::Result<((usize, usize, usize), ComponentRevision)> { - let metadata = executor.get_worker_metadata(&worker_id).await?; - let counts = update_counts(&metadata); - info!( - "{label}: revision {}, (pending, successful, failed) = {counts:?}", - metadata.component_revision - ); - Ok((counts, metadata.component_revision)) - }; - - // ── Forward ───────────────────────────────────────────────────────────── - let forward = executor - .update_component(&component.id, "it_agent_counters_v2_release") - .await?; - executor - .auto_update_worker(&worker_id, forward.revision, false) - .await?; - executor - .invoke_and_await_agent(&component, &counter_id, "increment", data_value!()) - .await?; - let (forward_counts, forward_at) = report("after the forward update").await?; - assert_eq!( - forward_counts, - (0, 1, 0), - "the forward update must be recorded as successful" - ); - assert_eq!(forward_at, forward.revision); - - // ── Back ──────────────────────────────────────────────────────────────── - let back = executor - .update_component(&component.id, "it_agent_counters_release") - .await?; - assert_ne!(back.revision, forward.revision); - executor - .auto_update_worker(&worker_id, back.revision, false) - .await?; - executor - .invoke_and_await_agent(&component, &counter_id, "increment", data_value!()) - .await?; - let (back_counts, back_at) = report("after the rollback").await?; - - // Only now, when no further update depends on it, ask which code is running. - let running = executor - .invoke_and_await_agent(&component, &counter_id, "component_version", data_value!()) - .await?; - info!("running code after the rollback: {running:?}"); - - assert_eq!( - back_counts, - (0, 2, 0), - "the rollback must be recorded as a second successful update; it was at revision \ - {back_at}, and the running code reported {running:?}" - ); - assert_eq!( - running, - data_value!(1u32), - "after rolling back to a revision carrying the original build, the agent must run it" - ); - - Ok(()) -} diff --git a/golem-worker-executor/tests/lib.rs b/golem-worker-executor/tests/lib.rs index c7ceae0bba..36540218f4 100644 --- a/golem-worker-executor/tests/lib.rs +++ b/golem-worker-executor/tests/lib.rs @@ -153,16 +153,6 @@ test_component!( "it_agent_counters_release", "it:agent-counters" ); -// The same component with `Counter::component_version` returning 2 instead of -// 1, and nothing else different. Declared under the same component name on -// purpose: an update can only swap one build for the other underneath running -// agents if the agent types match exactly. -test_component!( - agent_counters_v2, - "agent_counters_v2", - "it_agent_counters_v2_release", - "it:agent-counters" -); test_component!( http_tests, "http_tests", From a70a699a505c51100535c787d800f9177d47c7f4 Mon Sep 17 00:00:00 2001 From: Kaur Matas <33095685+kmatasfp@users.noreply.github.com> Date: Tue, 25 Aug 2026 12:33:12 -0700 Subject: [PATCH 24/40] Add S16 chaos scenario driver and storage outage account --- golem-test-framework/src/benchmark/config.rs | 2 + .../chaos_suites/cloud-chaos.yaml | 120 +++ integration-tests/src/benchmarks/all.rs | 4 + integration-tests/src/chaos/history.rs | 23 + integration-tests/src/chaos/mod.rs | 224 +++- integration-tests/src/chaos/outage.rs | 997 ++++++++++++++++++ integration-tests/src/chaos/reachability.rs | 87 +- integration-tests/src/chaos/result.rs | 9 +- integration-tests/src/chaos/scenarios/mod.rs | 2 + integration-tests/src/chaos/scenarios/s16.rs | 790 ++++++++++++++ integration-tests/src/chaos/split.rs | 62 ++ integration-tests/src/chaos/summary.rs | 21 + 12 files changed, 2257 insertions(+), 84 deletions(-) create mode 100644 integration-tests/src/chaos/outage.rs create mode 100644 integration-tests/src/chaos/scenarios/s16.rs diff --git a/golem-test-framework/src/benchmark/config.rs b/golem-test-framework/src/benchmark/config.rs index a0a90c0f02..239769e961 100644 --- a/golem-test-framework/src/benchmark/config.rs +++ b/golem-test-framework/src/benchmark/config.rs @@ -254,6 +254,8 @@ pub enum ChaosScenarioArg { S6, /// Executor pod kill while a component rollback is in flight. S9, + /// Executors cut off from the key-value PostgreSQL cluster. + S16, } /// Density subcommand action. diff --git a/integration-tests/chaos_suites/cloud-chaos.yaml b/integration-tests/chaos_suites/cloud-chaos.yaml index eec3558738..a2471ef439 100644 --- a/integration-tests/chaos_suites/cloud-chaos.yaml +++ b/integration-tests/chaos_suites/cloud-chaos.yaml @@ -853,3 +853,123 @@ scenarios: delaySecs: 5 signalTimeoutSecs: 1800 + + # S16 — key-value PostgreSQL outage recovery (GOL-379). + # + # The first fault in the suite aimed at something the platform depends on + # rather than at the platform itself. The executors keep running, keep their + # shards and keep answering the shard-manager. What they lose is the Aurora + # cluster underneath them, which on golem-dev carries promises, the + # running-workers set, user key-value data and the scheduler's own schema. + # + # The oplog is on a different cluster and the worker-status hot cache is in + # Redis, and neither is touched. That is the interesting part: the platform + # keeps the ability to record what it did and loses the ability to know what + # it is doing. + # + # There is no control group of executors, because all three share the cluster. + # The control is the baseline window, and the first thing the report has to + # establish is that throughput collapsed at all — a partition that silently + # failed to take hold would otherwise produce a clean-looking account of an + # undisturbed cluster. + - code: S16 + name: keyvalue-postgres-outage + enabled: true + + fault: + kind: network-partition + target: worker-executor + # Every executor, unlike every other partition in this suite. A storage + # outage that only some executors could see would be a routing fault + # wearing a database's clothes, and S3 already covers that shape. + mode: all + durationSecs: 120 + + phases: + # Long enough for cold starts, component loading and route warm-up to + # settle, and for the scheduler to be firing steadily, so the cut lands on + # a cluster in steady state. + baselineSecs: 300 + # Has to comfortably exceed the caller's 120s attempt timeout, for the + # same reason S3's does: an operation that never reaches its timeout + # inside the window leaves the stall invisible in the fault-window cells. + # It also has to be long enough for the executors' connection pools to + # give up and start retrying rather than merely block. + faultSecs: 180 + # Every stream has to get several whole operations after the heal, so a + # stream that recovered slowly is distinguishable from one that never + # did. Scheduled actions registered during the outage also have to fall + # due and fire inside the run rather than after it. + recoverySecs: 420 + + workload: + # The population read back agent by agent, so a lost increment localises + # to one agent out of 200. + durableAgents: 200 + # No durable state of their own, but they still need the running-workers + # set to start, so they say whether acceptance degraded separately from + # whether state survived. + ephemeralAgents: 50 + # Zero on purpose. The scheduled stream is driven by the `scheduled` block + # below instead, because only its registrations carry a token into the + # target's fire log and scheduler lag cannot be measured without one. + # Setting both is refused at load time. + scheduledAgents: 0 + # Promises live in the cluster being taken away, so this stream is + # directly on the fault path rather than incidental to it. + promiseAgents: 50 + # No quota stream: a lease renewal failing would add a shard-manager + # failure mode to a scenario that is about storage. + quotaAgents: 0 + ratePerSec: 100 + + scheduled: + # 100 targets, each with its own emitter, same as S10. Also the resolution + # of the report: a lost action localises to one target out of a hundred. + targets: 100 + intervalMillis: 2000 + # 500 actions accepted and not yet run at any instant, so the outage + # begins with a large population of work the platform has promised to do + # and cannot currently read. + leadSecs: 10 + # What recovering a pending action may cost. Larger than S10's 60s and + # deliberately so: there is no shard reassignment here, but the scheduler + # cannot claim or acknowledge anything for the whole 180s outage, so the + # floor is the outage itself. 240s covers the outage plus a lease TTL and + # some catch-up without being so generous that a regression hides inside + # it. + # + # Recorded, not asserted, like every other budget in the suite. + leaseBudgetSecs: 240 + + storage: + # The Aurora writer endpoint the workflow names in the NetworkChaos + # manifest's `externalTargets`. Recorded here so the archived result says + # which storage the run was about; the driver never resolves or contacts + # it. + endpoint: golem-postgres-dev-keyvalue.cluster-cgfyoqmjq7tc.us-east-1.rds.amazonaws.com + # The most of its baseline throughput the workload may keep during the + # outage for the fault to count as observed. Not zero: an operation + # submitted late in the fault window can still be waiting when the + # database returns, and then confirms into the window it was submitted in. + # At 100 ops/s over 180s that is a handful against a baseline of + # thousands, so anything near 15% means the partition did not take hold. + outageCeilingPercent: 15 + # What serving again may cost once the cluster is reachable. Recorded, not + # asserted: this is really a measure of how long sqlx's pool takes to + # notice, plus whatever recovery scan the executors run, and how long that + # may be is a judgement. + recoveryBudgetSecs: 120 + + retryPolicy: + # Identical to the others and load-bearing for the same reason, with one + # consequence specific to S16: the retry is what turns a single stalled + # attempt into a second one under the same key. If the database comes back + # in between, that retry lands against work the first attempt may already + # have half-done, which is exactly what the exactly-once probe is here to + # rule on. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 diff --git a/integration-tests/src/benchmarks/all.rs b/integration-tests/src/benchmarks/all.rs index 7a9b9876f7..4300462b53 100644 --- a/integration-tests/src/benchmarks/all.rs +++ b/integration-tests/src/benchmarks/all.rs @@ -598,6 +598,7 @@ async fn run_chaos( ChaosScenarioArg::S7 => chaos::ScenarioCode::S7, ChaosScenarioArg::S6 => chaos::ScenarioCode::S6, ChaosScenarioArg::S9 => chaos::ScenarioCode::S9, + ChaosScenarioArg::S16 => chaos::ScenarioCode::S16, }; let config = suite .scenario(code, allow_disabled) @@ -647,6 +648,9 @@ async fn run_chaos( chaos::ScenarioCode::S9 => { chaos::scenarios::s9::run(&config, &manifest, &deps, &signals, &outputs).await } + chaos::ScenarioCode::S16 => { + chaos::scenarios::s16::run(&config, &manifest, &deps, &signals, &outputs).await + } }; deps.kill_all().await; diff --git a/integration-tests/src/chaos/history.rs b/integration-tests/src/chaos/history.rs index fc1d380cf9..f225da4a7a 100644 --- a/integration-tests/src/chaos/history.rs +++ b/integration-tests/src/chaos/history.rs @@ -349,6 +349,29 @@ impl OperationRecord { pub fn had_successful_attempt(&self) -> bool { self.outcome == Outcome::Confirmed || self.attempt_log.iter().any(|a| a.succeeded) } + + /// Attempts at this operation that hit the client's attempt timeout rather + /// than answering. + /// + /// Matched on the message [`crate::chaos::workload`] writes for a timed-out + /// attempt. A structured flag would be better, but the attempt log is an + /// archived shape that older results already carry, and this reads it + /// without changing it. + /// + /// The distinction it draws is the one a stalled dependency needs: an + /// operation that timed out and then answered on its retry was rescued by + /// the caller, not returned by the platform, and the outcome alone cannot + /// say so. + pub fn attempts_timed_out(&self) -> u64 { + self.attempt_log + .iter() + .filter(|a| { + a.error + .as_deref() + .is_some_and(|e| e.contains("attempt timed out")) + }) + .count() as u64 + } } /// What a scheduled action recorded when it ran, and the log it was recorded diff --git a/integration-tests/src/chaos/mod.rs b/integration-tests/src/chaos/mod.rs index a0d11e55a1..74050a4f91 100644 --- a/integration-tests/src/chaos/mod.rs +++ b/integration-tests/src/chaos/mod.rs @@ -38,6 +38,7 @@ pub mod deletions; pub mod errors; pub mod fires; pub mod history; +pub mod outage; pub mod ownership; pub mod pinned; pub mod prep; @@ -89,6 +90,8 @@ pub enum ScenarioCode { S6, /// Executor pod kill while a component rollback is in flight. S9, + /// Executors cut off from the key-value PostgreSQL cluster. + S16, } impl ScenarioCode { @@ -105,13 +108,14 @@ impl ScenarioCode { ScenarioCode::S7 => "S7", ScenarioCode::S6 => "S6", ScenarioCode::S9 => "S9", + ScenarioCode::S16 => "S16", } } /// Every scenario this driver implements. The suite YAML is checked against /// this list, so a scenario cannot be enabled in YAML without code behind /// it, nor implemented without an operational switch in front of it. - pub const ALL: [ScenarioCode; 11] = [ + pub const ALL: [ScenarioCode; 12] = [ ScenarioCode::S1, ScenarioCode::S3, ScenarioCode::S5, @@ -123,6 +127,7 @@ impl ScenarioCode { ScenarioCode::S11, ScenarioCode::S12, ScenarioCode::S13, + ScenarioCode::S16, ]; pub fn parse(s: &str) -> Option { @@ -646,6 +651,54 @@ impl RollbackConfig { } } +/// Settings for a storage-outage scenario (GOL-379). +/// +/// Not a workload shape, unlike the blocks above it: S16 drives the mixed +/// workload and the scheduled-registration workload that already exist, because +/// the fault is not about which agents are driven but about which *dependency* +/// is taken away from all of them. What this block carries is the thing being +/// taken away and the two numbers the account is judged by. +/// +/// The endpoint is recorded rather than acted on, like every other entry in +/// [`FaultConfig`]. The driver never resolves it, connects to it or cuts +/// anything off from it; the workflow does that, and this exists so an archived +/// result says which storage the run was about rather than leaving a reader to +/// infer it from the scenario name. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StorageConfig { + /// The storage endpoint the workflow partitions the executors from, as a + /// hostname. Chaos Mesh resolves it in the controller, so this is the same + /// string that appears in the NetworkChaos manifest's `externalTargets`. + pub endpoint: String, + /// The most of its baseline throughput the workload may keep during the + /// fault, as a percentage, for the outage to count as observed. + /// + /// A run above this line did not take the storage away, whatever the fault + /// status said, and every other number in the report then describes an + /// undisturbed cluster. That is reported as inconclusive rather than clean: + /// a healthy-looking result from a fault that never landed is the worst + /// artifact this suite can produce. + /// + /// It cannot be zero. Operations submitted in the last seconds before the + /// heal confirm just after it and are counted in the fault window they were + /// submitted in, so a genuine outage still leaves a small non-zero cell. + /// Read it together with `quietMs` — see [`crate::chaos::outage`]. + pub outage_ceiling_percent: f64, + /// What serving again may cost once the storage is reachable, and the + /// number each stream's recovery gap is reported against. Recorded rather + /// than asserted, like every other budget in the suite: how long a + /// connection pool may take to notice its database came back is a + /// judgement, and the number is in the result either way. + pub recovery_budget_secs: u64, +} + +impl StorageConfig { + pub fn recovery_budget(&self) -> Duration { + Duration::from_secs(self.recovery_budget_secs) + } +} + /// One step of the executor scale schedule the workflow runs during the fault. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -736,6 +789,10 @@ pub struct ScenarioConfig { /// The component rollback. Absent for scenarios that do not run one. #[serde(default)] pub rollback: Option, + /// Storage-outage settings. Absent for scenarios that do not take a + /// storage dependency away. + #[serde(default)] + pub storage: Option, /// Shard-ownership oracle settings. Absent for scenarios that do not sample /// executor assignments. #[serde(default)] @@ -882,6 +939,57 @@ impl ScenarioConfig { }) } + /// The storage-outage block. See [`Self::require_workload`]. + pub fn require_storage(&self) -> anyhow::Result<&StorageConfig> { + let config = self.storage.as_ref().ok_or_else(|| { + anyhow::anyhow!( + "chaos scenario {} needs a `storage` block in the suite YAML", + self.code + ) + })?; + // A zero ceiling can never be met: an operation submitted just before + // the heal confirms just after it and is counted in the fault window it + // was submitted in, so even a total outage leaves a small non-zero + // cell. Every run would then report the outage as not observed, and the + // one verdict that exists to catch a fault which never landed would be + // stuck on. + if config.outage_ceiling_percent <= 0.0 { + anyhow::bail!( + "chaos scenario {}: outageCeilingPercent is {}, which no real outage can meet, \ + so every run would report the fault as not observed", + self.code, + config.outage_ceiling_percent + ); + } + if config.endpoint.trim().is_empty() { + anyhow::bail!( + "chaos scenario {}: storage.endpoint is empty, so the result could not say which \ + storage the run took away", + self.code + ); + } + // Checked here rather than in the driver so a bad YAML fails the build + // instead of a maintenance window. Both blocks write to the scheduled + // stream, but only the `scheduled` block's registrations carry a token + // into the target's fire log — see `ScheduleEmitter::schedule_fire_at` + // in the counters component. Driving both would leave the fire account + // pairing the mixed workload's tokenless registrations against nothing + // and reporting every one of them as an action that never ran. + if let (Some(workload), Some(_)) = (&self.workload, &self.scheduled) + && workload.scheduled_agents > 0 + { + anyhow::bail!( + "chaos scenario {}: the mixed workload drives {} scheduled agents while a \ + `scheduled` block is also present, and only the latter's registrations carry a \ + token into the fire log, so the fire account would report every mixed-workload \ + registration as one that never fired", + self.code, + workload.scheduled_agents + ); + } + Ok(config) + } + /// The pinned workload block. See [`Self::require_workload`]. pub fn require_pinned(&self) -> anyhow::Result<&PinnedConfig> { self.pinned.as_ref().ok_or_else(|| { @@ -1021,6 +1129,7 @@ mod tests { revert: None, delete: None, rollback: None, + storage: None, ownership: None, scale_during_fault: None, retry_policy: RetryPolicy::default(), @@ -1057,6 +1166,7 @@ mod tests { isolation: None, delete: None, rollback: None, + storage: None, revert: Some(RevertConfig { agents: 10, increments_per_round: increments, @@ -1101,6 +1211,112 @@ mod tests { assert_eq!(config.require_revert().unwrap().net_per_round(), 0); } + fn storage_config( + endpoint: &str, + ceiling: f64, + scheduled_agents: u32, + scheduled_block: bool, + ) -> ScenarioConfig { + ScenarioConfig { + code: "S16".to_string(), + name: "keyvalue-postgres-outage".to_string(), + enabled: true, + fault: FaultConfig { + kind: "network-partition".to_string(), + target: "worker-executor".to_string(), + mode: "all".to_string(), + target_count: None, + duration_secs: 180, + }, + phases: PhaseConfig { + baseline_secs: 1, + fault_secs: 1, + recovery_secs: 1, + }, + workload: Some(WorkloadConfig { + durable_agents: 10, + ephemeral_agents: 0, + scheduled_agents, + promise_agents: 0, + quota_agents: 0, + rate_per_sec: 10, + }), + pinned: None, + scheduled: scheduled_block.then_some(ScheduledConfig { + targets: 10, + interval_millis: 2000, + lead_secs: 10, + lease_budget_secs: 240, + }), + promise: None, + isolation: None, + delete: None, + rollback: None, + storage: Some(StorageConfig { + endpoint: endpoint.to_string(), + outage_ceiling_percent: ceiling, + recovery_budget_secs: 120, + }), + revert: None, + ownership: None, + scale_during_fault: None, + retry_policy: RetryPolicy::default(), + signal_timeout_secs: 1, + } + } + + /// A zero ceiling can never be met, so the one verdict that exists to catch + /// a fault which never landed would be stuck on for every run. + #[test] + fn a_storage_outage_ceiling_of_zero_is_refused() { + let error = storage_config("db.example", 0.0, 0, true) + .require_storage() + .unwrap_err() + .to_string(); + assert!( + error.contains("no real outage can meet"), + "the message has to say why, got: {error}" + ); + } + + /// Without an endpoint the archived result could not say which storage the + /// run took away. + #[test] + fn a_storage_block_with_no_endpoint_is_refused() { + assert!( + storage_config(" ", 15.0, 0, true) + .require_storage() + .is_err() + ); + } + + /// Both blocks write to the scheduled stream and only one of them carries a + /// token into the fire log, so driving both would report every tokenless + /// registration as an action that never ran. Caught at load time rather + /// than discovered in the report. + #[test] + fn driving_the_scheduled_stream_from_both_blocks_is_refused() { + let error = storage_config("db.example", 15.0, 50, true) + .require_storage() + .unwrap_err() + .to_string(); + assert!( + error.contains("token into the fire log"), + "the message has to say why, got: {error}" + ); + } + + /// The same mixed-workload setting is fine on its own: without a + /// `scheduled` block there is only one writer to the stream. + #[test] + fn scheduled_agents_alone_are_allowed() { + assert!( + storage_config("db.example", 15.0, 50, false) + .require_storage() + .is_ok() + ); + } + /// The retry defaults are load-bearing for correctness, not just for load. #[test] fn retry_policy_defaults_to_one_same_key_transport_only_retry() { @@ -1165,6 +1381,7 @@ mod tests { assert_eq!(ScenarioCode::parse("s7"), Some(ScenarioCode::S7)); assert_eq!(ScenarioCode::parse("s6"), Some(ScenarioCode::S6)); assert_eq!(ScenarioCode::parse("s9"), Some(ScenarioCode::S9)); + assert_eq!(ScenarioCode::parse("s16"), Some(ScenarioCode::S16)); assert_eq!(ScenarioCode::parse("S99"), None); } @@ -1210,6 +1427,11 @@ mod tests { entry.require_workload().unwrap(); entry.require_rollback().unwrap(); } + ScenarioCode::S16 => { + entry.require_workload().unwrap(); + entry.require_scheduled().unwrap(); + entry.require_storage().unwrap(); + } } } } diff --git a/integration-tests/src/chaos/outage.rs b/integration-tests/src/chaos/outage.rs new file mode 100644 index 0000000000..021ca6e0c7 --- /dev/null +++ b/integration-tests/src/chaos/outage.rs @@ -0,0 +1,997 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! What a storage outage cost, and whether the platform came back from it +//! (GOL-379). +//! +//! ### The control is time, not a pod +//! +//! Every other partition scenario in this suite keeps a group of agents on the +//! healthy side of the cut and reads the verdict off the disagreement between +//! the two groups. A storage outage has no healthy side. All three executors +//! share one key-value cluster, so cutting them off from it cuts off everything +//! at once, and an agent that happened to live elsewhere would be no better +//! served. +//! +//! So the comparison runs along the other axis. Each stream is measured against +//! **its own before-fault rate**, and the question the report answers first is +//! whether that rate collapsed at all. It has to be asked explicitly, because a +//! storage partition that silently failed to take hold produces a report full +//! of healthy numbers, and that must read as "this run tested nothing" rather +//! than as a pass. +//! +//! ### Throughput, not success rate +//! +//! The same reason [`crate::chaos::reachability`] gives, arriving by a +//! different route. The mixed workload gives every stream its own in-flight +//! budget ([`crate::chaos::workload::start`]), so a stream whose operations +//! stop returning does not fail over and over: it fills its budget, stops +//! submitting, and offers nothing further. A success rate would read that as a +//! handful of failures out of a handful of attempts. Confirmed operations per +//! second is the number that collapses. +//! +//! The per-stream budget is also what makes a per-stream row worth printing. +//! With one shared pool a single stalled stream drains the budget for all of +//! them and every row degrades together, which is exactly the unattributable +//! result S1 produced before the budgets were split. +//! +//! ### What the streams are actually testing +//! +//! The key-value cluster holds more than its name suggests. Reading the +//! executor's deployment: promises, the running-workers set and user key-value +//! data go to its `KeyValue` namespaces, and the scheduler keeps its own schema +//! on the same cluster. The oplog is on a different Aurora cluster and the +//! worker-status hot cache is in Redis, and neither is touched here. +//! +//! That is why every stream degrades rather than only the obviously storage-shaped +//! ones. A durable increment needs the running-workers set before it can run at +//! all, so `durable` is not a control group and must not be read as one. +//! +//! ### What fails the run +//! +//! Two things, and both are statements about the experiment rather than about +//! latency: +//! +//! * [`OutageViolation::OutageNotObserved`] — the workload kept working, so the +//! fault did not land where the run says it did. +//! * [`OutageViolation::StreamNeverRecovered`] — a stream that was working +//! before the outage produced nothing at all after the heal. +//! +//! Recovery time is recorded against the configured budget and never asserted +//! on, like every other budget in the suite. How long a connection pool may +//! take to notice its database is back is a judgement, and the number is in the +//! result either way. + +use crate::chaos::errors::ErrorClass; +use crate::chaos::history::{OperationRecord, Outcome, Stream}; +use crate::chaos::split::{FaultWindow, Window, round2, window_secs, window_start}; +use crate::chaos::summary::LatencyStats; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; +use std::time::Duration; + +/// What an outage finding is about. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum OutageViolation { + /// The workload kept most of its baseline throughput while the storage was + /// supposed to be unreachable. Whatever the fault status said, the + /// executors could still reach the database, and every other number in this + /// report describes an undisturbed cluster. + OutageNotObserved, + /// A stream that was confirming operations before the outage confirmed + /// nothing at all after the heal. + StreamNeverRecovered, +} + +impl OutageViolation { + pub fn as_str(self) -> &'static str { + match self { + OutageViolation::OutageNotObserved => "outage-not-observed", + OutageViolation::StreamNeverRecovered => "stream-never-recovered", + } + } +} + +impl std::fmt::Display for OutageViolation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// One violation, against one stream. The aggregate verdict carries no stream. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct OutageFinding { + pub violation: OutageViolation, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stream: Option, + pub detail: String, +} + +/// What one stream managed in one window. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StreamThroughputCell { + pub stream: Stream, + pub window: Window, + /// Agents of this stream that offered at least one operation in this + /// window. Below the stream's pool size means emitters were stalled across + /// the whole window rather than merely slowed. + pub agents_active: usize, + pub submitted: u64, + pub confirmed: u64, + pub rejected: u64, + pub indeterminate: u64, + /// Attempts that hit the client's attempt timeout rather than answering. + pub attempts_timed_out: u64, + pub window_secs: f64, + pub confirmed_per_sec: f64, + /// This cell's rate against the same stream's own before-fault rate. `None` + /// for the before-fault cell itself, and for a stream that never had a + /// baseline to compare against. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub share_of_baseline_percent: Option, + /// How long into this window the stream waited before offering anything. + /// + /// The number that stops a small non-zero during-fault rate being read as + /// residual service: an operation submitted just before the heal can confirm + /// just after it and still be counted here. `quietMs` next to `windowSecs` + /// says whether the stream was serving throughout or answered only at the + /// end, and it needs no threshold to do it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub quiet_ms: Option, + pub latency: LatencyStats, +} + +/// The operations the outage began underneath. +/// +/// These were submitted before the storage went away and were still running +/// when it did, which makes them the population at risk: the platform may have +/// executed them, may have half-executed them, and cannot tell the client +/// which. They cannot be read off the cells, because every trace of them is +/// attributed to the `before-fault` row they were submitted in. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StreamCaughtInFlight { + pub stream: Stream, + pub operations: u64, + /// Distinct agents they belonged to. + pub agents: usize, + pub confirmed: u64, + pub rejected: u64, + pub indeterminate: u64, + /// Submission to final outcome, across every attempt. + pub duration: LatencyStats, + pub attempts_timed_out: u64, + /// The most attempts any one of them needed. An operation that stalled and + /// then answered on a later attempt was rescued by the caller's retry, not + /// returned by the platform, and the outcome alone cannot say so. + pub max_attempts: u32, + /// How many were still unresolved when the storage was reported reachable + /// again. + pub outlived_the_fault: u64, +} + +/// How long one stream took to serve anything again. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StreamRecovery { + pub stream: Stream, + /// Milliseconds from the heal to this stream's first confirmed operation. + /// `None` means it never confirmed one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub first_confirmed_ms: Option, + /// Whether that exceeded the configured budget. Recorded, not asserted. + pub over_budget: bool, +} + +/// How operations failed while the storage was unreachable. +/// +/// The acceptance criteria ask for fault-window failures, and a count alone +/// does not answer the question an operator has: whether the platform refused +/// the work definitively, or accepted it and then lost the ability to say what +/// happened. That is exactly the [`ErrorClass`] split, so the histogram is +/// keyed on it. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FaultWindowErrors { + pub stream: Stream, + pub class: ErrorClass, + pub operations: u64, + /// One message, for the operator to paste into a log query. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub example: Option, +} + +/// The storage-outage account. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StorageOutageReport { + /// The endpoint the workflow was asked to cut the executors off from, + /// recorded so an archived result says which storage the run was about + /// rather than leaving it to the scenario name. + pub endpoint: String, + /// The thresholds from the suite YAML, recorded so an archived cell can be + /// read years later against the numbers it was judged by rather than + /// against today's config. + pub outage_ceiling_percent: f64, + pub recovery_budget_ms: u64, + /// The whole workload's during-fault rate as a share of its own baseline. + /// `None` for a run that never learned when the fault was. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub share_of_baseline_percent: Option, + pub cells: Vec, + /// What the outage began underneath, per stream. Empty for a run that never + /// learned when the fault was. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub caught_in_flight: Vec, + pub recovery: Vec, + pub fault_window_errors: Vec, + pub findings: Vec, +} + +/// One stream's per-window accumulation, before it becomes a cell. +#[derive(Default)] +struct Tally { + agents: BTreeSet, + first_submitted: Option>, + submitted: u64, + confirmed: u64, + rejected: u64, + indeterminate: u64, + attempts_timed_out: u64, + durations: Vec, +} + +impl StorageOutageReport { + /// Builds the account from the operation history. + /// + /// `fault` is what the workflow reported. Without it every record lands in + /// [`Window::Unknown`] and the report carries counts but no verdict, which + /// is the honest outcome for a run that never learned when the fault was: + /// both thresholds are defined relative to a before-and-during comparison + /// that cannot be made. + pub fn build( + records: &[OperationRecord], + fault: Option, + endpoint: &str, + outage_ceiling_percent: f64, + recovery_budget: Duration, + ) -> Self { + let mut tallies: BTreeMap<(Stream, Window), Tally> = BTreeMap::new(); + let mut first_submitted: Option> = None; + let mut last_completed: Option> = None; + + for record in records { + let window = Window::of(record.submitted_at, fault); + let tally = tallies.entry((record.stream, window)).or_default(); + + tally.agents.insert(record.agent.clone()); + tally.first_submitted = Some(match tally.first_submitted { + Some(at) if at <= record.submitted_at => at, + _ => record.submitted_at, + }); + tally.submitted += 1; + match record.outcome { + Outcome::Confirmed => { + tally.confirmed += 1; + tally.durations.push(record.duration_ms); + } + Outcome::Rejected => tally.rejected += 1, + Outcome::Indeterminate => tally.indeterminate += 1, + } + tally.attempts_timed_out += record.attempts_timed_out(); + + first_submitted = Some(match first_submitted { + Some(at) if at <= record.submitted_at => at, + _ => record.submitted_at, + }); + if let Some(completed) = record.completed_at { + last_completed = Some(match last_completed { + Some(at) if at >= completed => at, + _ => completed, + }); + } + } + + // Baselines first: every other cell is expressed as a share of its own + // stream's before-fault rate, so a stream the workload drives rarely + // cannot make the picture look better or worse than it was. + let mut baseline_rate: BTreeMap = BTreeMap::new(); + let mut cells: Vec = Vec::new(); + for ((stream, window), tally) in &tallies { + let secs = window_secs(*window, fault, first_submitted, last_completed); + let rate = if secs > 0.0 { + tally.confirmed as f64 / secs + } else { + 0.0 + }; + if *window == Window::BeforeFault { + baseline_rate.insert(*stream, rate); + } + let quiet_ms = window_start(*window, fault, first_submitted) + .zip(tally.first_submitted) + .map(|(start, first)| (first - start).num_milliseconds().max(0) as u64); + cells.push(StreamThroughputCell { + stream: *stream, + window: *window, + agents_active: tally.agents.len(), + submitted: tally.submitted, + confirmed: tally.confirmed, + rejected: tally.rejected, + indeterminate: tally.indeterminate, + attempts_timed_out: tally.attempts_timed_out, + window_secs: round2(secs), + confirmed_per_sec: round2(rate), + share_of_baseline_percent: None, + quiet_ms, + latency: LatencyStats::from_durations(tally.durations.clone()), + }); + } + + for cell in &mut cells { + if cell.window == Window::BeforeFault { + continue; + } + if let Some(base) = baseline_rate.get(&cell.stream).copied() + && base > 0.0 + { + cell.share_of_baseline_percent = + Some(round2(cell.confirmed_per_sec / base * 100.0)); + } + } + cells.sort_by_key(|c| (c.stream, c.window)); + + let mut report = Self { + endpoint: endpoint.to_string(), + outage_ceiling_percent, + recovery_budget_ms: recovery_budget.as_millis().min(u64::MAX as u128) as u64, + share_of_baseline_percent: None, + cells, + caught_in_flight: caught_in_flight(records, fault), + recovery: Vec::new(), + fault_window_errors: fault_window_errors(records, fault), + findings: Vec::new(), + }; + + report.judge_outage(fault, first_submitted, last_completed); + report.judge_recovery(records, fault, recovery_budget, &baseline_rate); + report + } + + /// Did the outage land? Judged on the whole workload rather than per + /// stream: the claim under test is that the executors could not reach the + /// database, which is one claim about the cluster, and a low-volume stream + /// should not be able to flip it on its own. + fn judge_outage( + &mut self, + fault: Option, + first_submitted: Option>, + last_completed: Option>, + ) { + if fault.is_none() { + return; + } + let total = |window: Window| -> f64 { + let confirmed: u64 = self + .cells + .iter() + .filter(|c| c.window == window) + .map(|c| c.confirmed) + .sum(); + let secs = window_secs(window, fault, first_submitted, last_completed); + if secs > 0.0 { + confirmed as f64 / secs + } else { + 0.0 + } + }; + + let baseline = total(Window::BeforeFault); + if baseline <= 0.0 { + return; + } + let share = round2(total(Window::DuringFault) / baseline * 100.0); + self.share_of_baseline_percent = Some(share); + if share > self.outage_ceiling_percent { + self.findings.push(OutageFinding { + violation: OutageViolation::OutageNotObserved, + stream: None, + detail: format!( + "the workload held {share}% of its baseline throughput while {} was supposed \ + to be unreachable, above the {}% ceiling; the executors could still reach it", + self.endpoint, self.outage_ceiling_percent + ), + }); + } + } + + /// How long each stream took to serve again, and which never did. + /// + /// Only streams that were working *before* the outage are judged. A stream + /// the run never got going is a different problem, already reported by the + /// baseline gate in the driver, and calling it unrecovered here would put + /// the same failure in two places under two names. + fn judge_recovery( + &mut self, + records: &[OperationRecord], + fault: Option, + budget: Duration, + baseline_rate: &BTreeMap, + ) { + let Some(FaultWindow { + recovered_at: Some(recovered), + .. + }) = fault + else { + return; + }; + let budget_ms = budget.as_millis().min(i64::MAX as u128) as i64; + + for (stream, base) in baseline_rate { + if *base <= 0.0 { + continue; + } + let first = records + .iter() + .filter(|r| r.stream == *stream && r.outcome == Outcome::Confirmed) + .filter_map(|r| r.completed_at) + .filter(|at| *at >= recovered) + .min(); + + let first_confirmed_ms = + first.map(|at| (at - recovered).num_milliseconds().max(0) as u64); + let over_budget = first + .map(|at| (at - recovered).num_milliseconds() > budget_ms) + .unwrap_or(false); + + if first.is_none() { + self.findings.push(OutageFinding { + violation: OutageViolation::StreamNeverRecovered, + stream: Some(*stream), + detail: format!( + "the {stream} stream confirmed operations before the outage and none \ + after {} became reachable again", + self.endpoint + ), + }); + } + self.recovery.push(StreamRecovery { + stream: *stream, + first_confirmed_ms, + over_budget, + }); + } + } + + /// One stream's cell for one window, for the tests and for anything reading + /// the report without wanting to scan the whole list. + pub fn cell(&self, stream: Stream, window: Window) -> Option<&StreamThroughputCell> { + self.cells + .iter() + .find(|c| c.stream == stream && c.window == window) + } + + /// Whether the report found anything a human has to act on. + pub fn has_findings(&self) -> bool { + !self.findings.is_empty() + } + + /// Lines that need a human. Empty on a run where the outage landed and + /// everything came back. + pub fn attention_lines(&self) -> Vec { + self.findings + .iter() + .map(|f| match f.stream { + Some(stream) => format!("{}: {stream}: {}", f.violation, f.detail), + None => format!("{}: {}", f.violation, f.detail), + }) + .collect() + } + + /// Context a reader needs in order to read the cells, which is not itself a + /// problem. + pub fn note_lines(&self) -> Vec { + let mut lines = Vec::new(); + if let Some(share) = self.share_of_baseline_percent { + lines.push(format!( + "Storage outage: the workload held {share}% of its baseline throughput while {} \ + was unreachable (ceiling {}%)", + self.endpoint, self.outage_ceiling_percent + )); + } else { + lines.push(format!( + "Storage outage: no fault window was reported, so the {} cells carry counts but \ + no verdict", + self.endpoint + )); + } + let over: Vec = self + .recovery + .iter() + .filter(|r| r.over_budget) + .map(|r| { + format!( + "{} ({}ms)", + r.stream, + r.first_confirmed_ms.unwrap_or_default() + ) + }) + .collect(); + if !over.is_empty() { + lines.push(format!( + "Storage outage: {} took longer than the {}ms recovery budget to serve again: {}", + over.len(), + self.recovery_budget_ms, + over.join(", ") + )); + } + lines + } +} + +/// The operations that were submitted before the outage and still running when +/// it started. +fn caught_in_flight( + records: &[OperationRecord], + fault: Option, +) -> Vec { + let Some(window) = fault else { + return Vec::new(); + }; + let mut by_stream: BTreeMap> = BTreeMap::new(); + for record in records { + // Submitted before the cut, and either still unfinished when it landed + // or finished after it. An operation with no completion at all is + // included: the driver never learned how it ended, which is the same + // doubt in a starker form. + if record.submitted_at >= window.injected_at { + continue; + } + let still_running = record + .completed_at + .map(|at| at >= window.injected_at) + .unwrap_or(true); + if still_running { + by_stream.entry(record.stream).or_default().push(record); + } + } + + by_stream + .into_iter() + .map(|(stream, caught)| StreamCaughtInFlight { + stream, + operations: caught.len() as u64, + agents: caught + .iter() + .map(|r| r.agent.as_str()) + .collect::>() + .len(), + confirmed: count_of(&caught, Outcome::Confirmed), + rejected: count_of(&caught, Outcome::Rejected), + indeterminate: count_of(&caught, Outcome::Indeterminate), + duration: LatencyStats::from_durations( + caught.iter().map(|r| r.duration_ms).collect::>(), + ), + attempts_timed_out: caught.iter().map(|r| r.attempts_timed_out()).sum(), + max_attempts: caught.iter().map(|r| r.attempts).max().unwrap_or(0), + outlived_the_fault: caught + .iter() + .filter(|r| match (r.completed_at, window.recovered_at) { + (Some(at), Some(recovered)) => at >= recovered, + (None, _) => true, + _ => false, + }) + .count() as u64, + }) + .collect() +} + +/// How the operations submitted during the outage failed, by stream and class. +fn fault_window_errors( + records: &[OperationRecord], + fault: Option, +) -> Vec { + let mut tallies: BTreeMap<(Stream, ErrorClass), (u64, Option)> = BTreeMap::new(); + for record in records { + if Window::of(record.submitted_at, fault) != Window::DuringFault { + continue; + } + let Some(class) = record.error_class else { + continue; + }; + let entry = tallies.entry((record.stream, class)).or_insert((0, None)); + entry.0 += 1; + if entry.1.is_none() { + entry.1.clone_from(&record.error); + } + } + + let mut rows: Vec = tallies + .into_iter() + .map( + |((stream, class), (operations, example))| FaultWindowErrors { + stream, + class, + operations, + example, + }, + ) + .collect(); + // Commonest first: an operator reading this wants the dominant failure mode + // before the long tail. + rows.sort_by(|a, b| { + b.operations + .cmp(&a.operations) + .then((a.stream, a.class).cmp(&(b.stream, b.class))) + }); + rows +} + +fn count_of(records: &[&OperationRecord], outcome: Outcome) -> u64 { + records.iter().filter(|r| r.outcome == outcome).count() as u64 +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::chaos::history::{AttemptRecord, Phase}; + use chrono::TimeDelta; + use test_r::test; + + const ENDPOINT: &str = "golem-postgres-dev-keyvalue.cluster-example.rds.amazonaws.com"; + const CEILING: f64 = 15.0; + const AGENT: &str = "chaos-s16-durable-0000"; + + fn t0() -> DateTime { + DateTime::parse_from_rfc3339("2026-08-25T12:00:00Z") + .unwrap() + .with_timezone(&Utc) + } + + fn fault() -> FaultWindow { + FaultWindow { + injected_at: t0(), + recovered_at: Some(t0() + TimeDelta::seconds(180)), + } + } + + /// One operation, submitted `offset` seconds from the moment the storage + /// was taken away. Negative offsets are the baseline. + fn op(stream: Stream, offset_secs: i64, outcome: Outcome) -> OperationRecord { + let submitted_at = t0() + TimeDelta::seconds(offset_secs); + OperationRecord { + op_id: 0, + stream, + phase: Phase::Baseline, + agent: AGENT.to_string(), + method: "increment".to_string(), + idempotency_key: format!("{stream}-{offset_secs}"), + submitted_at, + completed_at: Some(submitted_at + TimeDelta::milliseconds(20)), + attempts: 1, + outcome, + duration_ms: 20, + returned_value: Some(1), + first_attempt_value: None, + error: None, + error_class: None, + attempt_log: vec![AttemptRecord { + attempt: 1, + started_at: submitted_at, + duration_ms: 20, + returned_value: Some(1), + succeeded: outcome == Outcome::Confirmed, + error_class: None, + error: None, + }], + } + } + + /// An operation that hung until the client gave up, twice: the shape every + /// invocation takes while the database is unreachable. + fn stalled(stream: Stream, offset_secs: i64) -> OperationRecord { + let mut record = op(stream, offset_secs, Outcome::Indeterminate); + record.duration_ms = 245_000; + record.completed_at = Some(record.submitted_at + TimeDelta::seconds(245)); + record.returned_value = None; + record.attempts = 2; + record.error_class = Some(ErrorClass::Transport); + record.error = Some("attempt timed out after 120s".to_string()); + record.attempt_log = (1..=2) + .map(|attempt| AttemptRecord { + attempt, + started_at: record.submitted_at, + duration_ms: 120_000, + returned_value: None, + succeeded: false, + error_class: Some(ErrorClass::Transport), + error: Some("attempt timed out after 120s".to_string()), + }) + .collect(); + record + } + + /// 300s of baseline on both streams, then `during` confirmations while the + /// storage is gone, then 240s of recovery. + fn history(during: usize, recovery: bool) -> Vec { + let mut records = Vec::new(); + for second in 1..=300 { + records.push(op(Stream::Durable, -second, Outcome::Confirmed)); + records.push(op(Stream::Scheduled, -second, Outcome::Confirmed)); + } + for i in 0..during { + records.push(op(Stream::Durable, i as i64, Outcome::Confirmed)); + records.push(op(Stream::Scheduled, i as i64, Outcome::Confirmed)); + } + if recovery { + for second in 181..=420 { + records.push(op(Stream::Durable, second, Outcome::Confirmed)); + records.push(op(Stream::Scheduled, second, Outcome::Confirmed)); + } + } + records + } + + fn build(records: &[OperationRecord]) -> StorageOutageReport { + StorageOutageReport::build( + records, + Some(fault()), + ENDPOINT, + CEILING, + Duration::from_secs(120), + ) + } + + /// The healthy shape: throughput collapses while the database is gone and + /// comes back afterwards, and the report raises nothing. + #[test] + fn an_outage_that_lands_and_heals_has_no_findings() { + let report = build(&history(2, true)); + + assert!( + report.findings.is_empty(), + "expected no findings, got {:?}", + report.findings + ); + let during = report.cell(Stream::Durable, Window::DuringFault).unwrap(); + assert!( + during.share_of_baseline_percent.unwrap() < CEILING, + "the stream should have collapsed, got {during:?}" + ); + } + + /// The one thing this scenario cannot afford to be quiet about. A partition + /// that never took hold leaves every other number describing a cluster + /// nothing happened to, and a reader has to be told rather than left to + /// infer it from cells that all look fine. + #[test] + fn a_workload_that_kept_working_reports_the_outage_as_not_observed() { + // 180 confirmations per stream across the 180s fault window is the + // baseline cadence carrying straight on through it. + let report = build(&history(180, true)); + + assert!( + report + .findings + .iter() + .any(|f| f.violation == OutageViolation::OutageNotObserved), + "expected an outage-not-observed finding, got {:?}", + report.findings + ); + assert!( + report.share_of_baseline_percent.unwrap() > CEILING, + "the aggregate share should be above the ceiling, got {:?}", + report.share_of_baseline_percent + ); + } + + /// The verdict is on the whole workload rather than per stream, so a stream + /// the run barely drives cannot flip it on its own. + #[test] + fn one_busy_stream_holding_up_cannot_be_outvoted_by_a_quiet_one() { + let mut records = history(0, true); + // A trickle of a third stream that keeps working throughout: two + // operations in the whole fault window against a baseline of hundreds. + for i in 0..2 { + records.push(op(Stream::Promise, i, Outcome::Confirmed)); + } + for second in 1..=10 { + records.push(op(Stream::Promise, -second, Outcome::Confirmed)); + } + let report = build(&records); + + assert!( + !report + .findings + .iter() + .any(|f| f.violation == OutageViolation::OutageNotObserved), + "a trickle should not make the outage look unobserved, got {:?}", + report.findings + ); + } + + /// A stream that was working before the outage and confirmed nothing after + /// the heal is the platform losing a mechanism, not a slow recovery. + #[test] + fn a_stream_that_never_came_back_is_a_finding() { + let mut records = history(0, true); + records.retain(|r| !(r.stream == Stream::Scheduled && r.submitted_at > t0())); + let report = build(&records); + + let finding = report + .findings + .iter() + .find(|f| f.violation == OutageViolation::StreamNeverRecovered) + .expect("expected a stream-never-recovered finding"); + assert_eq!(finding.stream, Some(Stream::Scheduled)); + } + + /// A stream the run never got going is the driver's problem, already + /// reported by the baseline gate. Calling it unrecovered here would put one + /// failure in two places under two names. + #[test] + fn a_stream_that_never_worked_at_all_is_not_reported_as_unrecovered() { + let mut records = history(0, true); + records.retain(|r| r.stream != Stream::Scheduled); + // Present throughout, never confirming. + for second in 1..=300 { + records.push(op(Stream::Scheduled, -second, Outcome::Rejected)); + } + let report = build(&records); + + assert!( + !report + .findings + .iter() + .any(|f| f.stream == Some(Stream::Scheduled)), + "a stream that never worked should not be reported as unrecovered, got {:?}", + report.findings + ); + } + + /// Recovery is measured and reported against the budget, never asserted on. + #[test] + fn a_slow_recovery_is_recorded_rather_than_a_finding() { + let mut records = history(0, false); + // First confirmation 200s after the heal, past the 120s budget. + records.push(op(Stream::Durable, 380, Outcome::Confirmed)); + records.push(op(Stream::Scheduled, 380, Outcome::Confirmed)); + let report = build(&records); + + assert!( + report.findings.is_empty(), + "a slow recovery must not be a finding, got {:?}", + report.findings + ); + assert!( + report.recovery.iter().all(|r| r.over_budget), + "both streams should be recorded as over budget, got {:?}", + report.recovery + ); + } + + /// Without a fault window nothing can be placed either side of the outage, + /// so the report carries counts and refuses to reach a verdict. + #[test] + fn without_a_fault_window_there_is_no_verdict() { + let report = StorageOutageReport::build( + &history(180, true), + None, + ENDPOINT, + CEILING, + Duration::from_secs(120), + ); + + assert!(report.findings.is_empty()); + assert_eq!(report.share_of_baseline_percent, None); + assert!(report.caught_in_flight.is_empty()); + assert!( + report + .cells + .iter() + .all(|c| c.window == Window::Unknown && c.share_of_baseline_percent.is_none()), + "every cell should be unplaceable, got {:?}", + report.cells + ); + } + + /// The number that stops a small non-zero during-fault cell reading as + /// residual service: it says the stream was silent for almost the whole + /// window and answered only at the end. + #[test] + fn the_during_fault_cell_says_how_long_the_stream_stayed_quiet() { + let mut records = history(0, true); + // One confirmation, 170s into the 180s outage. + records.push(op(Stream::Durable, 170, Outcome::Confirmed)); + let report = build(&records); + + let during = report.cell(Stream::Durable, Window::DuringFault).unwrap(); + assert_eq!(during.confirmed, 1); + assert_eq!(during.quiet_ms, Some(170_000)); + } + + /// The operations at risk. They were submitted before the cut and are + /// attributed to the before-fault row, which is the last place anyone looks + /// for the damage. + #[test] + fn operations_running_when_the_storage_went_away_are_reported_apart() { + let mut records = history(0, true); + // Submitted 30s before the cut, still running when it landed. + for _ in 0..3 { + records.push(stalled(Stream::Durable, -30)); + } + let report = build(&records); + + let caught = report + .caught_in_flight + .iter() + .find(|c| c.stream == Stream::Durable) + .expect("expected a caught-in-flight row for the durable stream"); + assert_eq!(caught.operations, 3); + assert_eq!(caught.indeterminate, 3); + assert_eq!(caught.attempts_timed_out, 6); + assert_eq!(caught.max_attempts, 2); + // 245s from 30s before the cut lands 35s past the 180s heal. + assert_eq!(caught.outlived_the_fault, 3); + } + + /// An operator reading the fault window wants the dominant failure mode + /// first, and wants to know whether the platform refused the work or lost + /// track of it. + #[test] + fn fault_window_failures_are_grouped_by_class_commonest_first() { + let mut records = history(0, true); + for i in 0..5 { + records.push(stalled(Stream::Durable, i)); + } + let mut rejected = op(Stream::Durable, 10, Outcome::Rejected); + rejected.error_class = Some(ErrorClass::Response); + rejected.error = Some("agent not found".to_string()); + records.push(rejected); + let report = build(&records); + + assert_eq!(report.fault_window_errors.len(), 2); + assert_eq!(report.fault_window_errors[0].class, ErrorClass::Transport); + assert_eq!(report.fault_window_errors[0].operations, 5); + assert_eq!( + report.fault_window_errors[0].example.as_deref(), + Some("attempt timed out after 120s") + ); + assert_eq!(report.fault_window_errors[1].class, ErrorClass::Response); + assert_eq!(report.fault_window_errors[1].operations, 1); + } + + /// Findings are hoisted for a human; the share line is context and belongs + /// with the notes, not with the things to act on. + #[test] + fn the_share_line_is_a_note_and_the_findings_are_attention() { + let report = build(&history(180, true)); + + assert!( + report + .attention_lines() + .iter() + .any(|l| l.contains("outage-not-observed")) + ); + assert!( + report + .note_lines() + .iter() + .any(|l| l.contains("of its baseline throughput")) + ); + } +} diff --git a/integration-tests/src/chaos/reachability.rs b/integration-tests/src/chaos/reachability.rs index b40b755494..e5996ad5e7 100644 --- a/integration-tests/src/chaos/reachability.rs +++ b/integration-tests/src/chaos/reachability.rs @@ -46,7 +46,9 @@ //! baseline of hundreds. A cell anywhere near the ceiling means something else. use crate::chaos::history::{OperationRecord, Outcome, Stream}; -use crate::chaos::split::{FaultWindow, Group, PodSplit, Window}; +use crate::chaos::split::{ + FaultWindow, Group, PodSplit, Window, round2, window_secs, window_start, +}; use crate::chaos::summary::LatencyStats; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; @@ -276,7 +278,7 @@ impl ReachabilityReport { Outcome::Rejected => tally.rejected += 1, Outcome::Indeterminate => tally.indeterminate += 1, } - tally.attempts_timed_out += timed_out(record); + tally.attempts_timed_out += record.attempts_timed_out(); first_submitted = Some(match first_submitted { Some(at) if at <= record.submitted_at => at, @@ -397,7 +399,7 @@ impl ReachabilityReport { duration: LatencyStats::from_durations( caught.iter().map(|r| r.duration_ms).collect(), ), - attempts_timed_out: caught.iter().map(|r| timed_out(r)).sum(), + attempts_timed_out: caught.iter().map(|r| r.attempts_timed_out()).sum(), max_attempts: caught.iter().map(|r| r.attempts).max().unwrap_or(0), outlived_the_fault: caught .iter() @@ -631,90 +633,11 @@ impl ReachabilityReport { } } -/// When a window began, for the windows that have a fixed start. -/// -/// The fault's own boundaries come from the workflow's timestamps, which is -/// what makes them comparable across runs. The baseline has no boundary of its -/// own, so it starts at the first operation the run offered. -fn window_start( - window: Window, - fault: Option, - first_submitted: Option>, -) -> Option> { - match (window, fault) { - (Window::BeforeFault, Some(_)) => first_submitted, - (Window::DuringFault, Some(w)) => Some(w.injected_at), - (Window::AfterFault, Some(w)) => w.recovered_at, - _ => None, - } -} - -/// How long a window lasted, in seconds. -/// -/// The fault's own windows come from the workflow's timestamps, which is what -/// makes them comparable across runs. The two open-ended ones are bounded by -/// the workload instead: the baseline starts at the first operation offered, -/// and recovery ends at the last one that came back. -fn window_secs( - window: Window, - fault: Option, - first_submitted: Option>, - last_completed: Option>, -) -> f64 { - let seconds = |from: DateTime, to: DateTime| { - (to - from).num_milliseconds().max(0) as f64 / 1000.0 - }; - match (window, fault) { - (Window::BeforeFault, Some(w)) => first_submitted - .map(|first| seconds(first, w.injected_at)) - .unwrap_or(0.0), - (Window::DuringFault, Some(w)) => match (w.recovered_at, last_completed) { - (Some(recovered), _) => seconds(w.injected_at, recovered), - // A run that never saw the heal: the fault ran to whatever the last - // operation saw, which is the most that can be claimed. - (None, Some(last)) => seconds(w.injected_at, last), - (None, None) => 0.0, - }, - ( - Window::AfterFault, - Some(FaultWindow { - recovered_at: Some(recovered), - .. - }), - ) => last_completed - .map(|last| seconds(recovered, last)) - .unwrap_or(0.0), - _ => 0.0, - } -} - /// Operations of one outcome. fn count_of(records: &[&OperationRecord], outcome: Outcome) -> u64 { records.iter().filter(|r| r.outcome == outcome).count() as u64 } -/// Attempts of one operation that hit the client's attempt timeout. -/// -/// Matched on the message `crate::chaos::workload` writes for a timed-out -/// attempt. A structured flag would be better, but the attempt log is an -/// archived shape shared with every other scenario and this reads it without -/// changing it. -fn timed_out(record: &OperationRecord) -> u64 { - record - .attempt_log - .iter() - .filter(|a| { - a.error - .as_deref() - .is_some_and(|e| e.contains("attempt timed out")) - }) - .count() as u64 -} - -fn round2(value: f64) -> f64 { - (value * 100.0).round() / 100.0 -} - #[cfg(test)] mod tests { use super::*; diff --git a/integration-tests/src/chaos/result.rs b/integration-tests/src/chaos/result.rs index 9f058c5c3c..08fd25d12d 100644 --- a/integration-tests/src/chaos/result.rs +++ b/integration-tests/src/chaos/result.rs @@ -30,7 +30,7 @@ use crate::chaos::split::PodSplit; use crate::chaos::summary::{ChaosSummary, TerminationReason}; use crate::chaos::{ DeleteConfig, FaultConfig, IsolationConfig, PinnedConfig, PromiseConfig, RetryPolicy, - RevertConfig, RollbackConfig, ScheduledConfig, WorkloadConfig, + RevertConfig, RollbackConfig, ScheduledConfig, StorageConfig, WorkloadConfig, }; use chrono::{DateTime, Utc}; use golem_test_framework::benchmark::RunMetadata; @@ -179,6 +179,10 @@ pub struct ChaosResult { /// The component rollback the run was configured with, if any. #[serde(default, skip_serializing_if = "Option::is_none")] pub rollback: Option, + /// The storage the run was configured to take away, and the thresholds its + /// account was judged by. Present only for S16. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub storage: Option, pub retry_policy: RetryPolicy, pub scope: RunScope, pub summary: ChaosSummary, @@ -257,6 +261,7 @@ mod tests { delete: None, delete_selection: None, rollback: None, + storage: None, retry_policy: RetryPolicy::default(), scope: RunScope { environment_id: "env-1".to_string(), @@ -1007,6 +1012,7 @@ mod sample_artifact { delete: None, delete_selection: None, rollback: None, + storage: None, retry_policy: RetryPolicy::default(), scope: RunScope { environment_id: "0192f000-0000-7000-8000-000000000001".to_string(), @@ -1228,6 +1234,7 @@ mod sample_artifact { delete: None, delete_selection: None, rollback: None, + storage: None, retry_policy: RetryPolicy::default(), scope: RunScope { environment_id: "0192f000-0000-7000-8000-000000000001".to_string(), diff --git a/integration-tests/src/chaos/scenarios/mod.rs b/integration-tests/src/chaos/scenarios/mod.rs index b0232d777d..dd792a29d0 100644 --- a/integration-tests/src/chaos/scenarios/mod.rs +++ b/integration-tests/src/chaos/scenarios/mod.rs @@ -30,6 +30,7 @@ pub mod s10; pub mod s11; pub mod s12; pub mod s13; +pub mod s16; pub mod s3; pub mod s5; pub mod s6; @@ -132,6 +133,7 @@ pub fn build_result(config: &ScenarioConfig, outcome: ScenarioOutcome) -> ChaosR delete: config.delete.clone(), delete_selection: outcome.delete_selection, rollback: config.rollback.clone(), + storage: config.storage.clone(), retry_policy: config.retry_policy.clone(), scope: outcome.scope, summary: outcome.summary, diff --git a/integration-tests/src/chaos/scenarios/s16.rs b/integration-tests/src/chaos/scenarios/s16.rs new file mode 100644 index 0000000000..5bb1adfc33 --- /dev/null +++ b/integration-tests/src/chaos/scenarios/s16.rs @@ -0,0 +1,790 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! S16 — key-value PostgreSQL outage recovery (GOL-379). +//! +//! The first scenario in the suite that breaks something the platform depends +//! on rather than something the platform *is*. Every fault before this one +//! removed a golem process or a link between two of them, and in each case some +//! part of the cluster stayed healthy and could be compared against. Here the +//! executors keep running, keep their shards, keep answering the shard-manager, +//! and simply cannot reach the database underneath them. +//! +//! ## What the fault takes away +//! +//! More than the name suggests. Reading the golem-dev executor deployment, the +//! key-value Aurora cluster carries four things: +//! +//! * promises, +//! * the running-workers set, +//! * user-defined key-value data, +//! * and the scheduler, in its own schema on the same cluster. +//! +//! The oplog lives on a different Aurora cluster and the worker-status hot +//! cache lives in Redis, and the partition touches neither. That combination is +//! what makes the scenario worth running: the platform keeps the ability to +//! *record* what it did and loses the ability to know *what it is doing*. +//! +//! It also means no stream here is a control group. A durable increment needs +//! the running-workers set before it can start, so `durable` degrades along +//! with everything else and must not be read as untouched. +//! +//! ## The control is the baseline, not another pod +//! +//! S1 and S3 keep executors on the healthy side of the cut and read the verdict +//! off the disagreement between the two groups. There is no healthy side here: +//! all three executors share one cluster. So the comparison runs along time +//! instead, and every stream is measured against its own before-fault rate. See +//! [`crate::chaos::outage`] for what that costs and what it still answers. +//! +//! The first question it has to answer is whether the outage landed at all. A +//! partition that failed to take hold produces a report full of healthy numbers +//! and no error anywhere, which is the worst artifact this suite can produce, +//! so [`OutageViolation::OutageNotObserved`] is a named finding rather than +//! something a reader is left to infer from the cells. +//! +//! ## Why the scheduled stream is driven separately +//! +//! The mixed workload's scheduled stream registers through `schedule_poll_at`, +//! which increments a counter and records nothing else. That is enough to ask +//! "did every registration eventually fire" and useless for asking "how late". +//! Scheduler lag is one of the two things GOL-379 asks this run to record, and +//! the only place a due time survives is the target's own fire log, so S16 +//! drives the token-carrying registration loop from [`crate::chaos::scheduled`] +//! instead and leaves `scheduledAgents` at zero in the mixed workload. Setting +//! both is refused at load time — see `ScenarioConfig::require_storage`. +//! +//! One consequence worth stating plainly for anyone reading the result: the +//! fire account's `group` axis is degenerate here. S16 kills no executor, so +//! every target is reported as `elsewhere` and only the `window` axis carries +//! information. The delays to read are the `during-fault` and `after-fault` +//! cells against `before-fault`. +//! +//! ## What fails the run +//! +//! The same narrow set S3 fails on, and for the same reason: this suite reports +//! rather than judges, and the bar for failing outright is "the run produced +//! nothing worth interpreting". +//! +//! * A token-level scheduled-fire violation. An accepted action that never ran, +//! ran twice, or ran after being refused is a statement about one named +//! registration with no band of doubt around it. +//! * A key that the exactly-once probe shows executed twice. +//! * A workload that never confirmed anything at all. +//! +//! Everything the storage account finds is loud without being fatal: it lands +//! in `attention`, which CI annotates. An outage that did not land and a stream +//! that never came back are both things a human has to look at, and neither is +//! improved by turning the job red. +//! +//! Run-by-run findings live in the S16 runbook in golem-cloud, not here. + +use crate::chaos::fires::{FaultWindow, ScheduleFireReport}; +use crate::chaos::history::{OperationHistory, OperationRecord, Outcome, Phase, Stream}; +use crate::chaos::outage::{OutageViolation, StorageOutageReport}; +use crate::chaos::prep::ChaosPrepManifest; +use crate::chaos::probe; +use crate::chaos::result::{ChaosResult, PhaseWindow, Phases, RunScope}; +use crate::chaos::scenarios::{ + OutputPaths, ScenarioOutcome, WARMUP_SETTLE, build_result, exactly_once_termination, + read_counters, readback_for, signal_termination, snapshot_routing, wait_for_settled_routing, + write_outputs, +}; +use crate::chaos::scheduled; +use crate::chaos::signal::{BaselineReady, FaultSignals}; +use crate::chaos::summary::{ + AgentReadback, ChaosSummary, ExactlyOnceReport, Note, TerminationReason, +}; +use crate::chaos::workload::{self, PhaseMarker, WorkloadContext}; +use crate::chaos::{ScenarioCode, ScenarioConfig, ScheduledConfig}; +use chrono::Utc; +use golem_test_framework::config::BenchmarkTestDependencies; +use golem_test_framework::dsl::TestDsl; +use std::collections::BTreeSet; +use std::time::Duration; +use tracing::{info, warn}; + +/// Extra quiet after the last scheduled action is due, before anything is read +/// back. +/// +/// The rest of the settle is derived from the configuration, the same way S10 +/// derives it: the final registration falls due one `lead` after the workload +/// stops, and an action the outage delayed can cost up to one lease budget on +/// top. Reading before that has elapsed would report actions as lost that were +/// merely late, which is the one mistake this scenario cannot afford — a +/// storage outage is precisely the thing that makes work late. +const SETTLE_MARGIN: Duration = Duration::from_secs(30); + +/// How many targets to sample after the baseline to prove actions are firing at +/// all. +/// +/// A smoke test rather than a measurement: if the scheduling path is broken +/// every target is equally broken, and the point is to fail before spending the +/// fault window on a run that would report a clean account of nothing. +const FIRE_PROOF_SAMPLE: usize = 5; + +pub async fn run( + config: &ScenarioConfig, + manifest: &ChaosPrepManifest, + deps: &BenchmarkTestDependencies, + signals: &FaultSignals, + outputs: &OutputPaths, +) -> anyhow::Result { + let started_at = Utc::now(); + let workload_config = config.require_workload()?; + let scheduled_config = config.require_scheduled()?; + let storage_config = config.require_storage()?; + let history = OperationHistory::new(ScenarioCode::S16.as_str()); + let key_prefix = crate::chaos::scenario_key_prefix(ScenarioCode::S16); + + let user = manifest.user_context(deps); + let counters = user + .get_latest_component_revision(&manifest.counters_component_id) + .await?; + let promise = user + .get_latest_component_revision(&manifest.promise_component_id) + .await?; + + let ctx = WorkloadContext { + user, + counters, + promise, + history: history.clone(), + retry: config.retry_policy.clone(), + phase: PhaseMarker::new(Phase::Baseline), + key_prefix: key_prefix.clone(), + }; + + let scope = RunScope { + environment_id: manifest.environment_id.0.to_string(), + component_ids: vec![ + manifest.counters_component_id.0.to_string(), + manifest.promise_component_id.0.to_string(), + ], + agent_id_prefix: key_prefix.clone(), + idempotency_key_prefix: format!("{key_prefix}-"), + }; + + let targets: Vec = (0..scheduled_config.targets) + .map(|index| ctx.schedule_target_name(index)) + .collect(); + + let mut phases = Phases::default(); + let mut routing_snapshots = Vec::new(); + let mut fault_injected_at = None; + let mut fault_recovered_at = None; + let mut fault_id = None; + let mut fault_target_observed = None; + let mut attention_extra: Vec = Vec::new(); + + // Every early return below goes through `finish`, so an abort produces the + // same artifact shape as a completed run, with fewer phases filled in. + macro_rules! finish { + ($reason:expr, $records:expr, $readback:expr, $fires:expr, $outage:expr, $exactly:expr) => {{ + let mut summary = ChaosSummary::build( + $records, + $readback, + routing_snapshots.clone(), + fault_injected_at, + ); + summary.absorb(attention_extra.clone()); + if let Some(report) = $fires { + summary = summary.with_schedule_fires(report); + } + if let Some(report) = $outage { + summary = summary.with_storage_outage(report); + } + if let Some(report) = $exactly { + summary = summary.with_exactly_once(report); + } + let result = build_result( + config, + ScenarioOutcome { + started_at, + phases: phases.clone(), + fault_injected_at, + fault_recovered_at, + fault_id: fault_id.clone(), + fault_target_observed: fault_target_observed.clone(), + scope: scope.clone(), + summary, + termination_reason: $reason, + pinned_selection: None, + scheduled_selection: None, + promise_selection: None, + isolation_selection: None, + revert_selection: None, + delete_selection: None, + }, + ); + write_outputs(&result, &history, outputs)?; + return Ok(result); + }}; + } + + // ── Warm-up ───────────────────────────────────────────────────────────── + routing_snapshots.push(snapshot_routing(deps, "before-warmup").await); + attention_extra.push(wait_for_settled_routing(deps, &mut routing_snapshots).await); + info!( + "S16: warming {} schedule emitters and targets before the baseline", + targets.len() + ); + let warmed = scheduled::warm(&ctx, &targets).await; + info!("S16: warmed {warmed} agents, settling {WARMUP_SETTLE:?}"); + tokio::time::sleep(WARMUP_SETTLE).await; + + // ── Baseline ──────────────────────────────────────────────────────────── + info!( + "S16: baseline phase, mixed workload at {} ops/s plus {} schedule targets, for {:?}", + workload_config.rate_per_sec, + targets.len(), + config.phases.baseline() + ); + phases.baseline = Some(PhaseWindow::started(Utc::now())); + let mixed = workload::start(ctx.clone(), workload_config); + let schedules = scheduled::start(ctx.clone(), &targets, scheduled_config); + tokio::time::sleep(config.phases.baseline()).await; + routing_snapshots.push(snapshot_routing(deps, "before-fault").await); + if let Some(window) = phases.baseline.as_mut() { + window.end(Utc::now()); + } + + // Both handles have to be stopped on every exit path, and forgetting one + // leaves its emitters submitting into a history that has already been + // snapshotted. + macro_rules! stop_workloads { + () => {{ + mixed.stop().await; + schedules.stop().await; + }}; + } + + let baseline_operations = history.confirmed_in_phase(Phase::Baseline); + if baseline_operations == 0 { + // Taking a database away from a workload that never worked would + // measure nothing. Stop before touching the cluster. + warn!("S16: baseline produced no confirmed operations, aborting before injection"); + stop_workloads!(); + let records = history.snapshot(); + finish!( + TerminationReason::PlatformUnreachable { + detail: "no operation succeeded during the baseline phase".to_string(), + }, + &records, + Vec::new(), + None, + None, + None + ); + } + + // Registering is not firing. The scheduler is one of the three mechanisms + // this scenario is about, and a platform that accepted every registration + // and ran none of them would otherwise reach read-back and report a + // flawless account of a mechanism that never worked. + let sampled = sample_fire_count(&ctx, &targets).await; + if sampled == 0 { + warn!("S16: {baseline_operations} operations confirmed and no scheduled action has fired"); + stop_workloads!(); + let records = history.snapshot(); + finish!( + TerminationReason::StreamNeverSucceeded { + stream: Stream::Scheduled.to_string(), + }, + &records, + Vec::new(), + None, + None, + None + ); + } + info!( + "S16: baseline complete ({baseline_operations} confirmed ops, {sampled} fires across a \ + sample of {} targets), signalling readiness", + FIRE_PROOF_SAMPLE.min(targets.len()) + ); + + // ── Signal: ready for the fault ───────────────────────────────────────── + signals.write_baseline_ready(&BaselineReady { + scenario_code: ScenarioCode::S16.as_str().to_string(), + ready_at: Utc::now(), + baseline_operations, + // Nothing to aim. The partition is between every executor and one + // endpoint outside the cluster, so there is no pod for the driver to + // choose and no ownership to verify. + fault_target: None, + })?; + + // ── Fault ─────────────────────────────────────────────────────────────── + let injected = match signals.await_fault_injected(config.signal_timeout()).await { + Ok(injected) => injected, + Err(e) => { + warn!("S16: no fault-injected signal arrived: {e}"); + stop_workloads!(); + let records = history.snapshot(); + finish!( + signal_termination(&e), + &records, + Vec::new(), + None, + None, + None + ); + } + }; + info!( + "S16: fault {} ({} on {}) reported active at {}, {} is now unreachable from the executors", + injected.fault_id, + injected.kind, + injected.target, + injected.injected_at, + storage_config.endpoint + ); + fault_injected_at = Some(injected.injected_at); + fault_id = Some(injected.fault_id.clone()); + fault_target_observed = Some(injected.target.clone()); + ctx.phase.set(Phase::Fault); + phases.fault = Some(PhaseWindow::started(injected.injected_at)); + + let recovered = match signals.await_fault_recovered(config.signal_timeout()).await { + Ok(recovered) => recovered, + Err(e) => { + warn!("S16: no fault-recovered signal arrived: {e}"); + stop_workloads!(); + let records = history.snapshot(); + finish!( + signal_termination(&e), + &records, + Vec::new(), + None, + None, + None + ); + } + }; + info!( + "S16: fault cleared at {} ({})", + recovered.recovered_at, recovered.termination_reason + ); + fault_recovered_at = Some(recovered.recovered_at); + if let Some(window) = phases.fault.as_mut() { + window.end(recovered.recovered_at); + } + + // ── Recovery ──────────────────────────────────────────────────────────── + info!( + "S16: recovery phase, running for a further {:?}", + config.phases.recovery() + ); + ctx.phase.set(Phase::Recovery); + phases.recovery = Some(PhaseWindow::started(Utc::now())); + tokio::time::sleep(config.phases.recovery()).await; + + let skipped = schedules.skipped(); + // Stopping waits for in-flight operations to record themselves rather than + // cancelling them: an operation cancelled mid-flight is one the history + // cannot classify, and during a storage outage those are exactly the + // interesting ones. + stop_workloads!(); + + if let Some(window) = phases.recovery.as_mut() { + window.end(Utc::now()); + } + if skipped > 0 { + attention_extra.push(Note::attention(format!( + "S16 skipped {skipped} registration ticks because targets still had their budget of \ + {} in flight — the offered rate was clamped by the platform, so the phase counts \ + understate what the run intended to submit", + scheduled::MAX_IN_FLIGHT_PER_TARGET + ))); + } + routing_snapshots.push(snapshot_routing(deps, "after-recovery").await); + + // ── Read-back ─────────────────────────────────────────────────────────── + let settle = settle_before_readback(scheduled_config); + info!("S16: letting the last actions fall due and fire, {settle:?} before read-back"); + tokio::time::sleep(settle).await; + + let records = history.snapshot(); + let logs = scheduled::read_logs(&ctx, &targets).await; + // Archived alongside the operations, not just reduced into the report: the + // reduction is the part a later ticket is most likely to want to redo. + history.record_fire_logs(logs.clone()); + + let fault_window = fault_injected_at.map(|injected_at| FaultWindow { + injected_at, + recovered_at: fault_recovered_at, + }); + + let fires = ScheduleFireReport::build( + &records, + &logs, + scheduled_config.lead(), + fault_window, + // Empty on purpose: nothing was killed, so every target belongs to the + // report's `elsewhere` group and only its window axis carries meaning. + &BTreeSet::new(), + scheduled_config.lease_budget(), + ); + info!( + "S16: scheduled-fire account — {} registrations accepted, {} fired once, {} \ + inconclusive, {} unverifiable, {} findings", + fires.registrations_confirmed, + fires.fired_once, + fires.inconclusive, + fires.unverifiable, + fires.findings.len() + ); + + let outage = StorageOutageReport::build( + &records, + fault_window, + &storage_config.endpoint, + storage_config.outage_ceiling_percent, + storage_config.recovery_budget(), + ); + info!( + "S16: storage account — the workload held {:?}% of its baseline throughput while {} was \ + unreachable (ceiling {}%), {} findings", + outage.share_of_baseline_percent, + outage.endpoint, + outage.outage_ceiling_percent, + outage.findings.len() + ); + for finding in &outage.findings { + warn!("S16: {}: {}", finding.violation, finding.detail); + } + + let readback = read_back(&ctx, &records, workload_config, &logs).await; + + // The idempotency half of the account. A key that timed out while the + // database was gone may or may not have executed; re-invoking it under the + // same key says which, because a platform that stored the result replays it + // and one that did not runs the work again. + let before_probe = read_counters(&ctx, &records).await; + let probes = probe::probe_keys(&ctx, &records, Stream::Durable).await; + let after_probe = read_counters(&ctx, &records).await; + let exactly_once = ExactlyOnceReport::build( + &records, + &probes, + Stream::Durable, + &before_probe, + &after_probe, + ); + info!( + "S16: exactly-once account — {} keys checked, {} with a final result, {} recovered by \ + the probe, {} findings", + exactly_once.keys_checked, + exactly_once.keys_with_final_result, + exactly_once.keys_recovered_by_probe, + exactly_once.findings.len() + ); + + let reason = termination(&fires, &exactly_once, &records); + + finish!( + reason, + &records, + readback, + Some(fires), + Some(outage), + Some(exactly_once) + ); +} + +/// Why the run stopped. +/// +/// Kept apart from [`run`] so the precedence is testable without a cluster. The +/// order matters: a token-level fire violation and a duplicated key are both +/// statements about one named thing, and either is worth more to a reader than +/// "the workload never worked", which is a statement about the run rather than +/// about the platform. +fn termination( + fires: &ScheduleFireReport, + exactly_once: &ExactlyOnceReport, + records: &[OperationRecord], +) -> TerminationReason { + if fires.has_violations() { + return TerminationReason::ScheduledFireViolated { + findings: fires.findings.len() as u64, + first: fires + .findings + .first() + .map(|f| format!("{} on token {}", f.violation, f.token)) + .unwrap_or_default(), + }; + } + if let Some(reason) = exactly_once_termination(exactly_once) { + return reason; + } + if records.iter().all(|r| r.outcome != Outcome::Confirmed) { + return TerminationReason::StreamNeverSucceeded { + stream: Stream::Durable.to_string(), + }; + } + TerminationReason::Completed +} + +/// How long to wait after the workload stops before reading anything back. +/// +/// Same derivation as S10's, and load-bearing for the same reason: the last +/// registration falls due one `lead` after the loop stops, and a delayed action +/// can cost a lease budget on top of that. +fn settle_before_readback(config: &ScheduledConfig) -> Duration { + config.lead() + config.lease_budget() + SETTLE_MARGIN +} + +/// Reads durable state back for the streams that keep a count. +/// +/// The durable counters come from the mixed workload's own agents; the +/// scheduled counts come from the fire logs that were already read, because +/// re-reading `polls` would be a second round trip for a number the log already +/// carries. +async fn read_back( + ctx: &WorkloadContext, + records: &[OperationRecord], + config: &crate::chaos::WorkloadConfig, + logs: &[crate::chaos::history::TargetFireLog], +) -> Vec { + let mut readback = Vec::new(); + + for index in 0..config.durable_agents { + let agent = ctx.agent_name(Stream::Durable, index); + let scoped = records + .iter() + .filter(|r| r.stream == Stream::Durable && r.agent == agent); + if scoped.clone().next().is_none() { + continue; + } + let observed = workload::read_counter(ctx, &agent).await; + readback.extend(readback_for(Stream::Durable, &agent, scoped, observed)); + } + + for log in logs { + let scoped = records + .iter() + .filter(|r| r.stream == Stream::Scheduled && r.agent == log.agent); + let observed = match (log.polls, &log.error) { + (Some(polls), _) => Ok(polls), + (None, Some(error)) => Err(error.clone()), + (None, None) => Err(format!("target {} reported no poll count", log.agent)), + }; + readback.extend(readback_for( + Stream::Scheduled, + &log.agent, + scoped, + observed, + )); + } + + readback +} + +/// Reads the fire count of a few targets, to prove actions are firing at all. +async fn sample_fire_count(ctx: &WorkloadContext, targets: &[String]) -> u64 { + let mut total = 0u64; + for target in targets.iter().take(FIRE_PROOF_SAMPLE) { + match workload::read_polls(ctx, target).await { + Ok(polls) => total += polls, + Err(e) => warn!("S16: could not sample fires on {target}: {e}"), + } + } + total +} + +/// Whether the storage account found the outage never landed. +/// +/// Exposed for the runbook's benefit as much as the tests': "did this run test +/// anything" is the first question anyone asks of an S16 artifact, and it +/// should not need a reader to scan the findings list by eye. +pub fn outage_was_observed(report: &StorageOutageReport) -> bool { + !report + .findings + .iter() + .any(|f| f.violation == OutageViolation::OutageNotObserved) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::chaos::history::{AttemptRecord, FireRecord, TargetFireLog}; + use crate::chaos::outage::OutageFinding; + use chrono::{DateTime, TimeDelta}; + use test_r::test; + + const TARGET: &str = "chaos-s16-schedule-target-0000"; + const LEAD: Duration = Duration::from_secs(10); + const BUDGET: Duration = Duration::from_secs(240); + + fn t0() -> DateTime { + DateTime::parse_from_rfc3339("2026-08-25T12:00:00Z") + .unwrap() + .with_timezone(&Utc) + } + + fn registration(token: &str, outcome: Outcome) -> OperationRecord { + OperationRecord { + op_id: 0, + stream: Stream::Scheduled, + phase: Phase::Baseline, + agent: TARGET.to_string(), + method: "schedule_fire_at".to_string(), + idempotency_key: token.to_string(), + submitted_at: t0(), + completed_at: Some(t0() + TimeDelta::milliseconds(20)), + attempts: 1, + outcome, + duration_ms: 20, + returned_value: None, + first_attempt_value: None, + error: None, + error_class: None, + attempt_log: vec![AttemptRecord { + attempt: 1, + started_at: t0(), + duration_ms: 20, + returned_value: None, + succeeded: outcome == Outcome::Confirmed, + error_class: None, + error: None, + }], + } + } + + fn log(fired: &[&str]) -> TargetFireLog { + TargetFireLog { + agent: TARGET.to_string(), + polls: Some(fired.len() as u64), + fires: fired + .iter() + .map(|token| FireRecord { + token: (*token).to_string(), + scheduled_at: t0() + TimeDelta::seconds(10), + observed_at: t0() + TimeDelta::seconds(11), + }) + .collect(), + error: None, + } + } + + fn fires(records: &[OperationRecord], logs: &[TargetFireLog]) -> ScheduleFireReport { + ScheduleFireReport::build(records, logs, LEAD, None, &BTreeSet::new(), BUDGET) + } + + /// The empty account: nothing registered, nothing fired, nothing to say. + fn clean_exactly_once() -> ExactlyOnceReport { + ExactlyOnceReport::build( + &[], + &[], + Stream::Durable, + &std::collections::BTreeMap::new(), + &std::collections::BTreeMap::new(), + ) + } + + /// A registration the platform accepted whose action never ran is a + /// statement about one named token, and it outranks every aggregate. + #[test] + fn a_token_that_never_fired_fails_the_run() { + let records = vec![registration("token-a", Outcome::Confirmed)]; + let reason = termination( + &fires(&records, &[log(&[])]), + &clean_exactly_once(), + &records, + ); + + assert!( + matches!(reason, TerminationReason::ScheduledFireViolated { .. }), + "expected a fire violation, got {reason:?}" + ); + } + + /// A fire violation outranks "the workload never worked", because it says + /// something about the platform and the other says something about the run. + #[test] + fn a_fire_violation_outranks_a_workload_that_never_confirmed_anything() { + // Rejected registrations mean nothing confirmed anywhere, and a fire + // that happened anyway is the sharpest thing this suite can find. + let records = vec![registration("token-a", Outcome::Rejected)]; + let reason = termination( + &fires(&records, &[log(&["token-a"])]), + &clean_exactly_once(), + &records, + ); + + assert!( + matches!(reason, TerminationReason::ScheduledFireViolated { .. }), + "expected a fire violation, got {reason:?}" + ); + } + + /// A run where nothing confirmed at all says nothing about resilience, so + /// it fails rather than reporting a clean account of a workload that never + /// worked. + #[test] + fn a_workload_that_never_confirmed_anything_fails_the_run() { + let records = vec![registration("token-a", Outcome::Indeterminate)]; + let reason = termination( + &fires(&records, &[log(&["token-a"])]), + &clean_exactly_once(), + &records, + ); + + assert!( + matches!(reason, TerminationReason::StreamNeverSucceeded { .. }), + "expected a never-succeeded reason, got {reason:?}" + ); + } + + /// The healthy shape completes, and the storage account's own findings do + /// not change that: an outage that failed to land is loud in `attention` + /// and deliberately not fatal. + #[test] + fn a_healthy_run_completes() { + let records = vec![registration("token-a", Outcome::Confirmed)]; + let reason = termination( + &fires(&records, &[log(&["token-a"])]), + &clean_exactly_once(), + &records, + ); + + assert_eq!(reason, TerminationReason::Completed); + } + + /// The question every reader asks an S16 artifact first, answered without + /// scanning the findings list by eye. + #[test] + fn outage_observed_reads_the_verdict_off_the_findings() { + let mut report = + StorageOutageReport::build(&[], None, "db.example", 15.0, Duration::from_secs(120)); + assert!(outage_was_observed(&report)); + + report.findings.push(OutageFinding { + violation: OutageViolation::StreamNeverRecovered, + stream: Some(Stream::Durable), + detail: "unrelated".to_string(), + }); + assert!( + outage_was_observed(&report), + "only the outage verdict answers this question" + ); + + report.findings.push(OutageFinding { + violation: OutageViolation::OutageNotObserved, + stream: None, + detail: "kept working".to_string(), + }); + assert!(!outage_was_observed(&report)); + } +} diff --git a/integration-tests/src/chaos/split.rs b/integration-tests/src/chaos/split.rs index cd811fd1ef..357162a299 100644 --- a/integration-tests/src/chaos/split.rs +++ b/integration-tests/src/chaos/split.rs @@ -121,6 +121,68 @@ impl std::fmt::Display for Window { } } +/// When a window began, for the windows that have a fixed start. +/// +/// The fault's own boundaries come from the workflow's timestamps, which is +/// what makes them comparable across runs. The baseline has no boundary of its +/// own, so it starts at the first operation the run offered. +pub fn window_start( + window: Window, + fault: Option, + first_submitted: Option>, +) -> Option> { + match (window, fault) { + (Window::BeforeFault, Some(_)) => first_submitted, + (Window::DuringFault, Some(w)) => Some(w.injected_at), + (Window::AfterFault, Some(w)) => w.recovered_at, + _ => None, + } +} + +/// How long a window lasted, in seconds. +/// +/// The fault's own windows come from the workflow's timestamps, which is what +/// makes them comparable across runs. The two open-ended ones are bounded by +/// the workload instead: the baseline starts at the first operation offered, +/// and recovery ends at the last one that came back. +pub fn window_secs( + window: Window, + fault: Option, + first_submitted: Option>, + last_completed: Option>, +) -> f64 { + let seconds = |from: DateTime, to: DateTime| { + (to - from).num_milliseconds().max(0) as f64 / 1000.0 + }; + match (window, fault) { + (Window::BeforeFault, Some(w)) => first_submitted + .map(|first| seconds(first, w.injected_at)) + .unwrap_or(0.0), + (Window::DuringFault, Some(w)) => match (w.recovered_at, last_completed) { + (Some(recovered), _) => seconds(w.injected_at, recovered), + // A run that never saw the heal: the fault ran to whatever the last + // operation saw, which is the most that can be claimed. + (None, Some(last)) => seconds(w.injected_at, last), + (None, None) => 0.0, + }, + ( + Window::AfterFault, + Some(FaultWindow { + recovered_at: Some(recovered), + .. + }), + ) => last_completed + .map(|last| seconds(recovered, last)) + .unwrap_or(0.0), + _ => 0.0, + } +} + +/// Two decimal places, for the rates and shares that end up in a result. +pub fn round2(value: f64) -> f64 { + (value * 100.0).round() / 100.0 +} + /// The executor the fault will be aimed at, and how the agents divide around it. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] diff --git a/integration-tests/src/chaos/summary.rs b/integration-tests/src/chaos/summary.rs index e1a6a8021d..6dd277437e 100644 --- a/integration-tests/src/chaos/summary.rs +++ b/integration-tests/src/chaos/summary.rs @@ -50,6 +50,7 @@ use crate::chaos::fires::ScheduleFireReport; use crate::chaos::history::{Outcome, Phase, Stream}; +use crate::chaos::outage::StorageOutageReport; use crate::chaos::ownership::OwnershipSample; use crate::chaos::probe::KeyProbe; use crate::chaos::reachability::ReachabilityReport; @@ -593,6 +594,11 @@ pub struct ChaosSummary { /// back. Absent for scenarios that do not. #[serde(default, skip_serializing_if = "Option::is_none")] pub rollback: Option, + /// The storage-outage account, for scenarios that take a storage dependency + /// away from every executor at once. Absent for scenarios that do not, for + /// the same reason as `scheduleFires`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub storage_outage: Option, /// Shard-ownership samples, in the order they were taken. Empty for /// scenarios that do not sample executor assignments. /// @@ -728,6 +734,7 @@ impl ChaosSummary { truncation: None, resurrection: None, rollback: None, + storage_outage: None, ownership: Vec::new(), attention, notes: Vec::new(), @@ -841,6 +848,20 @@ impl ChaosSummary { self } + /// Attaches the storage-outage account and hoists everything it wants a + /// human to see into [`Self::attention`]. + /// + /// Same split as the others. The line worth calling out is the + /// outage-not-observed one: a partition that failed to take hold leaves + /// every cell underneath it describing an undisturbed cluster, and that has + /// to read as "this run tested nothing" rather than as a pass. + pub fn with_storage_outage(mut self, report: StorageOutageReport) -> Self { + self.attention.extend(report.attention_lines()); + self.notes.extend(report.note_lines()); + self.storage_outage = Some(report); + self + } + /// Attaches the shard-ownership samples and hoists their findings into /// [`Self::attention`]. /// From 27046f24ac5761e0e38783287179b2296b1d03a5 Mon Sep 17 00:00:00 2001 From: Kaur Matas <33095685+kmatasfp@users.noreply.github.com> Date: Tue, 25 Aug 2026 12:45:44 -0700 Subject: [PATCH 25/40] Drop the unused S16 outage-observed helper --- integration-tests/src/chaos/scenarios/s16.rs | 50 +++++++------------- 1 file changed, 18 insertions(+), 32 deletions(-) diff --git a/integration-tests/src/chaos/scenarios/s16.rs b/integration-tests/src/chaos/scenarios/s16.rs index 5bb1adfc33..83b553268d 100644 --- a/integration-tests/src/chaos/scenarios/s16.rs +++ b/integration-tests/src/chaos/scenarios/s16.rs @@ -92,7 +92,7 @@ use crate::chaos::fires::{FaultWindow, ScheduleFireReport}; use crate::chaos::history::{OperationHistory, OperationRecord, Outcome, Phase, Stream}; -use crate::chaos::outage::{OutageViolation, StorageOutageReport}; +use crate::chaos::outage::StorageOutageReport; use crate::chaos::prep::ChaosPrepManifest; use crate::chaos::probe; use crate::chaos::result::{ChaosResult, PhaseWindow, Phases, RunScope}; @@ -603,23 +603,11 @@ async fn sample_fire_count(ctx: &WorkloadContext, targets: &[String]) -> u64 { total } -/// Whether the storage account found the outage never landed. -/// -/// Exposed for the runbook's benefit as much as the tests': "did this run test -/// anything" is the first question anyone asks of an S16 artifact, and it -/// should not need a reader to scan the findings list by eye. -pub fn outage_was_observed(report: &StorageOutageReport) -> bool { - !report - .findings - .iter() - .any(|f| f.violation == OutageViolation::OutageNotObserved) -} - #[cfg(test)] mod tests { use super::*; use crate::chaos::history::{AttemptRecord, FireRecord, TargetFireLog}; - use crate::chaos::outage::OutageFinding; + use crate::chaos::outage::{OutageFinding, OutageViolation}; use chrono::{DateTime, TimeDelta}; use test_r::test; @@ -762,29 +750,27 @@ mod tests { assert_eq!(reason, TerminationReason::Completed); } - /// The question every reader asks an S16 artifact first, answered without - /// scanning the findings list by eye. + /// The storage account's findings never reach the termination reason. An + /// outage that failed to land is the loudest thing S16 can report and it is + /// deliberately not fatal: turning the job red would say the platform did + /// something wrong, and what actually went wrong is the experiment. #[test] - fn outage_observed_reads_the_verdict_off_the_findings() { - let mut report = + fn a_storage_finding_does_not_change_the_termination_reason() { + let mut outage = StorageOutageReport::build(&[], None, "db.example", 15.0, Duration::from_secs(120)); - assert!(outage_was_observed(&report)); - - report.findings.push(OutageFinding { - violation: OutageViolation::StreamNeverRecovered, - stream: Some(Stream::Durable), - detail: "unrelated".to_string(), - }); - assert!( - outage_was_observed(&report), - "only the outage verdict answers this question" - ); - - report.findings.push(OutageFinding { + outage.findings.push(OutageFinding { violation: OutageViolation::OutageNotObserved, stream: None, detail: "kept working".to_string(), }); - assert!(!outage_was_observed(&report)); + assert!(outage.has_findings()); + + let records = vec![registration("token-a", Outcome::Confirmed)]; + let reason = termination( + &fires(&records, &[log(&["token-a"])]), + &clean_exactly_once(), + &records, + ); + assert_eq!(reason, TerminationReason::Completed); } } From 410354a01246ac2f811d2acef5a5ce66cffda26f Mon Sep 17 00:00:00 2001 From: Kaur Matas <33095685+kmatasfp@users.noreply.github.com> Date: Tue, 25 Aug 2026 14:02:46 -0700 Subject: [PATCH 26/40] Attribute chaos outage throughput by completion rather than submission --- .../chaos_suites/cloud-chaos.yaml | 14 +- integration-tests/src/chaos/mod.rs | 28 +++ integration-tests/src/chaos/outage.rs | 166 +++++++++++++++--- integration-tests/src/chaos/split.rs | 20 +++ 4 files changed, 201 insertions(+), 27 deletions(-) diff --git a/integration-tests/chaos_suites/cloud-chaos.yaml b/integration-tests/chaos_suites/cloud-chaos.yaml index a2471ef439..4b975c7c8a 100644 --- a/integration-tests/chaos_suites/cloud-chaos.yaml +++ b/integration-tests/chaos_suites/cloud-chaos.yaml @@ -175,7 +175,11 @@ scenarios: # executors that stay connected keep getting told the truth, and the # disagreement between the two groups is the whole experiment. targetCount: 1 - durationSecs: 120 + # Must equal phases.faultSecs; see the note on S16. The joiner partition + # this scenario applies mid-fault is already sized from faultSecs, so a + # shorter value here also made the two partitions lapse at different + # times. + durationSecs: 180 phases: baselineSecs: 300 @@ -883,7 +887,13 @@ scenarios: # outage that only some executors could see would be a routing fault # wearing a database's clothes, and S3 already covers that shape. mode: all - durationSecs: 120 + # Must equal phases.faultSecs. Chaos Mesh recovers a NetworkChaos on its + # own `duration`, while the workflow holds the phase for faultSecs and + # only then deletes the object. Setting this lower lifts the partition + # early and leaves the tail of the fault phase measuring a healed + # cluster, which is how the first S16 run came to report 16.87% of + # baseline throughput during a total outage. + durationSecs: 180 phases: # Long enough for cold starts, component loading and route warm-up to diff --git a/integration-tests/src/chaos/mod.rs b/integration-tests/src/chaos/mod.rs index 74050a4f91..5aecac027c 100644 --- a/integration-tests/src/chaos/mod.rs +++ b/integration-tests/src/chaos/mod.rs @@ -1084,6 +1084,34 @@ mod tests { ); } + /// A `pod-kill` is instantaneous and its `duration` only governs how long + /// the object lingers, so those two numbers are free to differ. A + /// `network-partition` is not: Chaos Mesh takes the iptables rules down on + /// the object's own `duration`, while the workflow holds the phase for + /// `faultSecs` and only then deletes it. A shorter `durationSecs` therefore + /// heals the cluster part-way through the window the driver is still + /// attributing to the fault, and every during-fault number is measured + /// across a mix of the two states. + #[test] + fn a_partition_lasts_at_least_as_long_as_the_fault_phase_it_is_measured_over() { + let suite = ChaosSuite::load(suite_path()).unwrap(); + for entry in &suite.scenarios { + if entry.fault.kind != "network-partition" { + continue; + } + assert!( + entry.fault.duration_secs >= entry.phases.fault_secs, + "{}: fault.durationSecs ({}) is shorter than phases.faultSecs ({}), \ + so the partition lifts {}s before the fault phase ends and the rest \ + of that phase measures a healed cluster", + entry.code, + entry.fault.duration_secs, + entry.phases.fault_secs, + entry.phases.fault_secs - entry.fault.duration_secs, + ); + } + } + #[test] fn every_suite_entry_resolves_to_an_implemented_scenario() { let suite = ChaosSuite::load(suite_path()).unwrap(); diff --git a/integration-tests/src/chaos/outage.rs b/integration-tests/src/chaos/outage.rs index 021ca6e0c7..3b2e838c2d 100644 --- a/integration-tests/src/chaos/outage.rs +++ b/integration-tests/src/chaos/outage.rs @@ -75,7 +75,7 @@ use crate::chaos::errors::ErrorClass; use crate::chaos::history::{OperationRecord, Outcome, Stream}; -use crate::chaos::split::{FaultWindow, Window, round2, window_secs, window_start}; +use crate::chaos::split::{FaultWindow, Window, round2, window_end, window_secs, window_start}; use crate::chaos::summary::LatencyStats; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; @@ -131,31 +131,70 @@ pub struct StreamThroughputCell { /// window. Below the stream's pool size means emitters were stalled across /// the whole window rather than merely slowed. pub agents_active: usize, + /// Operations *offered* in this window, and how they eventually ended up. + /// Attributed by submission time, so an operation counted here may not have + /// been answered until a later window. pub submitted: u64, pub confirmed: u64, pub rejected: u64, pub indeterminate: u64, + /// Operations *answered* in this window, whenever they were offered. + /// + /// This is the one that says whether the platform was serving, and it is + /// deliberately not `confirmed`: during an outage the two differ by exactly + /// the work that was accepted while the storage was gone and answered only + /// once it came back. + pub served: u64, /// Attempts that hit the client's attempt timeout rather than answering. pub attempts_timed_out: u64, pub window_secs: f64, - pub confirmed_per_sec: f64, + pub served_per_sec: f64, /// This cell's rate against the same stream's own before-fault rate. `None` /// for the before-fault cell itself, and for a stream that never had a /// baseline to compare against. #[serde(default, skip_serializing_if = "Option::is_none")] pub share_of_baseline_percent: Option, - /// How long into this window the stream waited before offering anything. + /// The longest the stream answered nothing at all, anywhere in this window. /// /// The number that stops a small non-zero during-fault rate being read as - /// residual service: an operation submitted just before the heal can confirm - /// just after it and still be counted here. `quietMs` next to `windowSecs` - /// says whether the stream was serving throughout or answered only at the - /// end, and it needs no threshold to do it. + /// residual service. Measured against the window's own edges, so a stream + /// that fell silent at the start or stayed silent to the end is caught by + /// it too. A `quietMs` close to `windowSecs` is a total outage however the + /// rate arithmetic came out, and it needs no threshold to say so. #[serde(default, skip_serializing_if = "Option::is_none")] pub quiet_ms: Option, pub latency: LatencyStats, } +/// The longest stretch of a window in which nothing was answered. +/// +/// The window's own edges are the first and last comparison points, which is +/// what separates "answered steadily but slowly" from "answered nothing for two +/// minutes and then caught up in a burst". Those two produce the same count and +/// the same rate; only this tells them apart. +fn longest_silence_ms( + served_at: &[DateTime], + start: Option>, + end: Option>, +) -> Option { + let (start, end) = (start?, end?); + let mut marks: Vec> = served_at + .iter() + .copied() + .filter(|at| *at >= start && *at <= end) + .collect(); + marks.sort_unstable(); + marks.insert(0, start); + marks.push(end); + Some( + marks + .windows(2) + .map(|pair| (pair[1] - pair[0]).num_milliseconds().max(0) as u64) + .max() + .unwrap_or(0), + ) +} + /// The operations the outage began underneath. /// /// These were submitted before the storage went away and were still running @@ -247,13 +286,18 @@ pub struct StorageOutageReport { #[derive(Default)] struct Tally { agents: BTreeSet, - first_submitted: Option>, submitted: u64, confirmed: u64, rejected: u64, indeterminate: u64, attempts_timed_out: u64, durations: Vec, + /// When this stream actually answered inside this window, sorted later. + /// + /// Keyed on the window a confirmation *landed* in rather than the one its + /// operation was offered in, which is the only way either of the numbers + /// derived from it means what it says. + served_at: Vec>, } impl StorageOutageReport { @@ -280,10 +324,6 @@ impl StorageOutageReport { let tally = tallies.entry((record.stream, window)).or_default(); tally.agents.insert(record.agent.clone()); - tally.first_submitted = Some(match tally.first_submitted { - Some(at) if at <= record.submitted_at => at, - _ => record.submitted_at, - }); tally.submitted += 1; match record.outcome { Outcome::Confirmed => { @@ -295,6 +335,21 @@ impl StorageOutageReport { } tally.attempts_timed_out += record.attempts_timed_out(); + // Answered work is filed under the window it was answered in, which + // is usually but not always the one it was offered in. An operation + // offered during the outage and answered after the heal is service + // the fault window did not get, and counting it there is how a total + // outage reads as partial service. + if record.outcome == Outcome::Confirmed + && let Some(completed) = record.completed_at + { + tallies + .entry((record.stream, Window::of(completed, fault))) + .or_default() + .served_at + .push(completed); + } + first_submitted = Some(match first_submitted { Some(at) if at <= record.submitted_at => at, _ => record.submitted_at, @@ -314,28 +369,32 @@ impl StorageOutageReport { let mut cells: Vec = Vec::new(); for ((stream, window), tally) in &tallies { let secs = window_secs(*window, fault, first_submitted, last_completed); + let served = tally.served_at.len() as u64; let rate = if secs > 0.0 { - tally.confirmed as f64 / secs + served as f64 / secs } else { 0.0 }; if *window == Window::BeforeFault { baseline_rate.insert(*stream, rate); } - let quiet_ms = window_start(*window, fault, first_submitted) - .zip(tally.first_submitted) - .map(|(start, first)| (first - start).num_milliseconds().max(0) as u64); + let quiet_ms = longest_silence_ms( + &tally.served_at, + window_start(*window, fault, first_submitted), + window_end(*window, fault, last_completed), + ); cells.push(StreamThroughputCell { stream: *stream, window: *window, agents_active: tally.agents.len(), submitted: tally.submitted, confirmed: tally.confirmed, + served, rejected: tally.rejected, indeterminate: tally.indeterminate, attempts_timed_out: tally.attempts_timed_out, window_secs: round2(secs), - confirmed_per_sec: round2(rate), + served_per_sec: round2(rate), share_of_baseline_percent: None, quiet_ms, latency: LatencyStats::from_durations(tally.durations.clone()), @@ -349,8 +408,7 @@ impl StorageOutageReport { if let Some(base) = baseline_rate.get(&cell.stream).copied() && base > 0.0 { - cell.share_of_baseline_percent = - Some(round2(cell.confirmed_per_sec / base * 100.0)); + cell.share_of_baseline_percent = Some(round2(cell.served_per_sec / base * 100.0)); } } cells.sort_by_key(|c| (c.stream, c.window)); @@ -386,15 +444,15 @@ impl StorageOutageReport { return; } let total = |window: Window| -> f64 { - let confirmed: u64 = self + let served: u64 = self .cells .iter() .filter(|c| c.window == window) - .map(|c| c.confirmed) + .map(|c| c.served) .sum(); let secs = window_secs(window, fault, first_submitted, last_completed); if secs > 0.0 { - confirmed as f64 / secs + served as f64 / secs } else { 0.0 } @@ -915,13 +973,71 @@ mod tests { #[test] fn the_during_fault_cell_says_how_long_the_stream_stayed_quiet() { let mut records = history(0, true); - // One confirmation, 170s into the 180s outage. + // One confirmation, answered 170s into the 180s outage. records.push(op(Stream::Durable, 170, Outcome::Confirmed)); let report = build(&records); let during = report.cell(Stream::Durable, Window::DuringFault).unwrap(); - assert_eq!(during.confirmed, 1); - assert_eq!(during.quiet_ms, Some(170_000)); + assert_eq!(during.served, 1); + // Silent from the cut until that one answer, which is nearly the whole + // window — not the 10s that remained after it. + assert_eq!(during.quiet_ms, Some(170_020)); + } + + /// A stream answering steadily right through the window is the shape the + /// quiet number has to be able to tell apart from the one above. Both have + /// a non-zero during-fault count; only the silence separates them. + #[test] + fn a_stream_answering_throughout_the_window_is_never_quiet_for_long() { + let report = build(&history(180, true)); + + let during = report.cell(Stream::Durable, Window::DuringFault).unwrap(); + assert_eq!(during.served, 180); + assert!( + during.quiet_ms.is_some_and(|ms| ms < 2_000), + "a stream answering once a second should show no real silence, got {:?}", + during.quiet_ms + ); + } + + /// The regression the first S16 run turned up. + /// + /// The workload keeps offering work all through the outage, so anything + /// derived from submission times is busy no matter what the platform is + /// doing. Every one of these operations is offered during the fault and + /// answered only after the heal: the fault window served nothing, and the + /// report has to say so rather than crediting the window with work it did + /// not do. + #[test] + fn work_offered_during_the_outage_and_answered_after_it_is_not_during_fault_service() { + let mut records = history(0, true); + for second in 0..120 { + let mut record = op(Stream::Durable, second, Outcome::Confirmed); + // Offered inside the outage, answered once the storage returned. + record.completed_at = Some(t0() + TimeDelta::seconds(181)); + record.duration_ms = (181 - second) as u64 * 1_000; + records.push(record); + } + let report = build(&records); + + let during = report.cell(Stream::Durable, Window::DuringFault).unwrap(); + assert_eq!(during.submitted, 120, "they were offered during the fault"); + assert_eq!(during.confirmed, 120, "and they did all eventually confirm"); + assert_eq!( + during.served, 0, + "but none of it was served during the fault" + ); + assert_eq!(during.served_per_sec, 0.0); + assert_eq!( + during.quiet_ms, + Some(180_000), + "silent for the whole window" + ); + assert!( + report.findings.is_empty(), + "a total outage must not be reported as one that never landed, got {:?}", + report.findings + ); } /// The operations at risk. They were submitted before the cut and are diff --git a/integration-tests/src/chaos/split.rs b/integration-tests/src/chaos/split.rs index 357162a299..e156b5cd9e 100644 --- a/integration-tests/src/chaos/split.rs +++ b/integration-tests/src/chaos/split.rs @@ -139,6 +139,26 @@ pub fn window_start( } } +/// When a window closed. +/// +/// The mirror of [`window_start`], and needed wherever a gap has to be measured +/// against the window's own edges rather than against the first and last thing +/// that happened inside it: a stream that went silent for the whole back half +/// of a window has no later timestamp to compare against, so the edge is the +/// only thing that can show it. +pub fn window_end( + window: Window, + fault: Option, + last_completed: Option>, +) -> Option> { + match (window, fault) { + (Window::BeforeFault, Some(w)) => Some(w.injected_at), + (Window::DuringFault, Some(w)) => w.recovered_at.or(last_completed), + (Window::AfterFault, Some(_)) => last_completed, + _ => None, + } +} + /// How long a window lasted, in seconds. /// /// The fault's own windows come from the workflow's timestamps, which is what From 93e5bca66f54c2c20f6d83602845732387df5239 Mon Sep 17 00:00:00 2001 From: Kaur Matas <33095685+kmatasfp@users.noreply.github.com> Date: Tue, 25 Aug 2026 14:12:27 -0700 Subject: [PATCH 27/40] Attribute S3 reachability throughput by completion rather than submission --- integration-tests/src/chaos/outage.rs | 33 +---- integration-tests/src/chaos/reachability.rs | 130 +++++++++++++++----- integration-tests/src/chaos/split.rs | 29 +++++ 3 files changed, 129 insertions(+), 63 deletions(-) diff --git a/integration-tests/src/chaos/outage.rs b/integration-tests/src/chaos/outage.rs index 3b2e838c2d..ce9f9fa4d7 100644 --- a/integration-tests/src/chaos/outage.rs +++ b/integration-tests/src/chaos/outage.rs @@ -75,7 +75,9 @@ use crate::chaos::errors::ErrorClass; use crate::chaos::history::{OperationRecord, Outcome, Stream}; -use crate::chaos::split::{FaultWindow, Window, round2, window_end, window_secs, window_start}; +use crate::chaos::split::{ + FaultWindow, Window, longest_silence_ms, round2, window_end, window_secs, window_start, +}; use crate::chaos::summary::LatencyStats; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; @@ -166,35 +168,6 @@ pub struct StreamThroughputCell { pub latency: LatencyStats, } -/// The longest stretch of a window in which nothing was answered. -/// -/// The window's own edges are the first and last comparison points, which is -/// what separates "answered steadily but slowly" from "answered nothing for two -/// minutes and then caught up in a burst". Those two produce the same count and -/// the same rate; only this tells them apart. -fn longest_silence_ms( - served_at: &[DateTime], - start: Option>, - end: Option>, -) -> Option { - let (start, end) = (start?, end?); - let mut marks: Vec> = served_at - .iter() - .copied() - .filter(|at| *at >= start && *at <= end) - .collect(); - marks.sort_unstable(); - marks.insert(0, start); - marks.push(end); - Some( - marks - .windows(2) - .map(|pair| (pair[1] - pair[0]).num_milliseconds().max(0) as u64) - .max() - .unwrap_or(0), - ) -} - /// The operations the outage began underneath. /// /// These were submitted before the storage went away and were still running diff --git a/integration-tests/src/chaos/reachability.rs b/integration-tests/src/chaos/reachability.rs index e5996ad5e7..be1765d5e5 100644 --- a/integration-tests/src/chaos/reachability.rs +++ b/integration-tests/src/chaos/reachability.rs @@ -47,7 +47,8 @@ use crate::chaos::history::{OperationRecord, Outcome, Stream}; use crate::chaos::split::{ - FaultWindow, Group, PodSplit, Window, round2, window_secs, window_start, + FaultWindow, Group, PodSplit, Window, longest_silence_ms, round2, window_end, window_secs, + window_start, }; use crate::chaos::summary::LatencyStats; use chrono::{DateTime, Utc}; @@ -110,32 +111,42 @@ pub struct ThroughputCell { /// Below the group's size means emitters were stalled across the whole /// window rather than merely slowed. pub agents_active: usize, + /// Operations *offered* in this window, and how they eventually ended up. + /// Attributed by submission time, so an operation counted here may not have + /// been answered until a later window. pub submitted: u64, pub confirmed: u64, pub rejected: u64, pub indeterminate: u64, + /// Operations *answered* in this window, whenever they were offered. + /// + /// This is the one that says whether the group was being served, and it is + /// deliberately not `confirmed`: while an executor is unreachable the two + /// differ by exactly the work that was accepted and answered only once the + /// partition came down. + pub served: u64, /// Attempts that hit the client's attempt timeout rather than answering. /// The pending-then-timeout behaviour the scenario exists to make visible. pub attempts_timed_out: u64, pub window_secs: f64, - pub confirmed_per_sec: f64, + pub served_per_sec: f64, /// This cell's rate against the same group's own before-fault rate. `None` /// for the before-fault cell itself, and for a group that never had a /// baseline to compare against. #[serde(default, skip_serializing_if = "Option::is_none")] pub share_of_baseline_percent: Option, - /// How long into this window the group waited before offering anything. + /// The longest the group was answered nothing at all, anywhere in this + /// window. /// /// The number that stops a small non-zero rate being read as residual - /// service. An emitter holds one operation at a time, so a group whose - /// executor is unreachable offers nothing at all until something finally - /// answers it: a during-fault cell can therefore show a handful of - /// confirmations that all arrived in the last seconds of the window, once - /// the fault was already coming down. `quietMs` next to `windowSecs` says - /// which of the two happened, and it needs no threshold to do it. + /// service. A during-fault cell can show a handful of confirmations that + /// all arrived in the last seconds of the window, once the fault was + /// already coming down. Measured against the window's own edges, so a group + /// that fell silent at the start or stayed silent to the end is caught by + /// it too, and a `quietMs` close to `windowSecs` is a total outage however + /// the rate arithmetic came out. /// - /// `None` when the window has no fixed start, or when the group offered - /// nothing in it at all. + /// `None` when the window has no fixed bounds to measure against. #[serde(default, skip_serializing_if = "Option::is_none")] pub quiet_ms: Option, pub latency: LatencyStats, @@ -225,14 +236,18 @@ pub struct ReachabilityReport { #[derive(Default)] struct Tally { agents: BTreeSet, - /// Earliest submission in this cell, for [`ThroughputCell::quiet_ms`]. - first_submitted: Option>, submitted: u64, confirmed: u64, rejected: u64, indeterminate: u64, attempts_timed_out: u64, durations: Vec, + /// When this group actually answered inside this window, sorted later. + /// + /// Keyed on the window a confirmation *landed* in rather than the one its + /// operation was offered in, which is the only way either of the numbers + /// derived from it means what it says. + served_at: Vec>, } impl ReachabilityReport { @@ -265,10 +280,6 @@ impl ReachabilityReport { let tally = tallies.entry((group, window)).or_default(); tally.agents.insert(record.agent.clone()); - tally.first_submitted = Some(match tally.first_submitted { - Some(at) if at <= record.submitted_at => at, - _ => record.submitted_at, - }); tally.submitted += 1; match record.outcome { Outcome::Confirmed => { @@ -280,6 +291,21 @@ impl ReachabilityReport { } tally.attempts_timed_out += record.attempts_timed_out(); + // Answered work is filed under the window it was answered in, which + // is usually but not always the one it was offered in. An operation + // offered while the executor was unreachable and answered once the + // partition came down is not service the fault window delivered, + // and counting it there is how a total outage reads as partial. + if record.outcome == Outcome::Confirmed + && let Some(completed) = record.completed_at + { + tallies + .entry((group, Window::of(completed, fault))) + .or_default() + .served_at + .push(completed); + } + first_submitted = Some(match first_submitted { Some(at) if at <= record.submitted_at => at, _ => record.submitted_at, @@ -299,28 +325,32 @@ impl ReachabilityReport { let mut cells: Vec = Vec::new(); for ((group, window), tally) in &tallies { let secs = window_secs(*window, fault, first_submitted, last_completed); + let served = tally.served_at.len() as u64; let rate = if secs > 0.0 { - tally.confirmed as f64 / secs + served as f64 / secs } else { 0.0 }; if *window == Window::BeforeFault { baseline_rate.insert(*group, rate); } - let quiet_ms = window_start(*window, fault, first_submitted) - .zip(tally.first_submitted) - .map(|(start, first)| (first - start).num_milliseconds().max(0) as u64); + let quiet_ms = longest_silence_ms( + &tally.served_at, + window_start(*window, fault, first_submitted), + window_end(*window, fault, last_completed), + ); cells.push(ThroughputCell { group: *group, window: *window, agents_active: tally.agents.len(), submitted: tally.submitted, confirmed: tally.confirmed, + served, rejected: tally.rejected, indeterminate: tally.indeterminate, attempts_timed_out: tally.attempts_timed_out, window_secs: round2(secs), - confirmed_per_sec: round2(rate), + served_per_sec: round2(rate), share_of_baseline_percent: None, quiet_ms, latency: LatencyStats::from_durations(tally.durations.clone()), @@ -333,7 +363,7 @@ impl ReachabilityReport { } if let Some(baseline) = baseline_rate.get(&cell.group).filter(|r| **r > 0.0) { cell.share_of_baseline_percent = - Some(round2(cell.confirmed_per_sec / baseline * 100.0)); + Some(round2(cell.served_per_sec / baseline * 100.0)); } } cells.sort_by_key(|c| (c.group, c.window)); @@ -588,11 +618,11 @@ impl ReachabilityReport { for window in [Window::BeforeFault, Window::DuringFault, Window::AfterFault] { if let Some(cell) = self.cell(group, window) { lines.push(format!( - "S3 {} {}: {:.2} confirmed/s{}, {} submitted{}, {} indeterminate, {} \ + "S3 {} {}: {:.2} served/s{}, {} offered{}, {} indeterminate, {} \ attempt(s) timed out, {} of {} agents active", group.as_str(), window.as_str(), - cell.confirmed_per_sec, + cell.served_per_sec, cell.share_of_baseline_percent .map(|s| format!(" ({s:.1}% of baseline)")) .unwrap_or_default(), @@ -603,7 +633,7 @@ impl ReachabilityReport { .filter(|_| cell.window_secs > 0.0) .map(|q| { format!( - ", first offered {:.1}s into a {:.1}s window", + ", answered nothing for {:.1}s of a {:.1}s window", q as f64 / 1000.0, cell.window_secs ) @@ -916,7 +946,7 @@ mod tests { assert!(report.findings.is_empty()); assert!(report.cells.iter().all(|c| c.window == Window::Unknown)); - assert!(report.cells.iter().all(|c| c.confirmed_per_sec == 0.0)); + assert!(report.cells.iter().all(|c| c.served_per_sec == 0.0)); assert_eq!(report.recovery.count, 0); // And nothing is silently blamed on the agents themselves. assert!(report.agents_never_recovered.is_empty()); @@ -1087,7 +1117,7 @@ mod tests { /// arrived in the last four seconds of a 182-second window, once the fault /// was already coming down. #[test] - fn a_group_silent_until_the_heal_reports_how_long_it_offered_nothing() { + fn a_group_silent_until_the_heal_reports_how_long_it_answered_nothing() { let mut records = history(0, 180); records.retain(|r| { !(r.agent == ISOLATED @@ -1103,26 +1133,60 @@ mod tests { let cell = report.cell(Group::OnPod, Window::DuringFault).unwrap(); assert_eq!(cell.submitted, 4); - assert_eq!(cell.quiet_ms, Some(176_000)); + assert_eq!(cell.quiet_ms, Some(176_020)); assert!( cell.quiet_ms.unwrap() as f64 / 1000.0 > cell.window_secs * 0.9, "silence has to dominate the window, not trail it" ); - // Measured from the window's own start, not from the run's first - // operation: the baseline began 300s earlier. + // The same group's baseline answers once a second, so its longest + // silence is one interval. That contrast is the whole reading: 176s of + // silence is not a slower version of this, it is a different state. let baseline = report.cell(Group::OnPod, Window::BeforeFault).unwrap(); - assert_eq!(baseline.quiet_ms, Some(0)); + assert_eq!(baseline.quiet_ms, Some(1_000)); assert!( report .note_lines() .iter() - .any(|l| l.contains("first offered 176.0s into a 180.0s window")), + .any(|l| l.contains("answered nothing for 176.0s of a 180.0s window")), "notes were {:?}", report.note_lines() ); } + /// The regression the first S16 run turned up, in the module that shares + /// the construction. + /// + /// The workload keeps offering work to an unreachable executor all through + /// the partition, so anything derived from submission times looks busy no + /// matter what the platform is doing. Every one of these is offered during + /// the fault and answered only after the heal: the fault window served the + /// isolated group nothing, and the cell has to say so. + #[test] + fn work_offered_to_an_isolated_group_and_answered_after_the_heal_is_not_service() { + let mut records = history(0, 180); + records.retain(|r| { + !(r.agent == ISOLATED + && r.submitted_at >= t0() + && r.submitted_at < t0() + TimeDelta::seconds(180)) + }); + for offset in 0..120 { + let mut record = op(ISOLATED, offset, Outcome::Confirmed); + record.completed_at = Some(t0() + TimeDelta::seconds(181)); + record.duration_ms = (181 - offset) as u64 * 1_000; + records.push(record); + } + + let report = build(&records); + let cell = report.cell(Group::OnPod, Window::DuringFault).unwrap(); + + assert_eq!(cell.submitted, 120, "they were offered during the fault"); + assert_eq!(cell.confirmed, 120, "and they did all eventually confirm"); + assert_eq!(cell.served, 0, "but none of it was served during the fault"); + assert_eq!(cell.served_per_sec, 0.0); + assert_eq!(cell.quiet_ms, Some(180_000), "silent for the whole window"); + } + /// Work the cut caught and the run cannot account for. Not a finding, but /// it must sit next to the clean cells rather than be inferred from them. #[test] diff --git a/integration-tests/src/chaos/split.rs b/integration-tests/src/chaos/split.rs index e156b5cd9e..29d7d2e381 100644 --- a/integration-tests/src/chaos/split.rs +++ b/integration-tests/src/chaos/split.rs @@ -159,6 +159,35 @@ pub fn window_end( } } +/// The longest stretch of a window in which nothing was answered. +/// +/// The window's own edges are the first and last comparison points, which is +/// what separates "answered steadily but slowly" from "answered nothing for two +/// minutes and then caught up in a burst". Those two produce the same count and +/// the same rate; only this tells them apart. +pub fn longest_silence_ms( + served_at: &[DateTime], + start: Option>, + end: Option>, +) -> Option { + let (start, end) = (start?, end?); + let mut marks: Vec> = served_at + .iter() + .copied() + .filter(|at| *at >= start && *at <= end) + .collect(); + marks.sort_unstable(); + marks.insert(0, start); + marks.push(end); + Some( + marks + .windows(2) + .map(|pair| (pair[1] - pair[0]).num_milliseconds().max(0) as u64) + .max() + .unwrap_or(0), + ) +} + /// How long a window lasted, in seconds. /// /// The fault's own windows come from the workflow's timestamps, which is what From 5847a1470dbc2c0ba9d84da723e52d843b1fa258 Mon Sep 17 00:00:00 2001 From: Kaur Matas <33095685+kmatasfp@users.noreply.github.com> Date: Thu, 27 Aug 2026 10:30:20 -0700 Subject: [PATCH 28/40] Size the S16 fault window to the failover it models --- .../chaos_suites/cloud-chaos.yaml | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/integration-tests/chaos_suites/cloud-chaos.yaml b/integration-tests/chaos_suites/cloud-chaos.yaml index 4b975c7c8a..bd59050e5f 100644 --- a/integration-tests/chaos_suites/cloud-chaos.yaml +++ b/integration-tests/chaos_suites/cloud-chaos.yaml @@ -893,19 +893,36 @@ scenarios: # early and leaves the tail of the fault phase measuring a healed # cluster, which is how the first S16 run came to report 16.87% of # baseline throughput during a total outage. - durationSecs: 180 + durationSecs: 60 phases: # Long enough for cold starts, component loading and route warm-up to # settle, and for the scheduler to be firing steadily, so the cut lands on # a cluster in steady state. baselineSecs: 300 - # Has to comfortably exceed the caller's 120s attempt timeout, for the - # same reason S3's does: an operation that never reaches its timeout - # inside the window leaves the stall invisible in the fault-window cells. - # It also has to be long enough for the executors' connection pools to - # give up and start retrying rather than merely block. - faultSecs: 180 + # Sized to the outage this scenario models rather than to the caller's + # timeout: the worst case an AWS storage failover imposes, which is an + # Aurora reader promotion of about a minute and an ElastiCache Multi-AZ + # promotion of about thirty seconds. The key-value retry budget is built + # to cover exactly that, so an outage of this length is one the platform + # is expected to absorb without losing an executor. + # + # Deliberately shorter than S3's 180s, and not for the reason S3 gives. + # S3 needs the caller's 120s attempt timeout to fire inside the window + # because pending-then-timeout is the behaviour it exists to show. Here + # the opposite is the claim: invocations issued during the cut should + # still be waiting when the link returns and then complete. Throughput is + # attributed by completion, so the fault window still reads as a collapse + # and the recovery window as the catch-up, which is what says the + # partition took hold. + # + # Do not raise this past the retry budget without meaning to. The budget + # is 15 attempts, ~93s of capped backoff plus a 5s pool acquire apiece + # against a blackholed endpoint, so roughly 168s. Beyond that the + # recovery-index write in `status_flusher` exhausts and panics by design, + # and the scenario stops measuring absorption and starts measuring the + # crash it is supposed to have removed. + faultSecs: 60 # Every stream has to get several whole operations after the heal, so a # stream that recovered slowly is distinguishable from one that never # did. Scheduled actions registered during the outage also have to fall From 81c2e743596a0c7c03bbb3d698f0423e2a2db62f Mon Sep 17 00:00:00 2001 From: Kaur Matas <33095685+kmatasfp@users.noreply.github.com> Date: Thu, 27 Aug 2026 11:27:48 -0700 Subject: [PATCH 29/40] Judge the storage outage on quiet time rather than throughput share --- .../chaos_suites/cloud-chaos.yaml | 25 +- integration-tests/src/chaos/mod.rs | 80 ++++--- integration-tests/src/chaos/outage.rs | 218 +++++++++++++++--- integration-tests/src/chaos/scenarios/s16.rs | 12 +- 4 files changed, 269 insertions(+), 66 deletions(-) diff --git a/integration-tests/chaos_suites/cloud-chaos.yaml b/integration-tests/chaos_suites/cloud-chaos.yaml index bd59050e5f..d413d96334 100644 --- a/integration-tests/chaos_suites/cloud-chaos.yaml +++ b/integration-tests/chaos_suites/cloud-chaos.yaml @@ -975,13 +975,24 @@ scenarios: # which storage the run was about; the driver never resolves or contacts # it. endpoint: golem-postgres-dev-keyvalue.cluster-cgfyoqmjq7tc.us-east-1.rds.amazonaws.com - # The most of its baseline throughput the workload may keep during the - # outage for the fault to count as observed. Not zero: an operation - # submitted late in the fault window can still be waiting when the - # database returns, and then confirms into the window it was submitted in. - # At 100 ops/s over 180s that is a handful against a baseline of - # thousands, so anything near 15% means the partition did not take hold. - outageCeilingPercent: 15 + # The least of the fault window every stream must answer nothing at all + # for, as a share of that window, for the outage to count as observed. + # + # This was a ceiling on during-fault throughput as a share of baseline. + # That number is still reported, but it cannot be the verdict: it is a + # rate averaged over the whole window, while an absorbed outage does all + # its serving in the few seconds at the window's edges. The same handful + # of edge confirmations reads as ~8% of a 180s window and ~26% of a 60s + # one, so the threshold tracked the window length rather than the + # platform, and shortening this window to 60s duly tripped it on a + # partition that had plainly landed (run 33099228775: quiet 58.0s of + # 62.3s on every stream, yet 25.79% against a 15% ceiling). + # + # Quiet time is measured against the window's own edges, so it means the + # same thing whatever the window is. 50% is far from both outcomes it has + # to tell apart: a landed outage is quiet for over 90% of the window, an + # undisturbed cluster for a few percent. + outageQuietFloorPercent: 50 # What serving again may cost once the cluster is reachable. Recorded, not # asserted: this is really a measure of how long sqlx's pool takes to # notice, plus whatever recovery scan the executors run, and how long that diff --git a/integration-tests/src/chaos/mod.rs b/integration-tests/src/chaos/mod.rs index 5aecac027c..8df21e7ab9 100644 --- a/integration-tests/src/chaos/mod.rs +++ b/integration-tests/src/chaos/mod.rs @@ -671,20 +671,31 @@ pub struct StorageConfig { /// hostname. Chaos Mesh resolves it in the controller, so this is the same /// string that appears in the NetworkChaos manifest's `externalTargets`. pub endpoint: String, - /// The most of its baseline throughput the workload may keep during the - /// fault, as a percentage, for the outage to count as observed. + /// The least of the fault window every stream must answer nothing at all + /// for, as a percentage of that window, for the outage to count as + /// observed. /// - /// A run above this line did not take the storage away, whatever the fault + /// A run below this line did not take the storage away, whatever the fault /// status said, and every other number in the report then describes an /// undisturbed cluster. That is reported as inconclusive rather than clean: /// a healthy-looking result from a fault that never landed is the worst /// artifact this suite can produce. /// - /// It cannot be zero. Operations submitted in the last seconds before the - /// heal confirm just after it and are counted in the fault window they were - /// submitted in, so a genuine outage still leaves a small non-zero cell. - /// Read it together with `quietMs` — see [`crate::chaos::outage`]. - pub outage_ceiling_percent: f64, + /// This used to be a ceiling on during-fault throughput as a share of + /// baseline, and that number is still recorded. It stopped being the + /// verdict because it is a rate averaged over the whole window while all + /// the serving in an absorbed outage happens in the seconds at its edges, + /// so the same handful of edge confirmations reads as 8% of a 180s window + /// and 26% of a 60s one. The threshold then tracks the window length rather + /// than the platform, and shortening S16's window to 60s duly tripped it on + /// a partition that had plainly landed. + /// + /// Quiet time has no such coupling: it is measured against the window's own + /// edges, so it means the same thing whatever the window is. Every stream + /// is judged rather than the aggregate, because a stream still answering is + /// a fault that did not land on it and must not be outvoted by quieter + /// ones. + pub outage_quiet_floor_percent: f64, /// What serving again may cost once the storage is reachable, and the /// number each stream's recovery gap is reported against. Recorded rather /// than asserted, like every other budget in the suite: how long a @@ -947,18 +958,21 @@ impl ScenarioConfig { self.code ) })?; - // A zero ceiling can never be met: an operation submitted just before - // the heal confirms just after it and is counted in the fault window it - // was submitted in, so even a total outage leaves a small non-zero - // cell. Every run would then report the outage as not observed, and the - // one verdict that exists to catch a fault which never landed would be - // stuck on. - if config.outage_ceiling_percent <= 0.0 { + // A floor of 100% can never be met: a stream is only quiet between the + // answers it did give, and an operation submitted just before the heal + // confirms just after it, inside the window it was submitted in. Every + // run would then report the outage as not observed, and the one verdict + // that exists to catch a fault which never landed would be stuck on. A + // floor of zero is the opposite failure: every run passes, including + // the ones where nothing was ever cut off. + if !(0.0..100.0).contains(&config.outage_quiet_floor_percent) + || config.outage_quiet_floor_percent <= 0.0 + { anyhow::bail!( - "chaos scenario {}: outageCeilingPercent is {}, which no real outage can meet, \ - so every run would report the fault as not observed", + "chaos scenario {}: outageQuietFloorPercent is {}, which is not a share of the \ + fault window a real outage could be judged by", self.code, - config.outage_ceiling_percent + config.outage_quiet_floor_percent ); } if config.endpoint.trim().is_empty() { @@ -1241,7 +1255,7 @@ mod tests { fn storage_config( endpoint: &str, - ceiling: f64, + quiet_floor: f64, scheduled_agents: u32, scheduled_block: bool, ) -> ScenarioConfig { @@ -1282,7 +1296,7 @@ mod tests { rollback: None, storage: Some(StorageConfig { endpoint: endpoint.to_string(), - outage_ceiling_percent: ceiling, + outage_quiet_floor_percent: quiet_floor, recovery_budget_secs: 120, }), revert: None, @@ -1293,17 +1307,31 @@ mod tests { } } - /// A zero ceiling can never be met, so the one verdict that exists to catch - /// a fault which never landed would be stuck on for every run. + /// A floor of zero passes every run, including one where nothing was cut + /// off, so the verdict that exists to catch a fault which never landed + /// would never fire. #[test] - fn a_storage_outage_ceiling_of_zero_is_refused() { + fn a_storage_outage_quiet_floor_of_zero_is_refused() { let error = storage_config("db.example", 0.0, 0, true) .require_storage() .unwrap_err() .to_string(); assert!( - error.contains("no real outage can meet"), - "the message has to say why, got: {error}" + error.contains("outageQuietFloorPercent"), + "the message has to name the knob, got: {error}" + ); + } + + /// A stream is only quiet between the answers it did give, so no real + /// outage can be quiet for the whole window and the verdict would be stuck + /// on for every run. + #[test] + fn a_storage_outage_quiet_floor_of_a_whole_window_is_refused() { + assert!( + storage_config("db.example", 100.0, 0, true) + .require_storage() + .is_err(), + "a floor of the whole window can never be met" ); } @@ -1312,7 +1340,7 @@ mod tests { #[test] fn a_storage_block_with_no_endpoint_is_refused() { assert!( - storage_config(" ", 15.0, 0, true) + storage_config(" ", 50.0, 0, true) .require_storage() .is_err() ); diff --git a/integration-tests/src/chaos/outage.rs b/integration-tests/src/chaos/outage.rs index ce9f9fa4d7..37b6ad3ca5 100644 --- a/integration-tests/src/chaos/outage.rs +++ b/integration-tests/src/chaos/outage.rs @@ -239,12 +239,20 @@ pub struct StorageOutageReport { /// The thresholds from the suite YAML, recorded so an archived cell can be /// read years later against the numbers it was judged by rather than /// against today's config. - pub outage_ceiling_percent: f64, + pub outage_quiet_floor_percent: f64, pub recovery_budget_ms: u64, /// The whole workload's during-fault rate as a share of its own baseline. /// `None` for a run that never learned when the fault was. + /// + /// Recorded, no longer the verdict. It is a rate averaged over the whole + /// fault window, so it moves with the window length even when the platform + /// behaves identically. See `outage_quiet_floor_percent`. #[serde(default, skip_serializing_if = "Option::is_none")] pub share_of_baseline_percent: Option, + /// The least any one stream stayed silent during the fault, as a share of + /// that window. This is what the verdict is drawn from. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub quietest_stream_percent: Option, pub cells: Vec, /// What the outage began underneath, per stream. Empty for a run that never /// learned when the fault was. @@ -285,7 +293,7 @@ impl StorageOutageReport { records: &[OperationRecord], fault: Option, endpoint: &str, - outage_ceiling_percent: f64, + outage_quiet_floor_percent: f64, recovery_budget: Duration, ) -> Self { let mut tallies: BTreeMap<(Stream, Window), Tally> = BTreeMap::new(); @@ -388,9 +396,10 @@ impl StorageOutageReport { let mut report = Self { endpoint: endpoint.to_string(), - outage_ceiling_percent, + outage_quiet_floor_percent, recovery_budget_ms: recovery_budget.as_millis().min(u64::MAX as u128) as u64, share_of_baseline_percent: None, + quietest_stream_percent: None, cells, caught_in_flight: caught_in_flight(records, fault), recovery: Vec::new(), @@ -403,10 +412,25 @@ impl StorageOutageReport { report } - /// Did the outage land? Judged on the whole workload rather than per - /// stream: the claim under test is that the executors could not reach the - /// database, which is one claim about the cluster, and a low-volume stream - /// should not be able to flip it on its own. + /// Did the outage land? + /// + /// Judged on how long each stream answered *nothing*, as a share of the + /// fault window, and on the stream that managed it least. A stream still + /// serving is a fault that did not land on it, and averaging would let the + /// quiet ones outvote it. + /// + /// Quiet time rather than throughput because throughput is a rate over the + /// whole window while an absorbed outage does all its serving in the + /// seconds at the window's edges: the same handful of confirmations reads + /// as a small share of a long window and a large share of a short one, so a + /// throughput threshold tracks the window length rather than the platform. + /// Quiet time is measured against the window's own edges and does not move + /// when the window does. The share is still computed, and still reported, + /// as context. + /// + /// Streams with no before-fault serving are skipped. A stream the run never + /// got going says nothing about whether the storage went away, and the + /// driver's baseline gate already reports it. fn judge_outage( &mut self, fault: Option, @@ -435,16 +459,41 @@ impl StorageOutageReport { if baseline <= 0.0 { return; } - let share = round2(total(Window::DuringFault) / baseline * 100.0); - self.share_of_baseline_percent = Some(share); - if share > self.outage_ceiling_percent { + self.share_of_baseline_percent = + Some(round2(total(Window::DuringFault) / baseline * 100.0)); + + let served_before: BTreeSet = self + .cells + .iter() + .filter(|c| c.window == Window::BeforeFault && c.served > 0) + .map(|c| c.stream) + .collect(); + + let mut quiet: Vec<(Stream, f64)> = self + .cells + .iter() + .filter(|c| c.window == Window::DuringFault && c.window_secs > 0.0) + .filter(|c| served_before.contains(&c.stream)) + .filter_map(|c| { + c.quiet_ms + .map(|ms| (c.stream, round2(ms as f64 / (c.window_secs * 10.0)))) + }) + .collect(); + quiet.sort_by(|a, b| a.1.total_cmp(&b.1)); + + let Some(&(stream, quietest)) = quiet.first() else { + return; + }; + self.quietest_stream_percent = Some(quietest); + if quietest < self.outage_quiet_floor_percent { self.findings.push(OutageFinding { violation: OutageViolation::OutageNotObserved, - stream: None, + stream: Some(stream), detail: format!( - "the workload held {share}% of its baseline throughput while {} was supposed \ - to be unreachable, above the {}% ceiling; the executors could still reach it", - self.endpoint, self.outage_ceiling_percent + "{stream} kept answering through the fault window, silent for only \ + {quietest}% of it against a {}% floor, while {} was supposed to be \ + unreachable; the executors could still reach it", + self.outage_quiet_floor_percent, self.endpoint ), }); } @@ -538,10 +587,18 @@ impl StorageOutageReport { pub fn note_lines(&self) -> Vec { let mut lines = Vec::new(); if let Some(share) = self.share_of_baseline_percent { + let quiet = match self.quietest_stream_percent { + Some(q) => format!( + "the least quiet stream answered nothing for {q}% of the fault window \ + (floor {}%)", + self.outage_quiet_floor_percent + ), + None => "no stream had a before-fault baseline to be judged against".to_string(), + }; lines.push(format!( - "Storage outage: the workload held {share}% of its baseline throughput while {} \ - was unreachable (ceiling {}%)", - self.endpoint, self.outage_ceiling_percent + "Storage outage: {quiet} while {} was unreachable, and the workload held {share}% \ + of its baseline throughput across that window", + self.endpoint )); } else { lines.push(format!( @@ -684,7 +741,7 @@ mod tests { use test_r::test; const ENDPOINT: &str = "golem-postgres-dev-keyvalue.cluster-example.rds.amazonaws.com"; - const CEILING: f64 = 15.0; + const QUIET_FLOOR: f64 = 50.0; const AGENT: &str = "chaos-s16-durable-0000"; fn t0() -> DateTime { @@ -782,7 +839,7 @@ mod tests { records, Some(fault()), ENDPOINT, - CEILING, + QUIET_FLOOR, Duration::from_secs(120), ) } @@ -800,8 +857,9 @@ mod tests { ); let during = report.cell(Stream::Durable, Window::DuringFault).unwrap(); assert!( - during.share_of_baseline_percent.unwrap() < CEILING, - "the stream should have collapsed, got {during:?}" + report.quietest_stream_percent.unwrap() > QUIET_FLOOR, + "the streams should have gone silent, got {:?} from {during:?}", + report.quietest_stream_percent ); } @@ -824,16 +882,18 @@ mod tests { report.findings ); assert!( - report.share_of_baseline_percent.unwrap() > CEILING, - "the aggregate share should be above the ceiling, got {:?}", - report.share_of_baseline_percent + report.quietest_stream_percent.unwrap() < QUIET_FLOOR, + "a stream answering every second is never quiet for long, got {:?}", + report.quietest_stream_percent ); } - /// The verdict is on the whole workload rather than per stream, so a stream - /// the run barely drives cannot flip it on its own. + /// A stream the run barely drives cannot flip the verdict on its own. It is + /// judged on how long it was silent, not on its rate against a baseline, so + /// a trickle reads as the near-total silence it is rather than as a stream + /// holding up. #[test] - fn one_busy_stream_holding_up_cannot_be_outvoted_by_a_quiet_one() { + fn a_trickle_stream_does_not_make_the_outage_look_unobserved() { let mut records = history(0, true); // A trickle of a third stream that keeps working throughout: two // operations in the whole fault window against a baseline of hundreds. @@ -855,6 +915,108 @@ mod tests { ); } + /// The inverse of the trickle case, and the one the old aggregate verdict + /// could miss. A single stream still answering all the way through is a + /// fault that did not land on it, and the silence of the others must not + /// vote it down. + #[test] + fn a_stream_still_answering_is_a_finding_even_when_the_others_are_silent() { + let mut records = history(0, true); + for second in 1..=10 { + records.push(op(Stream::Promise, -second, Outcome::Confirmed)); + } + // Answering every second of the fault window, unlike durable and + // scheduled which say nothing at all. + for second in 0..180 { + records.push(op(Stream::Promise, second, Outcome::Confirmed)); + } + let report = build(&records); + + let finding = report + .findings + .iter() + .find(|f| f.violation == OutageViolation::OutageNotObserved) + .expect("a stream answering throughout is a fault that did not land on it"); + assert_eq!(finding.stream, Some(Stream::Promise)); + } + + /// An absorbed outage: silence across the window with the serving bunched + /// into the seconds at its two edges, which is the shape the platform + /// produces once storage failures are retried rather than fatal. + fn absorbed(fault_secs: i64) -> StorageOutageReport { + let mut records = Vec::new(); + for second in 1..=300 { + for _ in 0..10 { + records.push(op(Stream::Durable, -second, Outcome::Confirmed)); + records.push(op(Stream::Scheduled, -second, Outcome::Confirmed)); + } + } + for edge in [0, fault_secs - 1] { + for _ in 0..60 { + records.push(op(Stream::Durable, edge, Outcome::Confirmed)); + records.push(op(Stream::Scheduled, edge, Outcome::Confirmed)); + } + } + for second in fault_secs + 1..=fault_secs + 240 { + records.push(op(Stream::Durable, second, Outcome::Confirmed)); + records.push(op(Stream::Scheduled, second, Outcome::Confirmed)); + } + StorageOutageReport::build( + &records, + Some(FaultWindow { + injected_at: t0(), + recovered_at: Some(t0() + TimeDelta::seconds(fault_secs)), + }), + ENDPOINT, + QUIET_FLOOR, + Duration::from_secs(120), + ) + } + + /// The regression this floor exists for. + /// + /// An absorbed outage does all its serving in the seconds at the window's + /// edges, so a during-fault *rate* divides one fixed burst by the window + /// length: identical platform behaviour reads as a small share of a long + /// window and a large share of a short one. Shortening S16's window from + /// 180s to 60s duly tripped the old 15% ceiling on a partition that had + /// plainly landed. Quiet time is measured against the window's own edges + /// and does not move with it. + #[test] + fn the_verdict_does_not_move_when_the_fault_window_does() { + let long = absorbed(180); + let short = absorbed(60); + + for (secs, report) in [(180, &long), (60, &short)] { + assert!( + report + .findings + .iter() + .all(|f| f.violation != OutageViolation::OutageNotObserved), + "a {secs}s absorbed outage did land, got {:?}", + report.findings + ); + assert!( + report.quietest_stream_percent.unwrap() > QUIET_FLOOR, + "a {secs}s absorbed outage is silent for most of its window, got {:?}", + report.quietest_stream_percent + ); + } + + // And the number that used to decide it, kept as the demonstration: + // the same shape reads very differently at the two lengths, which is + // exactly why it could not stay the verdict. + let long_share = long.share_of_baseline_percent.unwrap(); + let short_share = short.share_of_baseline_percent.unwrap(); + // 15% was the old ceiling. The long window sits under it and the short + // one over it, on the same behaviour, which is the whole defect. + assert!( + long_share < 15.0 && short_share > 15.0, + "this test only means something if the old ceiling would have flipped between the \ + two windows, got {long_share}% over 180s against {short_share}% over 60s" + ); + } + /// A stream that was working before the outage and confirmed nothing after /// the heal is the platform losing a mechanism, not a slow recovery. #[test] @@ -923,7 +1085,7 @@ mod tests { &history(180, true), None, ENDPOINT, - CEILING, + QUIET_FLOOR, Duration::from_secs(120), ); diff --git a/integration-tests/src/chaos/scenarios/s16.rs b/integration-tests/src/chaos/scenarios/s16.rs index 83b553268d..ae0c478d8a 100644 --- a/integration-tests/src/chaos/scenarios/s16.rs +++ b/integration-tests/src/chaos/scenarios/s16.rs @@ -451,15 +451,17 @@ pub async fn run( &records, fault_window, &storage_config.endpoint, - storage_config.outage_ceiling_percent, + storage_config.outage_quiet_floor_percent, storage_config.recovery_budget(), ); info!( - "S16: storage account — the workload held {:?}% of its baseline throughput while {} was \ - unreachable (ceiling {}%), {} findings", - outage.share_of_baseline_percent, + "S16: storage account — the least quiet stream answered nothing for {:?}% of the fault \ + window (floor {}%) while {} was unreachable, holding {:?}% of baseline throughput, {} \ + findings", + outage.quietest_stream_percent, + outage.outage_quiet_floor_percent, outage.endpoint, - outage.outage_ceiling_percent, + outage.share_of_baseline_percent, outage.findings.len() ); for finding in &outage.findings { From ccffef3ca8c5ddbe8c382615129ff81b9ef1ae46 Mon Sep 17 00:00:00 2001 From: Kaur Matas <33095685+kmatasfp@users.noreply.github.com> Date: Thu, 27 Aug 2026 11:32:42 -0700 Subject: [PATCH 30/40] Add the S22 key-value outage that outlasts the retry budget --- golem-test-framework/src/benchmark/config.rs | 5 +- .../chaos_suites/cloud-chaos.yaml | 86 +++++++++++++++++++ integration-tests/src/benchmarks/all.rs | 8 +- integration-tests/src/chaos/mod.rs | 12 ++- integration-tests/src/chaos/scenarios/mod.rs | 10 ++- .../scenarios/{s16.rs => storage_outage.rs} | 68 ++++++++++----- 6 files changed, 156 insertions(+), 33 deletions(-) rename integration-tests/src/chaos/scenarios/{s16.rs => storage_outage.rs} (90%) diff --git a/golem-test-framework/src/benchmark/config.rs b/golem-test-framework/src/benchmark/config.rs index 239769e961..b62216159a 100644 --- a/golem-test-framework/src/benchmark/config.rs +++ b/golem-test-framework/src/benchmark/config.rs @@ -254,8 +254,11 @@ pub enum ChaosScenarioArg { S6, /// Executor pod kill while a component rollback is in flight. S9, - /// Executors cut off from the key-value PostgreSQL cluster. + /// Executors cut off from the key-value PostgreSQL cluster for about as + /// long as an AWS storage failover takes. S16, + /// The same cut, held for longer than the key-value retry budget. + S22, } /// Density subcommand action. diff --git a/integration-tests/chaos_suites/cloud-chaos.yaml b/integration-tests/chaos_suites/cloud-chaos.yaml index d413d96334..ca2d733d7f 100644 --- a/integration-tests/chaos_suites/cloud-chaos.yaml +++ b/integration-tests/chaos_suites/cloud-chaos.yaml @@ -1011,3 +1011,89 @@ scenarios: delaySecs: 5 signalTimeoutSecs: 1800 + + # S22 — the long-outage companion to S16 (GOL-499). Same cut, same driver, + # held past the point where the platform can absorb it. + # + # S16 asks whether golem rides out a storage failover. This asks what happens + # when the storage does not come back in time. The key-value retry budget is + # 15 attempts, roughly 93s of capped backoff plus a 5s pool acquire apiece + # against a blackholed endpoint, so about 168s. Past that the recovery-index + # write in `status_flusher` panics on purpose, on the grounds that an executor + # which cannot maintain its own running-workers index should be replaced + # rather than left running with agents it can no longer account for. + # + # So executor restarts are the *expected* outcome here, not a defect, and + # nothing in this entry treats them as one. What the run has to establish is + # that the exit is the intended one and that nothing is lost across it: the + # scheduled-fire account and the exactly-once account are the same assertions + # S16 makes, and they are the ones that fail the build. + - code: S22 + name: keyvalue-postgres-outage-past-budget + enabled: true + + fault: + kind: network-partition + target: worker-executor + # Every executor, as in S16. A budget exhausted on only some of them would + # leave the survivors serving and make the replay cost unreadable. + mode: all + # Must equal phases.faultSecs; see the note on S16. + durationSecs: 300 + + phases: + # Same as S16: long enough for cold starts, component loading and route + # warm-up to settle, and for the scheduler to be firing steadily. + baselineSecs: 300 + # Comfortably past the ~168s budget rather than just over it. A window + # that only just exceeds the budget would make the run's outcome depend + # on which second an operation happened to start in, so a rerun could + # come back with restarts or without them for no reason worth reporting. + # At 300s every tracked transition in the window exhausts. + faultSecs: 300 + # Longer than S16's 420s, and for a reason S16 does not have: the + # executors are replaced here, so recovery is not just a pool noticing + # its database came back. It is pods rescheduling, rejoining the routing + # table, and every agent that was on them replaying from oplog or + # snapshot. Scheduled actions registered during the outage also have to + # fall due and fire inside the run rather than after it. + recoverySecs: 600 + + workload: + # Identical to S16 on purpose. The two runs are meant to be read side by + # side, and a different population would make the difference between them + # a question about the workload rather than about the outage length. + durableAgents: 200 + ephemeralAgents: 50 + scheduledAgents: 0 + promiseAgents: 50 + quotaAgents: 0 + ratePerSec: 100 + + scheduled: + targets: 100 + intervalMillis: 2000 + leadSecs: 10 + # Wider than S16's 240s. The lease has to outlast the outage plus the + # replay behind it, or actions would be judged late for the recovery this + # scenario exists to measure. + leaseBudgetSecs: 480 + + storage: + endpoint: golem-postgres-dev-keyvalue.cluster-cgfyoqmjq7tc.us-east-1.rds.amazonaws.com + # Same floor as S16, and it should be met even more comfortably: an + # outage this long leaves the streams silent for nearly the whole window. + outageQuietFloorPercent: 50 + # Recorded, not asserted, as everywhere else — but read differently here. + # In S16 this measures a connection pool noticing. Here it measures a + # cluster rebuilding itself, and it is the number that says whether the + # deliberate crash is an acceptable trade. + recoveryBudgetSecs: 300 + + retryPolicy: + # Identical to S16's, for the same reason. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 diff --git a/integration-tests/src/benchmarks/all.rs b/integration-tests/src/benchmarks/all.rs index 4300462b53..01aeb7e651 100644 --- a/integration-tests/src/benchmarks/all.rs +++ b/integration-tests/src/benchmarks/all.rs @@ -599,6 +599,7 @@ async fn run_chaos( ChaosScenarioArg::S6 => chaos::ScenarioCode::S6, ChaosScenarioArg::S9 => chaos::ScenarioCode::S9, ChaosScenarioArg::S16 => chaos::ScenarioCode::S16, + ChaosScenarioArg::S22 => chaos::ScenarioCode::S22, }; let config = suite .scenario(code, allow_disabled) @@ -648,8 +649,11 @@ async fn run_chaos( chaos::ScenarioCode::S9 => { chaos::scenarios::s9::run(&config, &manifest, &deps, &signals, &outputs).await } - chaos::ScenarioCode::S16 => { - chaos::scenarios::s16::run(&config, &manifest, &deps, &signals, &outputs).await + code @ (chaos::ScenarioCode::S16 | chaos::ScenarioCode::S22) => { + chaos::scenarios::storage_outage::run( + code, &config, &manifest, &deps, &signals, &outputs, + ) + .await } }; diff --git a/integration-tests/src/chaos/mod.rs b/integration-tests/src/chaos/mod.rs index 8df21e7ab9..024d7ff7b5 100644 --- a/integration-tests/src/chaos/mod.rs +++ b/integration-tests/src/chaos/mod.rs @@ -90,8 +90,12 @@ pub enum ScenarioCode { S6, /// Executor pod kill while a component rollback is in flight. S9, - /// Executors cut off from the key-value PostgreSQL cluster. + /// Executors cut off from the key-value PostgreSQL cluster for about as + /// long as an AWS storage failover takes. S16, + /// The same cut, held for longer than the key-value retry budget, so the + /// executors are expected to be replaced rather than to ride it out. + S22, } impl ScenarioCode { @@ -109,13 +113,14 @@ impl ScenarioCode { ScenarioCode::S6 => "S6", ScenarioCode::S9 => "S9", ScenarioCode::S16 => "S16", + ScenarioCode::S22 => "S22", } } /// Every scenario this driver implements. The suite YAML is checked against /// this list, so a scenario cannot be enabled in YAML without code behind /// it, nor implemented without an operational switch in front of it. - pub const ALL: [ScenarioCode; 12] = [ + pub const ALL: [ScenarioCode; 13] = [ ScenarioCode::S1, ScenarioCode::S3, ScenarioCode::S5, @@ -128,6 +133,7 @@ impl ScenarioCode { ScenarioCode::S12, ScenarioCode::S13, ScenarioCode::S16, + ScenarioCode::S22, ]; pub fn parse(s: &str) -> Option { @@ -1483,7 +1489,7 @@ mod tests { entry.require_workload().unwrap(); entry.require_rollback().unwrap(); } - ScenarioCode::S16 => { + ScenarioCode::S16 | ScenarioCode::S22 => { entry.require_workload().unwrap(); entry.require_scheduled().unwrap(); entry.require_storage().unwrap(); diff --git a/integration-tests/src/chaos/scenarios/mod.rs b/integration-tests/src/chaos/scenarios/mod.rs index dd792a29d0..8c35522d81 100644 --- a/integration-tests/src/chaos/scenarios/mod.rs +++ b/integration-tests/src/chaos/scenarios/mod.rs @@ -14,9 +14,11 @@ //! Chaos scenario implementations. //! -//! One module per scenario code. Each one owns its phase choreography — which -//! is the part that differs, and the part worth reading — while everything -//! around it lives here: where artifacts go, how a signal failure becomes a +//! One module per scenario code, except where two codes are the same +//! choreography under different settings: `storage_outage` runs both S16 and +//! S22, which differ only in how long the storage is taken away for. Each +//! module owns its phase choreography — which is the part that differs, and the +//! part worth reading — while everything around it lives here: where artifacts go, how a signal failure becomes a //! termination reason, how a routing table is sampled, and how a result is //! assembled. //! @@ -30,13 +32,13 @@ pub mod s10; pub mod s11; pub mod s12; pub mod s13; -pub mod s16; pub mod s3; pub mod s5; pub mod s6; pub mod s7; pub mod s8; pub mod s9; +pub mod storage_outage; use crate::chaos::ScenarioConfig; use crate::chaos::history::{OperationHistory, OperationRecord, Stream}; diff --git a/integration-tests/src/chaos/scenarios/s16.rs b/integration-tests/src/chaos/scenarios/storage_outage.rs similarity index 90% rename from integration-tests/src/chaos/scenarios/s16.rs rename to integration-tests/src/chaos/scenarios/storage_outage.rs index ae0c478d8a..3644b9fe94 100644 --- a/integration-tests/src/chaos/scenarios/s16.rs +++ b/integration-tests/src/chaos/scenarios/storage_outage.rs @@ -12,7 +12,26 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! S16 — key-value PostgreSQL outage recovery (GOL-379). +//! Key-value storage outage: the shared choreography behind S16 and S22. +//! +//! Both codes run this module. They differ only in how long the storage is +//! taken away for, which is a suite setting, and in what a reader should expect +//! of the result: +//! +//! * **S16** (GOL-379) cuts for the length of an AWS storage failover, about a +//! minute. The key-value retry budget covers that, so the claim is that the +//! platform absorbs it: operations stall and then complete, and no executor +//! is lost. +//! * **S22** (GOL-499) cuts for longer than the retry budget. The budget is +//! then exhausted, the recovery-index write in `status_flusher` panics on +//! purpose, and executors are replaced. The claim is not survival but that +//! the exit is the intended one and nothing is lost across it. +//! +//! The driver is the same either way because the difference is one of +//! expectation, not of choreography. Nothing below asserts on which of the two +//! happened: the account it produces answers both questions, and the oracles +//! that fail the build — the scheduled-fire account and the exactly-once +//! account — are the ones both scenarios share. //! //! The first scenario in the suite that breaks something the platform depends //! on rather than something the platform *is*. Every fault before this one @@ -135,6 +154,7 @@ const SETTLE_MARGIN: Duration = Duration::from_secs(30); const FIRE_PROOF_SAMPLE: usize = 5; pub async fn run( + code: ScenarioCode, config: &ScenarioConfig, manifest: &ChaosPrepManifest, deps: &BenchmarkTestDependencies, @@ -145,8 +165,8 @@ pub async fn run( let workload_config = config.require_workload()?; let scheduled_config = config.require_scheduled()?; let storage_config = config.require_storage()?; - let history = OperationHistory::new(ScenarioCode::S16.as_str()); - let key_prefix = crate::chaos::scenario_key_prefix(ScenarioCode::S16); + let history = OperationHistory::new(code.as_str()); + let key_prefix = crate::chaos::scenario_key_prefix(code); let user = manifest.user_context(deps); let counters = user @@ -237,16 +257,16 @@ pub async fn run( routing_snapshots.push(snapshot_routing(deps, "before-warmup").await); attention_extra.push(wait_for_settled_routing(deps, &mut routing_snapshots).await); info!( - "S16: warming {} schedule emitters and targets before the baseline", + "{code}: warming {} schedule emitters and targets before the baseline", targets.len() ); let warmed = scheduled::warm(&ctx, &targets).await; - info!("S16: warmed {warmed} agents, settling {WARMUP_SETTLE:?}"); + info!("{code}: warmed {warmed} agents, settling {WARMUP_SETTLE:?}"); tokio::time::sleep(WARMUP_SETTLE).await; // ── Baseline ──────────────────────────────────────────────────────────── info!( - "S16: baseline phase, mixed workload at {} ops/s plus {} schedule targets, for {:?}", + "{code}: baseline phase, mixed workload at {} ops/s plus {} schedule targets, for {:?}", workload_config.rate_per_sec, targets.len(), config.phases.baseline() @@ -274,7 +294,7 @@ pub async fn run( if baseline_operations == 0 { // Taking a database away from a workload that never worked would // measure nothing. Stop before touching the cluster. - warn!("S16: baseline produced no confirmed operations, aborting before injection"); + warn!("{code}: baseline produced no confirmed operations, aborting before injection"); stop_workloads!(); let records = history.snapshot(); finish!( @@ -293,9 +313,11 @@ pub async fn run( // this scenario is about, and a platform that accepted every registration // and ran none of them would otherwise reach read-back and report a // flawless account of a mechanism that never worked. - let sampled = sample_fire_count(&ctx, &targets).await; + let sampled = sample_fire_count(code, &ctx, &targets).await; if sampled == 0 { - warn!("S16: {baseline_operations} operations confirmed and no scheduled action has fired"); + warn!( + "{code}: {baseline_operations} operations confirmed and no scheduled action has fired" + ); stop_workloads!(); let records = history.snapshot(); finish!( @@ -310,14 +332,14 @@ pub async fn run( ); } info!( - "S16: baseline complete ({baseline_operations} confirmed ops, {sampled} fires across a \ + "{code}: baseline complete ({baseline_operations} confirmed ops, {sampled} fires across a \ sample of {} targets), signalling readiness", FIRE_PROOF_SAMPLE.min(targets.len()) ); // ── Signal: ready for the fault ───────────────────────────────────────── signals.write_baseline_ready(&BaselineReady { - scenario_code: ScenarioCode::S16.as_str().to_string(), + scenario_code: code.as_str().to_string(), ready_at: Utc::now(), baseline_operations, // Nothing to aim. The partition is between every executor and one @@ -330,7 +352,7 @@ pub async fn run( let injected = match signals.await_fault_injected(config.signal_timeout()).await { Ok(injected) => injected, Err(e) => { - warn!("S16: no fault-injected signal arrived: {e}"); + warn!("{code}: no fault-injected signal arrived: {e}"); stop_workloads!(); let records = history.snapshot(); finish!( @@ -344,7 +366,7 @@ pub async fn run( } }; info!( - "S16: fault {} ({} on {}) reported active at {}, {} is now unreachable from the executors", + "{code}: fault {} ({} on {}) reported active at {}, {} is now unreachable from the executors", injected.fault_id, injected.kind, injected.target, @@ -360,7 +382,7 @@ pub async fn run( let recovered = match signals.await_fault_recovered(config.signal_timeout()).await { Ok(recovered) => recovered, Err(e) => { - warn!("S16: no fault-recovered signal arrived: {e}"); + warn!("{code}: no fault-recovered signal arrived: {e}"); stop_workloads!(); let records = history.snapshot(); finish!( @@ -374,7 +396,7 @@ pub async fn run( } }; info!( - "S16: fault cleared at {} ({})", + "{code}: fault cleared at {} ({})", recovered.recovered_at, recovered.termination_reason ); fault_recovered_at = Some(recovered.recovered_at); @@ -384,7 +406,7 @@ pub async fn run( // ── Recovery ──────────────────────────────────────────────────────────── info!( - "S16: recovery phase, running for a further {:?}", + "{code}: recovery phase, running for a further {:?}", config.phases.recovery() ); ctx.phase.set(Phase::Recovery); @@ -413,7 +435,7 @@ pub async fn run( // ── Read-back ─────────────────────────────────────────────────────────── let settle = settle_before_readback(scheduled_config); - info!("S16: letting the last actions fall due and fire, {settle:?} before read-back"); + info!("{code}: letting the last actions fall due and fire, {settle:?} before read-back"); tokio::time::sleep(settle).await; let records = history.snapshot(); @@ -438,7 +460,7 @@ pub async fn run( scheduled_config.lease_budget(), ); info!( - "S16: scheduled-fire account — {} registrations accepted, {} fired once, {} \ + "{code}: scheduled-fire account — {} registrations accepted, {} fired once, {} \ inconclusive, {} unverifiable, {} findings", fires.registrations_confirmed, fires.fired_once, @@ -455,7 +477,7 @@ pub async fn run( storage_config.recovery_budget(), ); info!( - "S16: storage account — the least quiet stream answered nothing for {:?}% of the fault \ + "{code}: storage account — the least quiet stream answered nothing for {:?}% of the fault \ window (floor {}%) while {} was unreachable, holding {:?}% of baseline throughput, {} \ findings", outage.quietest_stream_percent, @@ -465,7 +487,7 @@ pub async fn run( outage.findings.len() ); for finding in &outage.findings { - warn!("S16: {}: {}", finding.violation, finding.detail); + warn!("{code}: {}: {}", finding.violation, finding.detail); } let readback = read_back(&ctx, &records, workload_config, &logs).await; @@ -485,7 +507,7 @@ pub async fn run( &after_probe, ); info!( - "S16: exactly-once account — {} keys checked, {} with a final result, {} recovered by \ + "{code}: exactly-once account — {} keys checked, {} with a final result, {} recovered by \ the probe, {} findings", exactly_once.keys_checked, exactly_once.keys_with_final_result, @@ -594,12 +616,12 @@ async fn read_back( } /// Reads the fire count of a few targets, to prove actions are firing at all. -async fn sample_fire_count(ctx: &WorkloadContext, targets: &[String]) -> u64 { +async fn sample_fire_count(code: ScenarioCode, ctx: &WorkloadContext, targets: &[String]) -> u64 { let mut total = 0u64; for target in targets.iter().take(FIRE_PROOF_SAMPLE) { match workload::read_polls(ctx, target).await { Ok(polls) => total += polls, - Err(e) => warn!("S16: could not sample fires on {target}: {e}"), + Err(e) => warn!("{code}: could not sample fires on {target}: {e}"), } } total From 97bfeaec231fee3c0f1b87c4e762e13a86fdd03f Mon Sep 17 00:00:00 2001 From: Kaur Matas <33095685+kmatasfp@users.noreply.github.com> Date: Thu, 27 Aug 2026 12:30:31 -0700 Subject: [PATCH 31/40] Name the running scenario in the clamped-registration note --- integration-tests/src/chaos/scenarios/storage_outage.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration-tests/src/chaos/scenarios/storage_outage.rs b/integration-tests/src/chaos/scenarios/storage_outage.rs index 3644b9fe94..402f603290 100644 --- a/integration-tests/src/chaos/scenarios/storage_outage.rs +++ b/integration-tests/src/chaos/scenarios/storage_outage.rs @@ -425,8 +425,8 @@ pub async fn run( } if skipped > 0 { attention_extra.push(Note::attention(format!( - "S16 skipped {skipped} registration ticks because targets still had their budget of \ - {} in flight — the offered rate was clamped by the platform, so the phase counts \ + "{code} skipped {skipped} registration ticks because targets still had their budget \ + of {} in flight — the offered rate was clamped by the platform, so the phase counts \ understate what the run intended to submit", scheduled::MAX_IN_FLIGHT_PER_TARGET ))); From 674ba748ac2441753aa108877da0b3a70a412a3f Mon Sep 17 00:00:00 2001 From: Kaur Matas <33095685+kmatasfp@users.noreply.github.com> Date: Thu, 27 Aug 2026 15:58:45 -0700 Subject: [PATCH 32/40] Add the S14 indexed-oplog outage scenario --- golem-test-framework/src/benchmark/config.rs | 3 + .../chaos_suites/cloud-chaos.yaml | 98 ++++++++++++++++ integration-tests/src/benchmarks/all.rs | 5 +- integration-tests/src/chaos/mod.rs | 10 +- .../src/chaos/scenarios/storage_outage.rs | 110 +++++++++++------- 5 files changed, 179 insertions(+), 47 deletions(-) diff --git a/golem-test-framework/src/benchmark/config.rs b/golem-test-framework/src/benchmark/config.rs index b62216159a..61d0856587 100644 --- a/golem-test-framework/src/benchmark/config.rs +++ b/golem-test-framework/src/benchmark/config.rs @@ -259,6 +259,9 @@ pub enum ChaosScenarioArg { S16, /// The same cut, held for longer than the key-value retry budget. S22, + /// Executors cut off from the indexed-oplog PostgreSQL cluster for the + /// length of a writer failover. + S14, } /// Density subcommand action. diff --git a/integration-tests/chaos_suites/cloud-chaos.yaml b/integration-tests/chaos_suites/cloud-chaos.yaml index ca2d733d7f..f9aedd0a8b 100644 --- a/integration-tests/chaos_suites/cloud-chaos.yaml +++ b/integration-tests/chaos_suites/cloud-chaos.yaml @@ -1097,3 +1097,101 @@ scenarios: delaySecs: 5 signalTimeoutSecs: 1800 + + # S14 — the oplog half of the storage matrix (GOL-376). Same driver as S16 and + # S22, same cut, aimed at the other Aurora cluster. + # + # S16 takes away the cluster that says what the platform is doing: promises, + # the running-workers set, user key-value data and the scheduler's schema. S14 + # takes away the one that says what it did. The oplog is the only thing on the + # indexed cluster, so under this cut promises still resolve, the scheduler + # still claims and acknowledges on time, and the running-workers set is still + # writable — and nothing durable can be committed. The two scenarios are + # near-mirror images and are meant to be read side by side. + # + # The retry budget is not the interesting quantity here the way it is in S16 + # and S22. golem-dev gives indexed storage 200 attempts with a 10s cap, which + # is over half an hour of stalling, so this window cannot exhaust it. What is + # worth watching instead is the error classification: `retry_storage_op` + # panics immediately on any indexed-storage error it was not handed as + # transient, and transient means `PoolTimedOut` or `Io` and nothing else. An + # executor lost during this run is a misclassification finding, not an + # exhaustion one. + - code: S14 + name: indexed-oplog-postgres-outage + enabled: true + + fault: + kind: network-partition + target: worker-executor + # Every executor, as in S16 and for the same reason: a storage outage only + # some executors could see would be a routing fault wearing a database's + # clothes. + mode: all + # Must equal phases.faultSecs; see the note on S16. + durationSecs: 60 + + phases: + # Same as S16, so the two are comparable: long enough for cold starts, + # component loading and route warm-up to settle, and for the scheduler to + # be firing steadily. + baselineSecs: 300 + # The same worst-case Aurora writer promotion S16 is sized to, about a + # minute. GOL-376 was written with 30s, before that number had been + # worked out; 60s is the failover this is supposed to model and matching + # S16 is what makes the two results readable against each other. + faultSecs: 60 + # As S16: every stream needs several whole operations after the heal, and + # scheduled actions registered during the outage have to fall due and fire + # inside the run. + recoverySecs: 420 + + workload: + # Identical to S16, deliberately. The comparison between the two runs is + # the point, and a different population would turn the difference into a + # question about the workload. + durableAgents: 200 + ephemeralAgents: 50 + # Zero for the same reason as S16: the scheduled stream comes from the + # block below, because only its registrations carry a token into the + # target's fire log. Setting both is refused at load time. + scheduledAgents: 0 + # Promises live on the cluster this cut leaves alone, so unlike in S16 + # this stream is not directly on the fault path. It still degrades, + # because completing a promise wakes an agent that then cannot commit, + # and that gap between "the promise was written" and "the agent got + # anywhere" is worth having on the record. + promiseAgents: 50 + quotaAgents: 0 + ratePerSec: 100 + + scheduled: + targets: 100 + intervalMillis: 2000 + leadSecs: 10 + # Same as S16's, and it should be met with room to spare. The scheduler is + # outside this cut, so claims and acknowledgements keep their timing and + # the delay reduces to how long the fired invocation could not commit. + # That makes S14 the cleanest read of scheduler lag in the suite: in S16 + # and S22 the scheduler is inside the outage and the two costs cannot be + # separated. + leaseBudgetSecs: 240 + + storage: + # The indexed writer endpoint, not the key-value one. This is the only + # field that differs materially from S16. + endpoint: golem-postgres-dev-indexed.cluster-cgfyoqmjq7tc.us-east-1.rds.amazonaws.com + # Same floor and same reasoning as S16. + outageQuietFloorPercent: 50 + # Recorded, not asserted. Expected to be small: nothing is replaced here, + # so this is a connection pool noticing its database is back plus whatever + # the stalled commits still have to do. + recoveryBudgetSecs: 120 + + retryPolicy: + # Identical to S16's, for the same reason. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 diff --git a/integration-tests/src/benchmarks/all.rs b/integration-tests/src/benchmarks/all.rs index 01aeb7e651..3c069a38e3 100644 --- a/integration-tests/src/benchmarks/all.rs +++ b/integration-tests/src/benchmarks/all.rs @@ -600,6 +600,7 @@ async fn run_chaos( ChaosScenarioArg::S9 => chaos::ScenarioCode::S9, ChaosScenarioArg::S16 => chaos::ScenarioCode::S16, ChaosScenarioArg::S22 => chaos::ScenarioCode::S22, + ChaosScenarioArg::S14 => chaos::ScenarioCode::S14, }; let config = suite .scenario(code, allow_disabled) @@ -649,7 +650,9 @@ async fn run_chaos( chaos::ScenarioCode::S9 => { chaos::scenarios::s9::run(&config, &manifest, &deps, &signals, &outputs).await } - code @ (chaos::ScenarioCode::S16 | chaos::ScenarioCode::S22) => { + code @ (chaos::ScenarioCode::S14 + | chaos::ScenarioCode::S16 + | chaos::ScenarioCode::S22) => { chaos::scenarios::storage_outage::run( code, &config, &manifest, &deps, &signals, &outputs, ) diff --git a/integration-tests/src/chaos/mod.rs b/integration-tests/src/chaos/mod.rs index 024d7ff7b5..0e06323f16 100644 --- a/integration-tests/src/chaos/mod.rs +++ b/integration-tests/src/chaos/mod.rs @@ -96,6 +96,9 @@ pub enum ScenarioCode { /// The same cut, held for longer than the key-value retry budget, so the /// executors are expected to be replaced rather than to ride it out. S22, + /// Executors cut off from the indexed-oplog PostgreSQL cluster, the other + /// Aurora cluster underneath them, for the length of a writer failover. + S14, } impl ScenarioCode { @@ -114,13 +117,14 @@ impl ScenarioCode { ScenarioCode::S9 => "S9", ScenarioCode::S16 => "S16", ScenarioCode::S22 => "S22", + ScenarioCode::S14 => "S14", } } /// Every scenario this driver implements. The suite YAML is checked against /// this list, so a scenario cannot be enabled in YAML without code behind /// it, nor implemented without an operational switch in front of it. - pub const ALL: [ScenarioCode; 13] = [ + pub const ALL: [ScenarioCode; 14] = [ ScenarioCode::S1, ScenarioCode::S3, ScenarioCode::S5, @@ -132,6 +136,7 @@ impl ScenarioCode { ScenarioCode::S11, ScenarioCode::S12, ScenarioCode::S13, + ScenarioCode::S14, ScenarioCode::S16, ScenarioCode::S22, ]; @@ -1444,6 +1449,7 @@ mod tests { assert_eq!(ScenarioCode::parse("s6"), Some(ScenarioCode::S6)); assert_eq!(ScenarioCode::parse("s9"), Some(ScenarioCode::S9)); assert_eq!(ScenarioCode::parse("s16"), Some(ScenarioCode::S16)); + assert_eq!(ScenarioCode::parse("s14"), Some(ScenarioCode::S14)); assert_eq!(ScenarioCode::parse("S99"), None); } @@ -1489,7 +1495,7 @@ mod tests { entry.require_workload().unwrap(); entry.require_rollback().unwrap(); } - ScenarioCode::S16 | ScenarioCode::S22 => { + ScenarioCode::S14 | ScenarioCode::S16 | ScenarioCode::S22 => { entry.require_workload().unwrap(); entry.require_scheduled().unwrap(); entry.require_storage().unwrap(); diff --git a/integration-tests/src/chaos/scenarios/storage_outage.rs b/integration-tests/src/chaos/scenarios/storage_outage.rs index 402f603290..a4b6954a4b 100644 --- a/integration-tests/src/chaos/scenarios/storage_outage.rs +++ b/integration-tests/src/chaos/scenarios/storage_outage.rs @@ -12,38 +12,45 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Key-value storage outage: the shared choreography behind S16 and S22. +//! Storage outage: the shared choreography behind S14, S16 and S22. //! -//! Both codes run this module. They differ only in how long the storage is -//! taken away for, which is a suite setting, and in what a reader should expect -//! of the result: +//! All three codes run this module. They differ in which cluster is taken away +//! and for how long, both of which are suite settings, and in what a reader +//! should expect of the result: //! -//! * **S16** (GOL-379) cuts for the length of an AWS storage failover, about a -//! minute. The key-value retry budget covers that, so the claim is that the -//! platform absorbs it: operations stall and then complete, and no executor -//! is lost. -//! * **S22** (GOL-499) cuts for longer than the retry budget. The budget is -//! then exhausted, the recovery-index write in `status_flusher` panics on -//! purpose, and executors are replaced. The claim is not survival but that -//! the exit is the intended one and nothing is lost across it. +//! * **S16** (GOL-379) cuts the key-value cluster for the length of an AWS +//! storage failover, about a minute. The key-value retry budget covers that, +//! so the claim is that the platform absorbs it: operations stall and then +//! complete, and no executor is lost. +//! * **S22** (GOL-499) cuts the same cluster for longer than that budget. The +//! budget is then exhausted, the recovery-index write in `status_flusher` +//! panics on purpose, and executors are replaced. The claim is not survival +//! but that the exit is the intended one and nothing is lost across it. +//! * **S14** (GOL-376) cuts the *other* Aurora cluster, the one carrying the +//! oplog, again for the length of a failover. golem-dev gives the indexed +//! retry 200 attempts with a 10s cap, so the budget is not the question here +//! the way it is in the other two, and the claim is again absorption. //! -//! The driver is the same either way because the difference is one of -//! expectation, not of choreography. Nothing below asserts on which of the two -//! happened: the account it produces answers both questions, and the oracles -//! that fail the build — the scheduled-fire account and the exactly-once -//! account — are the ones both scenarios share. +//! The driver is the same in all three because the difference is one of +//! expectation, not of choreography. Nothing below asserts on which outcome +//! happened: the account it produces answers all three questions, and the +//! oracles that fail the build — the scheduled-fire account and the +//! exactly-once account — are the ones every one of them shares. //! -//! The first scenario in the suite that breaks something the platform depends -//! on rather than something the platform *is*. Every fault before this one -//! removed a golem process or a link between two of them, and in each case some -//! part of the cluster stayed healthy and could be compared against. Here the -//! executors keep running, keep their shards, keep answering the shard-manager, -//! and simply cannot reach the database underneath them. +//! The first scenarios in the suite that break something the platform depends +//! on rather than something the platform *is*. Every fault before these removed +//! a golem process or a link between two of them, and in each case some part of +//! the cluster stayed healthy and could be compared against. Here the executors +//! keep running, keep their shards, keep answering the shard-manager, and +//! simply cannot reach a database underneath them. //! //! ## What the fault takes away //! -//! More than the name suggests. Reading the golem-dev executor deployment, the -//! key-value Aurora cluster carries four things: +//! Two different halves of the platform, depending on the code, and they are +//! close to mirror images of each other. +//! +//! Reading the golem-dev executor deployment, the **key-value** cluster that +//! S16 and S22 cut carries four things: //! //! * promises, //! * the running-workers set, @@ -51,13 +58,19 @@ //! * and the scheduler, in its own schema on the same cluster. //! //! The oplog lives on a different Aurora cluster and the worker-status hot -//! cache lives in Redis, and the partition touches neither. That combination is -//! what makes the scenario worth running: the platform keeps the ability to -//! *record* what it did and loses the ability to know *what it is doing*. +//! cache lives in Redis, and that partition touches neither. So the platform +//! keeps the ability to *record* what it did and loses the ability to know +//! *what it is doing*. +//! +//! The **indexed** cluster that S14 cuts carries the oplog and nothing else. +//! Promises still resolve, the scheduler still claims and acknowledges on time, +//! the running-workers set is still writable. What goes is the ability to +//! commit anything durable at all, which is the opposite arrangement: the +//! platform knows exactly what it is doing and cannot record any of it. //! -//! It also means no stream here is a control group. A durable increment needs -//! the running-workers set before it can start, so `durable` degrades along -//! with everything else and must not be read as untouched. +//! Either way no stream is a control group. A durable increment needs the +//! running-workers set before it can start and the oplog before it can finish, +//! so `durable` degrades under both cuts and must not be read as untouched. //! //! ## The control is the baseline, not another pod //! @@ -78,17 +91,24 @@ //! The mixed workload's scheduled stream registers through `schedule_poll_at`, //! which increments a counter and records nothing else. That is enough to ask //! "did every registration eventually fire" and useless for asking "how late". -//! Scheduler lag is one of the two things GOL-379 asks this run to record, and -//! the only place a due time survives is the target's own fire log, so S16 -//! drives the token-carrying registration loop from [`crate::chaos::scheduled`] -//! instead and leaves `scheduledAgents` at zero in the mixed workload. Setting -//! both is refused at load time — see `ScenarioConfig::require_storage`. +//! Scheduler lag is one of the two things GOL-379 asks these runs to record, +//! and the only place a due time survives is the target's own fire log, so +//! these scenarios drive the token-carrying registration loop from +//! [`crate::chaos::scheduled`] instead and leave `scheduledAgents` at zero in +//! the mixed workload. Setting both is refused at load time — see +//! `ScenarioConfig::require_storage`. +//! +//! S14 is where that lag reads most directly. Its cut leaves the scheduler on +//! the healthy cluster, so claims and acknowledgements keep their timing and +//! what the delays measure is purely how long the fired invocation could not +//! commit. Under S16 and S22 the scheduler is inside the outage and the two +//! costs are not separable. //! //! One consequence worth stating plainly for anyone reading the result: the -//! fire account's `group` axis is degenerate here. S16 kills no executor, so -//! every target is reported as `elsewhere` and only the `window` axis carries -//! information. The delays to read are the `during-fault` and `after-fault` -//! cells against `before-fault`. +//! fire account's `group` axis is degenerate here. None of these scenarios +//! names a pod to kill, so every target is reported as `elsewhere` and only the +//! `window` axis carries information. The delays to read are the `during-fault` +//! and `after-fault` cells against `before-fault`. //! //! ## What fails the run //! @@ -107,7 +127,8 @@ //! that never came back are both things a human has to look at, and neither is //! improved by turning the job red. //! -//! Run-by-run findings live in the S16 runbook in golem-cloud, not here. +//! Run-by-run findings live in the per-scenario runbooks in golem-cloud, not +//! here. use crate::chaos::fires::{FaultWindow, ScheduleFireReport}; use crate::chaos::history::{OperationHistory, OperationRecord, Outcome, Phase, Stream}; @@ -775,9 +796,10 @@ mod tests { } /// The storage account's findings never reach the termination reason. An - /// outage that failed to land is the loudest thing S16 can report and it is - /// deliberately not fatal: turning the job red would say the platform did - /// something wrong, and what actually went wrong is the experiment. + /// outage that failed to land is the loudest thing these scenarios can + /// report, and it is deliberately not fatal: turning the job red would say + /// the platform did something wrong, and what actually went wrong is the + /// experiment. #[test] fn a_storage_finding_does_not_change_the_termination_reason() { let mut outage = From dde7824ee7dc65ba922ddeb5299dcc951341fcca Mon Sep 17 00:00:00 2001 From: Kaur Matas <33095685+kmatasfp@users.noreply.github.com> Date: Thu, 27 Aug 2026 17:27:48 -0700 Subject: [PATCH 33/40] Add the S18 Redis cache outage scenario --- golem-test-framework/src/benchmark/config.rs | 3 + .../chaos_suites/cloud-chaos.yaml | 104 ++++++++++++++++++ integration-tests/src/benchmarks/all.rs | 2 + integration-tests/src/chaos/mod.rs | 13 ++- .../src/chaos/scenarios/storage_outage.rs | 83 ++++++++++++-- 5 files changed, 191 insertions(+), 14 deletions(-) diff --git a/golem-test-framework/src/benchmark/config.rs b/golem-test-framework/src/benchmark/config.rs index 61d0856587..a8e7f565a6 100644 --- a/golem-test-framework/src/benchmark/config.rs +++ b/golem-test-framework/src/benchmark/config.rs @@ -262,6 +262,9 @@ pub enum ChaosScenarioArg { /// Executors cut off from the indexed-oplog PostgreSQL cluster for the /// length of a writer failover. S14, + /// Executors cut off from the Redis cache in front of the key-value layer, + /// for longer than a caller is willing to wait. + S18, } /// Density subcommand action. diff --git a/integration-tests/chaos_suites/cloud-chaos.yaml b/integration-tests/chaos_suites/cloud-chaos.yaml index f9aedd0a8b..14cb604080 100644 --- a/integration-tests/chaos_suites/cloud-chaos.yaml +++ b/integration-tests/chaos_suites/cloud-chaos.yaml @@ -1195,3 +1195,107 @@ scenarios: delaySecs: 5 signalTimeoutSecs: 1800 + + # --------------------------------------------------------------------------- + # S18 — Redis outage (GOL-384) + # + # The fourth scenario on the storage driver and the first that does not cut a + # database. golem-dev runs the key-value layer as `NamespaceRouted`: the + # `Worker`, `AgentStatus` and `AgentStatusCheckpoint` namespaces go to a Redis + # cache and everything else goes to Postgres. S16 cut the Postgres half and + # left this one standing; S18 cuts exactly what S16 left. + # + # It is sized differently from the other three, and the reason is in the + # config rather than in the ambition. Those three are measured against a + # platform retry budget — S16 and S14 stay inside one, S22 runs past one. This + # client has no budget to run past: golem-dev sets the cache's + # RETRIES__MAX_ATTEMPTS to 0, which the code turns into a fred reconnect + # policy with unlimited attempts, and fred's defaults then give the client no + # command timeout and an unbounded command buffer. A cache write during the + # cut does not fail. It waits. + # + # So the only clock that can run out is the caller's, and the window is set + # against that instead: past the 120s attempt timeout, so operations time out + # and retry under the same idempotency key while the original write is still + # queued inside fred. Two landings of one operation is what this run is for. + # --------------------------------------------------------------------------- + - code: S18 + name: redis-cache-outage + enabled: true + + fault: + kind: network-partition + target: worker-executor + # Every executor, as in the other three: a cache outage only some + # executors could see would be a routing fault wearing a cache's clothes. + mode: all + # Must equal phases.faultSecs; see the note on S16. + durationSecs: 180 + + phases: + # Same as the other three, so all four are comparable. + baselineSecs: 300 + # Past the workload's 120s ATTEMPT_TIMEOUT plus its 5s retry delay, so an + # operation that starts early in the window times out, retries, and has + # that retry still in flight when the heal arrives. At 60s nothing would + # time out and the run would only show slower operations completing, which + # the config already tells us without spending a run. + faultSecs: 180 + # As the others. The drain here is larger than S14's — three minutes of + # coalesced status writes and queued commands come back at once — so this + # needs to cover a burst rather than a trickle. + recoverySecs: 420 + + workload: + # Identical to S16 and S14, deliberately, so the difference between the + # four results is the fault and not the population. + durableAgents: 200 + ephemeralAgents: 50 + # Zero for the same reason as the others: the scheduled stream comes from + # the block below. Setting both is refused at load time. + scheduledAgents: 0 + # Expected to be one of the two streams that actually stops. Completing a + # promise suspends and wakes an agent, and a suspend is a lifecycle + # boundary, which is one of the few things that still crosses this cache + # synchronously. + promiseAgents: 50 + quotaAgents: 0 + ratePerSec: 100 + + scheduled: + targets: 100 + intervalMillis: 2000 + leadSecs: 10 + # Raised from the 240 the other three use, because the window itself is + # 180s. A fire at the start of the outage is delayed by roughly the whole + # window before it can make progress, and then joins a drain queue behind + # three minutes of backlog. 300 leaves headroom for that tail without + # making a delay the platform should be ashamed of look acceptable. + leaseBudgetSecs: 300 + + storage: + # The Redis primary endpoint, not either Aurora writer. Both Postgres + # clusters stay reachable for the whole run. + endpoint: master.golem-redis-dev.intsie.use1.cache.amazonaws.com + # Same floor as the other three, and it means something slightly + # different here. The degradation is expected to be partial: `ephemeral` + # and `promise` should stop, `durable` may keep serving from memory + # because the status blob was taken off the commit path on purpose. The + # floor is checked against the quietest stream, so it still reports + # whether the cut landed even when the run-wide share stays high. + outageQuietFloorPercent: 50 + # Recorded, not asserted. Expected to be larger than S14's: nothing is + # replaced, but three minutes of coalesced status writes and buffered + # commands all arrive at once when the connection returns. + recoveryBudgetSecs: 180 + + retryPolicy: + # Identical to the other three, and load-bearing here rather than + # incidental. Retrying transport errors once under the same idempotency + # key is the whole mechanism by which a stalled operation and its retry + # can both land, which is the question the run exists to answer. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 diff --git a/integration-tests/src/benchmarks/all.rs b/integration-tests/src/benchmarks/all.rs index 3c069a38e3..8a0cabea7c 100644 --- a/integration-tests/src/benchmarks/all.rs +++ b/integration-tests/src/benchmarks/all.rs @@ -601,6 +601,7 @@ async fn run_chaos( ChaosScenarioArg::S16 => chaos::ScenarioCode::S16, ChaosScenarioArg::S22 => chaos::ScenarioCode::S22, ChaosScenarioArg::S14 => chaos::ScenarioCode::S14, + ChaosScenarioArg::S18 => chaos::ScenarioCode::S18, }; let config = suite .scenario(code, allow_disabled) @@ -652,6 +653,7 @@ async fn run_chaos( } code @ (chaos::ScenarioCode::S14 | chaos::ScenarioCode::S16 + | chaos::ScenarioCode::S18 | chaos::ScenarioCode::S22) => { chaos::scenarios::storage_outage::run( code, &config, &manifest, &deps, &signals, &outputs, diff --git a/integration-tests/src/chaos/mod.rs b/integration-tests/src/chaos/mod.rs index 0e06323f16..43cf0c8855 100644 --- a/integration-tests/src/chaos/mod.rs +++ b/integration-tests/src/chaos/mod.rs @@ -99,6 +99,9 @@ pub enum ScenarioCode { /// Executors cut off from the indexed-oplog PostgreSQL cluster, the other /// Aurora cluster underneath them, for the length of a writer failover. S14, + /// Executors cut off from the Redis cache that fronts the key-value layer, + /// for longer than a caller is willing to wait. + S18, } impl ScenarioCode { @@ -118,13 +121,14 @@ impl ScenarioCode { ScenarioCode::S16 => "S16", ScenarioCode::S22 => "S22", ScenarioCode::S14 => "S14", + ScenarioCode::S18 => "S18", } } /// Every scenario this driver implements. The suite YAML is checked against /// this list, so a scenario cannot be enabled in YAML without code behind /// it, nor implemented without an operational switch in front of it. - pub const ALL: [ScenarioCode; 14] = [ + pub const ALL: [ScenarioCode; 15] = [ ScenarioCode::S1, ScenarioCode::S3, ScenarioCode::S5, @@ -138,6 +142,7 @@ impl ScenarioCode { ScenarioCode::S13, ScenarioCode::S14, ScenarioCode::S16, + ScenarioCode::S18, ScenarioCode::S22, ]; @@ -1450,6 +1455,7 @@ mod tests { assert_eq!(ScenarioCode::parse("s9"), Some(ScenarioCode::S9)); assert_eq!(ScenarioCode::parse("s16"), Some(ScenarioCode::S16)); assert_eq!(ScenarioCode::parse("s14"), Some(ScenarioCode::S14)); + assert_eq!(ScenarioCode::parse("s18"), Some(ScenarioCode::S18)); assert_eq!(ScenarioCode::parse("S99"), None); } @@ -1495,7 +1501,10 @@ mod tests { entry.require_workload().unwrap(); entry.require_rollback().unwrap(); } - ScenarioCode::S14 | ScenarioCode::S16 | ScenarioCode::S22 => { + ScenarioCode::S14 + | ScenarioCode::S16 + | ScenarioCode::S18 + | ScenarioCode::S22 => { entry.require_workload().unwrap(); entry.require_scheduled().unwrap(); entry.require_storage().unwrap(); diff --git a/integration-tests/src/chaos/scenarios/storage_outage.rs b/integration-tests/src/chaos/scenarios/storage_outage.rs index a4b6954a4b..84e4e87045 100644 --- a/integration-tests/src/chaos/scenarios/storage_outage.rs +++ b/integration-tests/src/chaos/scenarios/storage_outage.rs @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Storage outage: the shared choreography behind S14, S16 and S22. +//! Storage outage: the shared choreography behind S14, S16, S18 and S22. //! -//! All three codes run this module. They differ in which cluster is taken away -//! and for how long, both of which are suite settings, and in what a reader -//! should expect of the result: +//! All four codes run this module. They differ in which store is taken away and +//! for how long, both of which are suite settings, and in what a reader should +//! expect of the result: //! //! * **S16** (GOL-379) cuts the key-value cluster for the length of an AWS //! storage failover, about a minute. The key-value retry budget covers that, @@ -30,10 +30,15 @@ //! oplog, again for the length of a failover. golem-dev gives the indexed //! retry 200 attempts with a 10s cap, so the budget is not the question here //! the way it is in the other two, and the claim is again absorption. +//! * **S18** (GOL-384) cuts neither Aurora cluster but the Redis cache in front +//! of the key-value layer, and holds it past the caller's patience rather +//! than past any platform budget. There is no budget to exhaust: golem-dev +//! configures that client to retry forever. The claim is that a stall which +//! outlasts the caller does not turn one operation into two. //! -//! The driver is the same in all three because the difference is one of +//! The driver is the same in all four because the difference is one of //! expectation, not of choreography. Nothing below asserts on which outcome -//! happened: the account it produces answers all three questions, and the +//! happened: the account it produces answers all four questions, and the //! oracles that fail the build — the scheduled-fire account and the //! exactly-once account — are the ones every one of them shares. //! @@ -46,8 +51,9 @@ //! //! ## What the fault takes away //! -//! Two different halves of the platform, depending on the code, and they are -//! close to mirror images of each other. +//! Three different pieces of the platform, depending on the code. The two +//! Aurora cuts are close to mirror images of each other; the Redis cut is a +//! third thing again. //! //! Reading the golem-dev executor deployment, the **key-value** cluster that //! S16 and S22 cut carries four things: @@ -68,15 +74,22 @@ //! commit anything durable at all, which is the opposite arrangement: the //! platform knows exactly what it is doing and cannot record any of it. //! -//! Either way no stream is a control group. A durable increment needs the -//! running-workers set before it can start and the oplog before it can finish, -//! so `durable` degrades under both cuts and must not be read as untouched. +//! The **Redis cache** that S18 cuts is not a cluster at all but the front half +//! of the key-value layer. `NamespaceRoutedKeyValueStorage` sends the `Worker`, +//! `AgentStatus` and `AgentStatusCheckpoint` namespaces to it and everything +//! else to Postgres, so S18 removes exactly the part S16 leaves standing. Both +//! Aurora clusters stay reachable throughout. +//! +//! No stream is a control group under any of them. A durable increment needs +//! the running-workers set before it can start, the worker-status cache to +//! resolve its mode, and the oplog before it can finish, so `durable` degrades +//! under all three cuts and must not be read as untouched. //! //! ## The control is the baseline, not another pod //! //! S1 and S3 keep executors on the healthy side of the cut and read the verdict //! off the disagreement between the two groups. There is no healthy side here: -//! all three executors share one cluster. So the comparison runs along time +//! all three executors share one store. So the comparison runs along time //! instead, and every stream is measured against its own before-fault rate. See //! [`crate::chaos::outage`] for what that costs and what it still answers. //! @@ -86,6 +99,52 @@ //! so [`OutageViolation::OutageNotObserved`] is a named finding rather than //! something a reader is left to infer from the cells. //! +//! ## Why S18 is held longer than the others +//! +//! The first three scenarios are sized against a platform budget: S16 and S14 +//! stay inside one, S22 deliberately runs past it. S18 has no budget to run +//! past. golem-dev sets +//! `GOLEM__KEY_VALUE_STORAGE__CONFIG__CACHE__CONFIG__RETRIES__MAX_ATTEMPTS` to +//! `0`, which `RedisPool::configured` hands to fred as a `ReconnectPolicy` with +//! unlimited attempts, and it passes no performance or connection config, so +//! fred's defaults apply: no command timeout and an unbounded command buffer. +//! A cache write during the cut therefore never fails. It waits, for as long as +//! the cut lasts. +//! +//! That makes the `unwrap_or_else(|err| panic!(...))` on the cache path in +//! `WorkerService` unreachable through this fault, and it makes a short cut +//! uninformative: at 60s every caller is still inside its 120s attempt timeout, +//! so the run would only show operations taking a minute longer and completing. +//! The window is set past that timeout instead, so callers give up and retry +//! under the same idempotency key while the original write is still sitting in +//! fred's buffer. Whether those two land as one operation or two is the +//! question the scenario exists to answer, and the exactly-once account is +//! where it shows. +//! +//! ## What is expected to stall, and what is not +//! +//! Less than the whole platform, and this is the part worth reading the result +//! carefully for. `AgentStatusFlusher` took the status blob off the commit path +//! deliberately: a status change only marks the agent dirty, and a background +//! sweeper coalesces the writes. So a durable agent that is already resident +//! commits to the oplog without touching Redis synchronously, and the blob it +//! cannot flush is derivable from the oplog anyway, which is what makes the +//! staleness safe rather than merely tolerated. +//! +//! What does cross Redis synchronously is a lifecycle boundary — suspend, evict, +//! reattach — and a `get_agent_mode` miss. So the prediction is a *partial* +//! degradation: `ephemeral`, whose agents are created and torn down per +//! operation, and `promise`, whose agents suspend, should go quiet; `durable` +//! may keep serving from memory throughout. +//! +//! Two consequences for reading the report. `shareOfBaselinePercent` should sit +//! higher here than in S16 or S14 without that meaning the fault was weaker, +//! and `quietestStreamPercent` is the number that shows the cut landed, because +//! it reports the stream that stopped rather than the average of one that did +//! and one that did not. Whether a durable agent that kept committing against a +//! frozen status cache still recovers correctly is the second open question of +//! the run, after the exactly-once one. +//! //! ## Why the scheduled stream is driven separately //! //! The mixed workload's scheduled stream registers through `schedule_poll_at`, From 06e38c6c68cdb22eb952cbe79871b51ca85b427b Mon Sep 17 00:00:00 2001 From: Kaur Matas <33095685+kmatasfp@users.noreply.github.com> Date: Thu, 27 Aug 2026 20:16:12 -0700 Subject: [PATCH 34/40] Give each storage scenario its own outage expectation --- .../chaos_suites/cloud-chaos.yaml | 59 ++- integration-tests/src/chaos/mod.rs | 217 +++++++-- integration-tests/src/chaos/outage.rs | 416 ++++++++++++++++-- .../src/chaos/scenarios/storage_outage.rs | 55 ++- 4 files changed, 639 insertions(+), 108 deletions(-) diff --git a/integration-tests/chaos_suites/cloud-chaos.yaml b/integration-tests/chaos_suites/cloud-chaos.yaml index 14cb604080..6bd37f5ce1 100644 --- a/integration-tests/chaos_suites/cloud-chaos.yaml +++ b/integration-tests/chaos_suites/cloud-chaos.yaml @@ -992,7 +992,13 @@ scenarios: # same thing whatever the window is. 50% is far from both outcomes it has # to tell apart: a landed outage is quiet for over 90% of the window, an # undisturbed cluster for a few percent. - outageQuietFloorPercent: 50 + expect: + # Everything is behind this cut, so every stream that was serving is + # expected to stop. See `OutageExpectation` for why the rule is named + # per scenario rather than shared: S18 cuts a store only part of the + # workload touches, and this rule reads that arrangement backwards. + kind: whole-workload + quietFloorPercent: 50 # What serving again may cost once the cluster is reachable. Recorded, not # asserted: this is really a measure of how long sqlx's pool takes to # notice, plus whatever recovery scan the executors run, and how long that @@ -1081,9 +1087,12 @@ scenarios: storage: endpoint: golem-postgres-dev-keyvalue.cluster-cgfyoqmjq7tc.us-east-1.rds.amazonaws.com - # Same floor as S16, and it should be met even more comfortably: an - # outage this long leaves the streams silent for nearly the whole window. - outageQuietFloorPercent: 50 + # Same rule and floor as S16, and it should be met even more comfortably: + # an outage this long leaves the streams silent for nearly the whole + # window. + expect: + kind: whole-workload + quietFloorPercent: 50 # Recorded, not asserted, as everywhere else — but read differently here. # In S16 this measures a connection pool noticing. Here it measures a # cluster rebuilding itself, and it is the number that says whether the @@ -1181,8 +1190,10 @@ scenarios: # The indexed writer endpoint, not the key-value one. This is the only # field that differs materially from S16. endpoint: golem-postgres-dev-indexed.cluster-cgfyoqmjq7tc.us-east-1.rds.amazonaws.com - # Same floor and same reasoning as S16. - outageQuietFloorPercent: 50 + # Same rule, floor and reasoning as S16. + expect: + kind: whole-workload + quietFloorPercent: 50 # Recorded, not asserted. Expected to be small: nothing is replaced here, # so this is a connection pool noticing its database is back plus whatever # the stalled commits still have to do. @@ -1277,13 +1288,35 @@ scenarios: # The Redis primary endpoint, not either Aurora writer. Both Postgres # clusters stay reachable for the whole run. endpoint: master.golem-redis-dev.intsie.use1.cache.amazonaws.com - # Same floor as the other three, and it means something slightly - # different here. The degradation is expected to be partial: `ephemeral` - # and `promise` should stop, `durable` may keep serving from memory - # because the status blob was taken off the commit path on purpose. The - # floor is checked against the quietest stream, so it still reports - # whether the cut landed even when the run-wide share stays high. - outageQuietFloorPercent: 50 + # The one scenario in this family that does not use the whole-workload + # rule, because this cut is partial by construction and that rule reads a + # partial cut backwards. Run 33130077355 proved it: `durable` held 100% + # of its baseline throughout, exactly as NamespaceRoutedKeyValueStorage + # says it should, and the shared rule reported the partition as one that + # never landed. + # + # `ephemeral` is the only stream named. Its agents are created and torn + # down per operation, so every one of them crosses a lifecycle boundary, + # which is one of the few things that still reaches this cache + # synchronously. It went silent for 99.997% of that first window. + # + # `promise` was expected to be the second and was not: the mixed + # workload's promise stream is get_promise+complete in one round trip + # against a durable agent and never suspends. The suspending variant is + # `promise-wait`, which S11 drives and this scenario does not enable. + # + # The serving floor is the half a shared rule cannot state. Durable, + # scheduled and promise are claimed to be off this fault path because of + # where the code routes their namespaces, and that claim is worth + # asserting: a stall there would mean the status blob is not off the + # commit path the way AgentStatusFlusher describes. All three held + # 99.94-100.06% on the first run, so 50 catches a real regression without + # tripping on noise. + expect: + kind: partial-workload + silenced: [ephemeral] + quietFloorPercent: 50 + servingFloorPercent: 50 # Recorded, not asserted. Expected to be larger than S14's: nothing is # replaced, but three minutes of coalesced status writes and buffered # commands all arrive at once when the connection returns. diff --git a/integration-tests/src/chaos/mod.rs b/integration-tests/src/chaos/mod.rs index 43cf0c8855..92ff422862 100644 --- a/integration-tests/src/chaos/mod.rs +++ b/integration-tests/src/chaos/mod.rs @@ -59,6 +59,7 @@ pub mod waiters; pub mod wakeups; pub mod workload; +use crate::chaos::history::Stream; use anyhow::Context; use serde::{Deserialize, Serialize}; use std::path::Path; @@ -687,31 +688,16 @@ pub struct StorageConfig { /// hostname. Chaos Mesh resolves it in the controller, so this is the same /// string that appears in the NetworkChaos manifest's `externalTargets`. pub endpoint: String, - /// The least of the fault window every stream must answer nothing at all - /// for, as a percentage of that window, for the outage to count as - /// observed. - /// - /// A run below this line did not take the storage away, whatever the fault - /// status said, and every other number in the report then describes an - /// undisturbed cluster. That is reported as inconclusive rather than clean: - /// a healthy-looking result from a fault that never landed is the worst - /// artifact this suite can produce. - /// - /// This used to be a ceiling on during-fault throughput as a share of - /// baseline, and that number is still recorded. It stopped being the - /// verdict because it is a rate averaged over the whole window while all - /// the serving in an absorbed outage happens in the seconds at its edges, - /// so the same handful of edge confirmations reads as 8% of a 180s window - /// and 26% of a 60s one. The threshold then tracks the window length rather - /// than the platform, and shortening S16's window to 60s duly tripped it on - /// a partition that had plainly landed. + /// What this scenario expects its cut to do to the workload, and therefore + /// what the run treats as evidence the cut landed. /// - /// Quiet time has no such coupling: it is measured against the window's own - /// edges, so it means the same thing whatever the window is. Every stream - /// is judged rather than the aggregate, because a stream still answering is - /// a fault that did not land on it and must not be outvoted by quieter - /// ones. - pub outage_quiet_floor_percent: f64, + /// Deliberately per-scenario rather than one shared rule. The *account* the + /// driver produces is shared because it is factual: throughput per stream + /// per window, quiet time, latency, what was caught in flight. The + /// *verdict* is not, because what a cut is supposed to do depends on what + /// it cut, and a rule that reads correctly for one arrangement can be + /// confidently wrong about another. See [`OutageExpectation`]. + pub expect: OutageExpectation, /// What serving again may cost once the storage is reachable, and the /// number each stream's recovery gap is reported against. Recorded rather /// than asserted, like every other budget in the suite: how long a @@ -720,6 +706,112 @@ pub struct StorageConfig { pub recovery_budget_secs: u64, } +/// What a storage cut is expected to do to the workload. +/// +/// The two variants exist because the storage scenarios cut two different +/// *kinds* of thing, not two instances of one thing. S16, S22 and S14 each take +/// away a database every stream depends on, so "did anything keep serving" is a +/// sound test of whether the fault landed. S18 takes away a cache only part of +/// the workload touches, and under that arrangement the same rule is not merely +/// too strict — it is backwards. It reported S18's first run as an outage that +/// never happened while the partition had plainly landed, because `durable` +/// held 100% of its baseline throughout, exactly as the design says it should. +/// +/// Parameterising the old rule with a stream list would have fixed that one +/// message and left the deeper problem: for a partial cut, the streams that +/// *keep working* carry as much information as the ones that stop. If `durable` +/// had gone quiet under S18, the status blob would not be off the commit path +/// the way `AgentStatusFlusher` claims, and the shared rule would have called +/// that a clean pass. So the partial variant asserts both halves. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde( + tag = "kind", + rename_all = "kebab-case", + rename_all_fields = "camelCase" +)] +pub enum OutageExpectation { + /// Everything is behind the cut, so every stream that was serving before it + /// must fall silent. S16, S22 and S14. + WholeWorkload { + /// The least of the fault window each stream must answer nothing at all + /// for, as a percentage of that window. + /// + /// This used to be a ceiling on during-fault throughput as a share of + /// baseline, and that number is still recorded. It stopped being the + /// verdict because it is a rate averaged over the whole window while + /// all the serving in an absorbed outage happens in the seconds at its + /// edges, so the same handful of edge confirmations reads as 8% of a + /// 180s window and 26% of a 60s one. The threshold then tracks the + /// window length rather than the platform, and shortening S16's window + /// to 60s duly tripped it on a partition that had plainly landed. + /// + /// Quiet time has no such coupling: it is measured against the window's + /// own edges, so it means the same thing whatever the window is. + quiet_floor_percent: f64, + }, + /// Only part of the workload is behind the cut. The named streams must fall + /// silent and every other stream that was serving before it must keep + /// serving. S18. + PartialWorkload { + /// The streams this cut is expected to stop. Their silence is the + /// evidence the fault landed, so naming a stream the workload never + /// drives would leave the run with no such evidence and no complaint — + /// which `require_storage` refuses at load time rather than discovering + /// during a maintenance window. + silenced: Vec, + /// As [`Self::WholeWorkload::quiet_floor_percent`], but applied only to + /// the streams named above. + quiet_floor_percent: f64, + /// The least of its own baseline rate an *unnamed* stream must still + /// hold during the cut. + /// + /// This is the half a shared rule cannot express. A partial cut makes a + /// claim about the streams it does not touch, and that claim is worth + /// asserting: it is derived from where the code routes each namespace, + /// so a stream stalling here says the routing is not what the source + /// says it is. + serving_floor_percent: f64, + }, +} + +impl OutageExpectation { + /// The quiet floor, which both variants carry and apply to a different set. + pub fn quiet_floor_percent(&self) -> f64 { + match self { + OutageExpectation::WholeWorkload { + quiet_floor_percent, + } + | OutageExpectation::PartialWorkload { + quiet_floor_percent, + .. + } => *quiet_floor_percent, + } + } + + /// Whether this stream is one whose silence would prove the cut landed. + /// + /// `WholeWorkload` says yes to everything, which is what makes it the + /// stricter rule rather than merely a different one. + pub fn expects_silence(&self, stream: Stream) -> bool { + match self { + OutageExpectation::WholeWorkload { .. } => true, + OutageExpectation::PartialWorkload { silenced, .. } => silenced.contains(&stream), + } + } + + /// The floor an unnamed stream's during-fault throughput must clear, or + /// `None` where the expectation makes no claim about streams still serving. + pub fn serving_floor_percent(&self) -> Option { + match self { + OutageExpectation::WholeWorkload { .. } => None, + OutageExpectation::PartialWorkload { + serving_floor_percent, + .. + } => Some(*serving_floor_percent), + } + } +} + impl StorageConfig { pub fn recovery_budget(&self) -> Duration { Duration::from_secs(self.recovery_budget_secs) @@ -967,6 +1059,29 @@ impl ScenarioConfig { } /// The storage-outage block. See [`Self::require_workload`]. + /// Whether this scenario's configuration actually produces operations on a + /// stream. + /// + /// Counts the blocks that drive traffic rather than the streams a run + /// happens to record, so it can be answered from the YAML alone before any + /// cluster time is spent. `Scheduled` has two possible sources and either + /// one counts; the streams that belong to other scenario shapes are not + /// reachable from a `storage` scenario's config at all. + fn drives_stream(&self, stream: Stream) -> bool { + let workload = self.workload.as_ref(); + match stream { + Stream::Durable => workload.is_some_and(|w| w.durable_agents > 0), + Stream::Ephemeral => workload.is_some_and(|w| w.ephemeral_agents > 0), + Stream::Promise => workload.is_some_and(|w| w.promise_agents > 0), + Stream::Quota => workload.is_some_and(|w| w.quota_agents > 0), + Stream::Scheduled => { + workload.is_some_and(|w| w.scheduled_agents > 0) + || self.scheduled.as_ref().is_some_and(|s| s.targets > 0) + } + Stream::PinnedHttp | Stream::PromiseWait | Stream::Delete | Stream::Revert => false, + } + } + pub fn require_storage(&self) -> anyhow::Result<&StorageConfig> { let config = self.storage.as_ref().ok_or_else(|| { anyhow::anyhow!( @@ -981,16 +1096,47 @@ impl ScenarioConfig { // that exists to catch a fault which never landed would be stuck on. A // floor of zero is the opposite failure: every run passes, including // the ones where nothing was ever cut off. - if !(0.0..100.0).contains(&config.outage_quiet_floor_percent) - || config.outage_quiet_floor_percent <= 0.0 + let quiet_floor = config.expect.quiet_floor_percent(); + if !(0.0..100.0).contains(&quiet_floor) || quiet_floor <= 0.0 { + anyhow::bail!( + "chaos scenario {}: expect.quietFloorPercent is {quiet_floor}, which is not a \ + share of the fault window a real outage could be judged by", + self.code + ); + } + if let Some(serving_floor) = config.expect.serving_floor_percent() + && (!(0.0..=100.0).contains(&serving_floor) || serving_floor <= 0.0) { anyhow::bail!( - "chaos scenario {}: outageQuietFloorPercent is {}, which is not a share of the \ - fault window a real outage could be judged by", - self.code, - config.outage_quiet_floor_percent + "chaos scenario {}: expect.servingFloorPercent is {serving_floor}, which is not a \ + share of a stream's own baseline it could be held to", + self.code ); } + // A partial cut proves it landed by naming the streams it stops, so a + // name the workload never drives leaves the run with no evidence and no + // complaint — the exact failure the quiet floor exists to prevent, + // reintroduced one level up. Refused here rather than at judge time so + // it costs a build rather than a maintenance window. + if let OutageExpectation::PartialWorkload { silenced, .. } = &config.expect { + if silenced.is_empty() { + anyhow::bail!( + "chaos scenario {}: expect.silenced is empty, so nothing in the run would \ + show whether the cut landed at all", + self.code + ); + } + for stream in silenced { + if !self.drives_stream(*stream) { + anyhow::bail!( + "chaos scenario {}: expect.silenced names `{stream}`, which this \ + scenario's workload never drives, so its silence during the fault \ + would prove nothing", + self.code + ); + } + } + } if config.endpoint.trim().is_empty() { anyhow::bail!( "chaos scenario {}: storage.endpoint is empty, so the result could not say which \ @@ -1312,7 +1458,9 @@ mod tests { rollback: None, storage: Some(StorageConfig { endpoint: endpoint.to_string(), - outage_quiet_floor_percent: quiet_floor, + expect: OutageExpectation::WholeWorkload { + quiet_floor_percent: quiet_floor, + }, recovery_budget_secs: 120, }), revert: None, @@ -1333,7 +1481,7 @@ mod tests { .unwrap_err() .to_string(); assert!( - error.contains("outageQuietFloorPercent"), + error.contains("expect.quietFloorPercent"), "the message has to name the knob, got: {error}" ); } @@ -1501,10 +1649,7 @@ mod tests { entry.require_workload().unwrap(); entry.require_rollback().unwrap(); } - ScenarioCode::S14 - | ScenarioCode::S16 - | ScenarioCode::S18 - | ScenarioCode::S22 => { + ScenarioCode::S14 | ScenarioCode::S16 | ScenarioCode::S18 | ScenarioCode::S22 => { entry.require_workload().unwrap(); entry.require_scheduled().unwrap(); entry.require_storage().unwrap(); diff --git a/integration-tests/src/chaos/outage.rs b/integration-tests/src/chaos/outage.rs index 37b6ad3ca5..0a0d857558 100644 --- a/integration-tests/src/chaos/outage.rs +++ b/integration-tests/src/chaos/outage.rs @@ -54,25 +54,61 @@ //! on the same cluster. The oplog is on a different Aurora cluster and the //! worker-status hot cache is in Redis, and neither is touched here. //! -//! That is why every stream degrades rather than only the obviously storage-shaped -//! ones. A durable increment needs the running-workers set before it can run at -//! all, so `durable` is not a control group and must not be read as one. +//! That is why every stream degrades under that cut rather than only the +//! obviously storage-shaped ones. A durable increment needs the running-workers +//! set before it can run at all, so `durable` is not a control group there and +//! must not be read as one. +//! +//! S18 cuts the Redis half instead, and the picture inverts. Only `Worker`, +//! `AgentStatus` and `AgentStatusCheckpoint` live there, the status blob is +//! written off the commit path by a background sweeper, and what still crosses +//! the cache synchronously is a lifecycle boundary or a `get_agent_mode` miss. +//! So `ephemeral` stops and `durable`, `scheduled` and `promise` carry on, and +//! the streams still answering are evidence rather than noise. //! //! ### What fails the run //! -//! Two things, and both are statements about the experiment rather than about +//! Three things, and all are statements about the experiment rather than about //! latency: //! -//! * [`OutageViolation::OutageNotObserved`] — the workload kept working, so the -//! fault did not land where the run says it did. +//! * [`OutageViolation::OutageNotObserved`] — a stream the cut was supposed to +//! stop kept working, so the fault did not land where the run says it did. +//! * [`OutageViolation::UnexpectedStall`] — a stream the cut was *not* supposed +//! to touch stopped anyway. //! * [`OutageViolation::StreamNeverRecovered`] — a stream that was working //! before the outage produced nothing at all after the heal. //! +//! ### Why the verdict is per scenario and the account is not +//! +//! Everything above the verdict is factual and shared: throughput per stream +//! per window, quiet time, latency, what the fault caught in flight. The +//! verdict is supplied by the scenario, as an +//! [`OutageExpectation`](crate::chaos::OutageExpectation), because what a cut +//! is supposed to do depends on what it cut. +//! +//! This was one rule until S18. That rule — every stream must fall silent — +//! is sound for S16, S22 and S14, where a database every stream depends on goes +//! away. S18 cuts a cache only part of the workload touches, and there the same +//! rule is not merely too strict but backwards: it reported S18's first run as +//! an outage that never landed, on the grounds that `durable` held 100% of its +//! baseline, which is exactly what the routing in +//! `NamespaceRoutedKeyValueStorage` says should happen. +//! +//! Parameterising the old rule with a list of streams to exempt would have +//! silenced that message and left the deeper gap. Under a partial cut the +//! streams that keep working carry as much information as the ones that stop: +//! had `durable` gone quiet under S18, the status blob would not be off the +//! commit path the way `AgentStatusFlusher` describes, and a rule that only +//! looks for silence would have called that its cleanest possible pass. So the +//! partial expectation asserts both halves and the whole-workload one keeps +//! asserting the single half that is all it can know. +//! //! Recovery time is recorded against the configured budget and never asserted //! on, like every other budget in the suite. How long a connection pool may //! take to notice its database is back is a judgement, and the number is in the //! result either way. +use crate::chaos::OutageExpectation; use crate::chaos::errors::ErrorClass; use crate::chaos::history::{OperationRecord, Outcome, Stream}; use crate::chaos::split::{ @@ -93,6 +129,17 @@ pub enum OutageViolation { /// executors could still reach the database, and every other number in this /// report describes an undisturbed cluster. OutageNotObserved, + /// A stream a partial cut was *not* supposed to touch stopped serving + /// anyway. + /// + /// The counterpart to [`Self::OutageNotObserved`], and the reason the + /// storage scenarios stopped sharing one rule. A partial cut is a claim + /// about routing: these namespaces go to the store being cut and those go + /// elsewhere. If a stream on the far side stalls, the routing is not what + /// the source says it is, and the older rule — which asked only whether + /// everything went quiet — would have called that its cleanest possible + /// pass. + UnexpectedStall, /// A stream that was confirming operations before the outage confirmed /// nothing at all after the heal. StreamNeverRecovered, @@ -102,6 +149,7 @@ impl OutageViolation { pub fn as_str(self) -> &'static str { match self { OutageViolation::OutageNotObserved => "outage-not-observed", + OutageViolation::UnexpectedStall => "unexpected-stall", OutageViolation::StreamNeverRecovered => "stream-never-recovered", } } @@ -236,10 +284,11 @@ pub struct StorageOutageReport { /// recorded so an archived result says which storage the run was about /// rather than leaving it to the scenario name. pub endpoint: String, - /// The thresholds from the suite YAML, recorded so an archived cell can be - /// read years later against the numbers it was judged by rather than - /// against today's config. - pub outage_quiet_floor_percent: f64, + /// The rule from the suite YAML, recorded whole so an archived cell can be + /// read years later against what it was judged by rather than against + /// today's config — including *which* rule, since the storage scenarios no + /// longer share one. + pub expect: OutageExpectation, pub recovery_budget_ms: u64, /// The whole workload's during-fault rate as a share of its own baseline. /// `None` for a run that never learned when the fault was. @@ -249,10 +298,26 @@ pub struct StorageOutageReport { /// behaves identically. See `outage_quiet_floor_percent`. #[serde(default, skip_serializing_if = "Option::is_none")] pub share_of_baseline_percent: Option, - /// The least any one stream stayed silent during the fault, as a share of - /// that window. This is what the verdict is drawn from. + /// The least any stream *expected to stop* stayed silent during the fault, + /// as a share of that window. This is what the quiet half of the verdict is + /// drawn from. + /// + /// Under `WholeWorkload` that is every stream, so this is the run-wide + /// minimum and means what it always did. Under `PartialWorkload` it covers + /// only the named streams, because the others are expected to keep + /// answering and their quiet time says nothing about whether the cut + /// landed. #[serde(default, skip_serializing_if = "Option::is_none")] pub quietest_stream_percent: Option, + /// The least of its own baseline any stream *expected to keep serving* held + /// during the fault. `None` under `WholeWorkload`, which expects none to. + /// + /// The other half of a partial cut's verdict, and the half no shared rule + /// could state: a partial cut claims the streams it does not touch carry on, + /// and that claim comes from where the code routes each namespace rather + /// than from an assumption about faults. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub least_serving_stream_percent: Option, pub cells: Vec, /// What the outage began underneath, per stream. Empty for a run that never /// learned when the fault was. @@ -293,7 +358,7 @@ impl StorageOutageReport { records: &[OperationRecord], fault: Option, endpoint: &str, - outage_quiet_floor_percent: f64, + expect: OutageExpectation, recovery_budget: Duration, ) -> Self { let mut tallies: BTreeMap<(Stream, Window), Tally> = BTreeMap::new(); @@ -396,10 +461,11 @@ impl StorageOutageReport { let mut report = Self { endpoint: endpoint.to_string(), - outage_quiet_floor_percent, + expect, recovery_budget_ms: recovery_budget.as_millis().min(u64::MAX as u128) as u64, share_of_baseline_percent: None, quietest_stream_percent: None, + least_serving_stream_percent: None, cells, caught_in_flight: caught_in_flight(records, fault), recovery: Vec::new(), @@ -469,33 +535,92 @@ impl StorageOutageReport { .map(|c| c.stream) .collect(); - let mut quiet: Vec<(Stream, f64)> = self + // Split by what this scenario claims the cut does, not by what a cut + // does in general. The two sets are judged by opposite tests, and a + // stream in neither — one that never served before the fault — is + // judged by neither, because it has no baseline to be read against. + let during: Vec<&StreamThroughputCell> = self .cells .iter() .filter(|c| c.window == Window::DuringFault && c.window_secs > 0.0) .filter(|c| served_before.contains(&c.stream)) - .filter_map(|c| { - c.quiet_ms - .map(|ms| (c.stream, round2(ms as f64 / (c.window_secs * 10.0)))) + .collect(); + + // Driven from `served_before` for the same reason as the serving list + // below: a stream that answered nothing at all leaves no during-fault + // cell, and that is total silence rather than missing data. Scoring the + // two sides of one fact differently — absence as 0% served but as + // "unknown" quiet — would let the same run read as a stall on one check + // and as no evidence on the other. + let mut quiet: Vec<(Stream, f64)> = served_before + .iter() + .copied() + .filter(|stream| self.expect.expects_silence(*stream)) + .filter_map(|stream| match during.iter().find(|c| c.stream == stream) { + Some(cell) => cell + .quiet_ms + .map(|ms| (stream, round2(ms as f64 / (cell.window_secs * 10.0)))), + None => Some((stream, 100.0)), }) .collect(); quiet.sort_by(|a, b| a.1.total_cmp(&b.1)); - let Some(&(stream, quietest)) = quiet.first() else { + if let Some(&(stream, quietest)) = quiet.first() { + self.quietest_stream_percent = Some(quietest); + let floor = self.expect.quiet_floor_percent(); + if quietest < floor { + self.findings.push(OutageFinding { + violation: OutageViolation::OutageNotObserved, + stream: Some(stream), + detail: format!( + "{stream} was expected to stop while {} was unreachable and kept \ + answering instead, silent for only {quietest}% of the fault window \ + against a {floor}% floor — so this run has no evidence the cut landed", + self.endpoint + ), + }); + } + } + + let Some(serving_floor) = self.expect.serving_floor_percent() else { return; }; - self.quietest_stream_percent = Some(quietest); - if quietest < self.outage_quiet_floor_percent { - self.findings.push(OutageFinding { - violation: OutageViolation::OutageNotObserved, - stream: Some(stream), - detail: format!( - "{stream} kept answering through the fault window, silent for only \ - {quietest}% of it against a {}% floor, while {} was supposed to be \ - unreachable; the executors could still reach it", - self.outage_quiet_floor_percent, self.endpoint - ), - }); + // Driven from the streams that served *before* the cut rather than from + // the during-fault cells, because the worst case leaves no such cell at + // all: a stream whose operations all stall produces nothing to tally, + // and reading the cells alone would let total silence escape the one + // check that exists to catch it. Absence is the strongest evidence + // here, so it is scored as zero rather than skipped. + let mut serving: Vec<(Stream, f64)> = served_before + .iter() + .copied() + .filter(|stream| !self.expect.expects_silence(*stream)) + .map(|stream| { + let share = during + .iter() + .find(|c| c.stream == stream) + .and_then(|c| c.share_of_baseline_percent) + .unwrap_or(0.0); + (stream, share) + }) + .collect(); + serving.sort_by(|a, b| a.1.total_cmp(&b.1)); + + if let Some(&(stream, least)) = serving.first() { + self.least_serving_stream_percent = Some(least); + if least < serving_floor { + self.findings.push(OutageFinding { + violation: OutageViolation::UnexpectedStall, + stream: Some(stream), + detail: format!( + "{stream} does not depend on {} and was expected to carry on through \ + the cut, but held only {least}% of its own baseline against a \ + {serving_floor}% floor — the storage this stream actually reaches is \ + not what the routing says it is", + self.endpoint + ), + }); + } } } @@ -587,17 +712,30 @@ impl StorageOutageReport { pub fn note_lines(&self) -> Vec { let mut lines = Vec::new(); if let Some(share) = self.share_of_baseline_percent { + // Says which streams the numbers are about, because under a partial + // cut they are about a subset and a reader who assumes otherwise + // draws the opposite conclusion from the same figure. let quiet = match self.quietest_stream_percent { Some(q) => format!( - "the least quiet stream answered nothing for {q}% of the fault window \ - (floor {}%)", - self.outage_quiet_floor_percent + "the streams expected to stop were silent for at least {q}% of the fault \ + window (floor {}%)", + self.expect.quiet_floor_percent() ), None => "no stream had a before-fault baseline to be judged against".to_string(), }; + let serving = match ( + self.least_serving_stream_percent, + self.expect.serving_floor_percent(), + ) { + (Some(least), Some(floor)) => format!( + ", the streams expected to carry on held at least {least}% of their own \ + baseline (floor {floor}%)" + ), + _ => String::new(), + }; lines.push(format!( - "Storage outage: {quiet} while {} was unreachable, and the workload held {share}% \ - of its baseline throughput across that window", + "Storage outage: {quiet}{serving} while {} was unreachable, and the workload held \ + {share}% of its baseline throughput across that window", self.endpoint )); } else { @@ -834,12 +972,90 @@ mod tests { records } + const SERVING_FLOOR: f64 = 50.0; + + /// A workload shaped like S18's: `durable` on the far side of the cut and + /// `ephemeral` behind it. `durable_during` and `ephemeral_during` say how + /// many confirmations each managed while the storage was gone, so one + /// helper covers "behaved as designed", "the wrong stream stalled" and + /// "nothing stopped at all". + fn partial_history(durable_during: usize, ephemeral_during: usize) -> Vec { + let mut records = Vec::new(); + for second in 1..=300 { + records.push(op(Stream::Durable, -second, Outcome::Confirmed)); + records.push(op(Stream::Ephemeral, -second, Outcome::Confirmed)); + } + for i in 0..durable_during { + records.push(op(Stream::Durable, i as i64, Outcome::Confirmed)); + } + for i in 0..ephemeral_during { + records.push(op(Stream::Ephemeral, i as i64, Outcome::Confirmed)); + } + for second in 181..=420 { + records.push(op(Stream::Durable, second, Outcome::Confirmed)); + records.push(op(Stream::Ephemeral, second, Outcome::Confirmed)); + } + records + } + + /// The same shape as [`partial_history`], but with the operations that did + /// not confirm still *recorded* as stalled attempts. + /// + /// This is what a real run looks like: the workload keeps submitting into a + /// stall, so the stream has a during-fault cell showing what it offered and + /// nothing served. S18's first run recorded 321 such ephemeral operations. + /// [`partial_history`] covers the other shape, where a stream produced no + /// during-fault record at all, and the two must reach the same verdict. + fn partial_history_with_stalls( + durable_confirmed: usize, + ephemeral_confirmed: usize, + ) -> Vec { + let mut records = Vec::new(); + for second in 1..=300 { + records.push(op(Stream::Durable, -second, Outcome::Confirmed)); + records.push(op(Stream::Ephemeral, -second, Outcome::Confirmed)); + } + for (stream, confirmed) in [ + (Stream::Durable, durable_confirmed), + (Stream::Ephemeral, ephemeral_confirmed), + ] { + for i in 0..180 { + if i < confirmed { + records.push(op(stream, i as i64, Outcome::Confirmed)); + } else { + records.push(stalled(stream, i as i64)); + } + } + } + for second in 181..=420 { + records.push(op(Stream::Durable, second, Outcome::Confirmed)); + records.push(op(Stream::Ephemeral, second, Outcome::Confirmed)); + } + records + } + + fn build_partial(records: &[OperationRecord]) -> StorageOutageReport { + StorageOutageReport::build( + records, + Some(fault()), + ENDPOINT, + OutageExpectation::PartialWorkload { + silenced: vec![Stream::Ephemeral], + quiet_floor_percent: QUIET_FLOOR, + serving_floor_percent: SERVING_FLOOR, + }, + Duration::from_secs(120), + ) + } + fn build(records: &[OperationRecord]) -> StorageOutageReport { StorageOutageReport::build( records, Some(fault()), ENDPOINT, - QUIET_FLOOR, + OutageExpectation::WholeWorkload { + quiet_floor_percent: QUIET_FLOOR, + }, Duration::from_secs(120), ) } @@ -888,6 +1104,124 @@ mod tests { ); } + /// The regression S18's first run produced, expressed as a test. + /// + /// Under a partial cut one stream stops and the others carry on by design. + /// The whole-workload rule reads that as a fault that never landed, because + /// the only question it can ask is whether *everything* went quiet. Run + /// 33130077355 hit exactly this: `durable` held 100% of its baseline while + /// the Redis partition had plainly landed on `ephemeral`. + #[test] + fn a_partial_cut_is_not_judged_by_the_streams_it_was_never_going_to_stop() { + let records = partial_history(180, 0); + + let shared = build(&records); + assert!( + shared + .findings + .iter() + .any(|f| f.violation == OutageViolation::OutageNotObserved), + "the whole-workload rule is expected to misjudge this, or the test guards nothing" + ); + + let report = build_partial(&records); + assert!( + report.findings.is_empty(), + "one stream stopping and the rest carrying on is the expected outcome here, got {:?}", + report.findings + ); + } + + /// The half no shared rule could state. + /// + /// A partial cut claims the streams it does not touch keep working, and + /// that claim comes from where the code routes each namespace rather than + /// from an assumption about faults. A stall on the far side means the + /// routing is not what the source says, and the older rule would have + /// called it the cleanest possible pass: everything went quiet. + #[test] + fn a_stream_the_cut_should_not_have_touched_stalling_is_a_finding() { + let report = build_partial(&partial_history(0, 0)); + + assert!( + report + .findings + .iter() + .any(|f| f.violation == OutageViolation::UnexpectedStall), + "durable does not depend on this endpoint and stopped anyway, got {:?}", + report.findings + ); + assert!( + !report + .findings + .iter() + .any(|f| f.violation == OutageViolation::OutageNotObserved), + "ephemeral did stop, so the cut was observed; got {:?}", + report.findings + ); + } + + /// The named stream refusing to stop still fails, which is the check the + /// partial rule inherits rather than replaces. + #[test] + fn a_partial_cut_whose_named_stream_kept_serving_is_still_not_observed() { + let report = build_partial(&partial_history(180, 180)); + + assert!( + report + .findings + .iter() + .any(|f| f.violation == OutageViolation::OutageNotObserved + && f.stream == Some(Stream::Ephemeral)), + "ephemeral was the stream expected to stop, got {:?}", + report.findings + ); + } + + /// Both numbers are reported, and each covers only the streams its own rule + /// judges. Reading the quiet figure as run-wide is how the first S18 report + /// told its reader to draw the wrong conclusion. + #[test] + fn each_reported_share_covers_only_the_streams_its_rule_judges() { + let report = build_partial(&partial_history(180, 0)); + + assert!( + report.quietest_stream_percent.unwrap() > QUIET_FLOOR, + "the quiet figure must describe ephemeral, which stopped, not durable, got {:?}", + report.quietest_stream_percent + ); + assert!( + report.least_serving_stream_percent.unwrap() > SERVING_FLOOR, + "the serving figure must describe durable, which carried on, got {:?}", + report.least_serving_stream_percent + ); + } + + /// The same two verdicts against the shape a real run produces, where the + /// stalled operations are recorded rather than absent. A stream that + /// offered work and served none of it must read the same as one that + /// offered nothing at all. + #[test] + fn a_recorded_stall_reads_the_same_as_a_stream_that_went_missing() { + let as_designed = build_partial(&partial_history_with_stalls(180, 0)); + assert!( + as_designed.findings.is_empty(), + "ephemeral stalling and durable carrying on is the expected outcome, got {:?}", + as_designed.findings + ); + + let wrong_stream = build_partial(&partial_history_with_stalls(0, 0)); + assert!( + wrong_stream + .findings + .iter() + .any(|f| f.violation == OutageViolation::UnexpectedStall + && f.stream == Some(Stream::Durable)), + "durable served nothing while off the fault path, got {:?}", + wrong_stream.findings + ); + } + /// A stream the run barely drives cannot flip the verdict on its own. It is /// judged on how long it was silent, not on its rate against a baseline, so /// a trickle reads as the near-total silence it is rather than as a stream @@ -968,7 +1302,9 @@ mod tests { recovered_at: Some(t0() + TimeDelta::seconds(fault_secs)), }), ENDPOINT, - QUIET_FLOOR, + OutageExpectation::WholeWorkload { + quiet_floor_percent: QUIET_FLOOR, + }, Duration::from_secs(120), ) } @@ -1085,7 +1421,9 @@ mod tests { &history(180, true), None, ENDPOINT, - QUIET_FLOOR, + OutageExpectation::WholeWorkload { + quiet_floor_percent: QUIET_FLOOR, + }, Duration::from_secs(120), ); diff --git a/integration-tests/src/chaos/scenarios/storage_outage.rs b/integration-tests/src/chaos/scenarios/storage_outage.rs index 84e4e87045..c0d5139d1c 100644 --- a/integration-tests/src/chaos/scenarios/storage_outage.rs +++ b/integration-tests/src/chaos/scenarios/storage_outage.rs @@ -132,18 +132,25 @@ //! staleness safe rather than merely tolerated. //! //! What does cross Redis synchronously is a lifecycle boundary — suspend, evict, -//! reattach — and a `get_agent_mode` miss. So the prediction is a *partial* -//! degradation: `ephemeral`, whose agents are created and torn down per -//! operation, and `promise`, whose agents suspend, should go quiet; `durable` -//! may keep serving from memory throughout. -//! -//! Two consequences for reading the report. `shareOfBaselinePercent` should sit -//! higher here than in S16 or S14 without that meaning the fault was weaker, -//! and `quietestStreamPercent` is the number that shows the cut landed, because -//! it reports the stream that stopped rather than the average of one that did -//! and one that did not. Whether a durable agent that kept committing against a -//! frozen status cache still recovers correctly is the second open question of -//! the run, after the exactly-once one. +//! reattach — and a `get_agent_mode` miss. Run 33130077355 settled which +//! streams that amounts to: `ephemeral` alone. Its agents are created and torn +//! down per operation, so every one crosses a boundary, and it was silent for +//! 99.997% of the window while `durable`, `scheduled` and `promise` held +//! 99.94–100.06% of their baselines. +//! +//! `promise` was expected to be the second and is not. The mixed workload's +//! promise stream is `get_promise+complete` in one round trip against a durable +//! agent and never suspends; the suspending variant is `promise-wait`, which +//! S11 drives and these scenarios do not enable. +//! +//! That partial shape is why S18 does not share the other three scenarios' +//! verdict. `shareOfBaselinePercent` sits far higher here — 77.83% against +//! S14's 22.39% — without the fault being weaker, and the run-wide quiet figure +//! the other three are judged on reads 0.05%, because `durable` never stopped. +//! Judged by that rule the run reported a partition that had plainly landed as +//! one that never happened. See [`crate::chaos::OutageExpectation`] for the +//! split, and for why the streams that keep working are asserted on rather than +//! merely exempted. //! //! ## Why the scheduled stream is driven separately //! @@ -553,17 +560,17 @@ pub async fn run( &records, fault_window, &storage_config.endpoint, - storage_config.outage_quiet_floor_percent, + storage_config.expect.clone(), storage_config.recovery_budget(), ); info!( - "{code}: storage account — the least quiet stream answered nothing for {:?}% of the fault \ - window (floor {}%) while {} was unreachable, holding {:?}% of baseline throughput, {} \ - findings", + "{code}: storage account — the streams expected to stop were silent for at least {:?}% of \ + the fault window (floor {}%) and the streams expected to carry on held at least {:?}% of \ + their baseline while {} was unreachable, {} findings", outage.quietest_stream_percent, - outage.outage_quiet_floor_percent, + outage.expect.quiet_floor_percent(), + outage.least_serving_stream_percent, outage.endpoint, - outage.share_of_baseline_percent, outage.findings.len() ); for finding in &outage.findings { @@ -710,6 +717,7 @@ async fn sample_fire_count(code: ScenarioCode, ctx: &WorkloadContext, targets: & #[cfg(test)] mod tests { use super::*; + use crate::chaos::OutageExpectation; use crate::chaos::history::{AttemptRecord, FireRecord, TargetFireLog}; use crate::chaos::outage::{OutageFinding, OutageViolation}; use chrono::{DateTime, TimeDelta}; @@ -861,8 +869,15 @@ mod tests { /// experiment. #[test] fn a_storage_finding_does_not_change_the_termination_reason() { - let mut outage = - StorageOutageReport::build(&[], None, "db.example", 15.0, Duration::from_secs(120)); + let mut outage = StorageOutageReport::build( + &[], + None, + "db.example", + OutageExpectation::WholeWorkload { + quiet_floor_percent: 15.0, + }, + Duration::from_secs(120), + ); outage.findings.push(OutageFinding { violation: OutageViolation::OutageNotObserved, stream: None, From e8fccdc1f0a2d07d2f1476607d6c837ec4e2486c Mon Sep 17 00:00:00 2001 From: Kaur Matas <33095685+kmatasfp@users.noreply.github.com> Date: Thu, 27 Aug 2026 20:38:12 -0700 Subject: [PATCH 35/40] Rename the storage outage driver to storage_fault --- integration-tests/src/benchmarks/all.rs | 2 +- integration-tests/src/chaos/mod.rs | 4 +- integration-tests/src/chaos/outage.rs | 18 +++--- integration-tests/src/chaos/result.rs | 9 ++- integration-tests/src/chaos/scenarios/mod.rs | 7 +- .../{storage_outage.rs => storage_fault.rs} | 8 +-- integration-tests/src/chaos/summary.rs | 64 ++++++++++++++++--- 7 files changed, 83 insertions(+), 29 deletions(-) rename integration-tests/src/chaos/scenarios/{storage_outage.rs => storage_fault.rs} (99%) diff --git a/integration-tests/src/benchmarks/all.rs b/integration-tests/src/benchmarks/all.rs index 8a0cabea7c..d437083cbf 100644 --- a/integration-tests/src/benchmarks/all.rs +++ b/integration-tests/src/benchmarks/all.rs @@ -655,7 +655,7 @@ async fn run_chaos( | chaos::ScenarioCode::S16 | chaos::ScenarioCode::S18 | chaos::ScenarioCode::S22) => { - chaos::scenarios::storage_outage::run( + chaos::scenarios::storage_fault::run( code, &config, &manifest, &deps, &signals, &outputs, ) .await diff --git a/integration-tests/src/chaos/mod.rs b/integration-tests/src/chaos/mod.rs index 92ff422862..35bd53e1f0 100644 --- a/integration-tests/src/chaos/mod.rs +++ b/integration-tests/src/chaos/mod.rs @@ -1475,7 +1475,7 @@ mod tests { /// off, so the verdict that exists to catch a fault which never landed /// would never fire. #[test] - fn a_storage_outage_quiet_floor_of_zero_is_refused() { + fn a_storage_fault_quiet_floor_of_zero_is_refused() { let error = storage_config("db.example", 0.0, 0, true) .require_storage() .unwrap_err() @@ -1490,7 +1490,7 @@ mod tests { /// outage can be quiet for the whole window and the verdict would be stuck /// on for every run. #[test] - fn a_storage_outage_quiet_floor_of_a_whole_window_is_refused() { + fn a_storage_fault_quiet_floor_of_a_whole_window_is_refused() { assert!( storage_config("db.example", 100.0, 0, true) .require_storage() diff --git a/integration-tests/src/chaos/outage.rs b/integration-tests/src/chaos/outage.rs index 0a0d857558..0661bc0f3c 100644 --- a/integration-tests/src/chaos/outage.rs +++ b/integration-tests/src/chaos/outage.rs @@ -279,7 +279,7 @@ pub struct FaultWindowErrors { /// The storage-outage account. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct StorageOutageReport { +pub struct StorageFaultReport { /// The endpoint the workflow was asked to cut the executors off from, /// recorded so an archived result says which storage the run was about /// rather than leaving it to the scenario name. @@ -346,7 +346,7 @@ struct Tally { served_at: Vec>, } -impl StorageOutageReport { +impl StorageFaultReport { /// Builds the account from the operation history. /// /// `fault` is what the workflow reported. Without it every record lands in @@ -1034,8 +1034,8 @@ mod tests { records } - fn build_partial(records: &[OperationRecord]) -> StorageOutageReport { - StorageOutageReport::build( + fn build_partial(records: &[OperationRecord]) -> StorageFaultReport { + StorageFaultReport::build( records, Some(fault()), ENDPOINT, @@ -1048,8 +1048,8 @@ mod tests { ) } - fn build(records: &[OperationRecord]) -> StorageOutageReport { - StorageOutageReport::build( + fn build(records: &[OperationRecord]) -> StorageFaultReport { + StorageFaultReport::build( records, Some(fault()), ENDPOINT, @@ -1277,7 +1277,7 @@ mod tests { /// An absorbed outage: silence across the window with the serving bunched /// into the seconds at its two edges, which is the shape the platform /// produces once storage failures are retried rather than fatal. - fn absorbed(fault_secs: i64) -> StorageOutageReport { + fn absorbed(fault_secs: i64) -> StorageFaultReport { let mut records = Vec::new(); for second in 1..=300 { for _ in 0..10 { @@ -1295,7 +1295,7 @@ mod tests { records.push(op(Stream::Durable, second, Outcome::Confirmed)); records.push(op(Stream::Scheduled, second, Outcome::Confirmed)); } - StorageOutageReport::build( + StorageFaultReport::build( &records, Some(FaultWindow { injected_at: t0(), @@ -1417,7 +1417,7 @@ mod tests { /// so the report carries counts and refuses to reach a verdict. #[test] fn without_a_fault_window_there_is_no_verdict() { - let report = StorageOutageReport::build( + let report = StorageFaultReport::build( &history(180, true), None, ENDPOINT, diff --git a/integration-tests/src/chaos/result.rs b/integration-tests/src/chaos/result.rs index 08fd25d12d..c8b149582b 100644 --- a/integration-tests/src/chaos/result.rs +++ b/integration-tests/src/chaos/result.rs @@ -39,7 +39,14 @@ use std::path::Path; /// Bumped when the on-disk shape changes incompatibly. Archived results outlive /// the tooling that reads them, so the shape has to say which shape it is. -pub const RESULT_SCHEMA_VERSION: u32 = 2; +/// +/// 3: the storage scenarios stopped sharing one verdict. `storage` carries an +/// `expect` block where it used to carry `outageQuietFloorPercent`, and the +/// storage-fault account carries the same block plus +/// `leastServingStreamPercent`. A version 2 result does not deserialise into +/// the version 3 types, which is what the bump is for; the report generator +/// reads both, because the runs already in the bucket are worth rendering. +pub const RESULT_SCHEMA_VERSION: u32 = 3; /// A phase's wall-clock extent. These are the numbers the workflow pins Grafana /// time ranges to, so they are recorded in UTC with no ambiguity. diff --git a/integration-tests/src/chaos/scenarios/mod.rs b/integration-tests/src/chaos/scenarios/mod.rs index 8c35522d81..88459b1076 100644 --- a/integration-tests/src/chaos/scenarios/mod.rs +++ b/integration-tests/src/chaos/scenarios/mod.rs @@ -15,8 +15,9 @@ //! Chaos scenario implementations. //! //! One module per scenario code, except where two codes are the same -//! choreography under different settings: `storage_outage` runs both S16 and -//! S22, which differ only in how long the storage is taken away for. Each +//! choreography under different settings: `storage_fault` runs S14, S16, S17, +//! S18 and S22, which differ in which store the fault is aimed at, what the +//! fault does to it, and for how long. Each //! module owns its phase choreography — which is the part that differs, and the //! part worth reading — while everything around it lives here: where artifacts go, how a signal failure becomes a //! termination reason, how a routing table is sampled, and how a result is @@ -38,7 +39,7 @@ pub mod s6; pub mod s7; pub mod s8; pub mod s9; -pub mod storage_outage; +pub mod storage_fault; use crate::chaos::ScenarioConfig; use crate::chaos::history::{OperationHistory, OperationRecord, Stream}; diff --git a/integration-tests/src/chaos/scenarios/storage_outage.rs b/integration-tests/src/chaos/scenarios/storage_fault.rs similarity index 99% rename from integration-tests/src/chaos/scenarios/storage_outage.rs rename to integration-tests/src/chaos/scenarios/storage_fault.rs index c0d5139d1c..17f0812dd9 100644 --- a/integration-tests/src/chaos/scenarios/storage_outage.rs +++ b/integration-tests/src/chaos/scenarios/storage_fault.rs @@ -198,7 +198,7 @@ use crate::chaos::fires::{FaultWindow, ScheduleFireReport}; use crate::chaos::history::{OperationHistory, OperationRecord, Outcome, Phase, Stream}; -use crate::chaos::outage::StorageOutageReport; +use crate::chaos::outage::StorageFaultReport; use crate::chaos::prep::ChaosPrepManifest; use crate::chaos::probe; use crate::chaos::result::{ChaosResult, PhaseWindow, Phases, RunScope}; @@ -310,7 +310,7 @@ pub async fn run( summary = summary.with_schedule_fires(report); } if let Some(report) = $outage { - summary = summary.with_storage_outage(report); + summary = summary.with_storage_fault(report); } if let Some(report) = $exactly { summary = summary.with_exactly_once(report); @@ -556,7 +556,7 @@ pub async fn run( fires.findings.len() ); - let outage = StorageOutageReport::build( + let outage = StorageFaultReport::build( &records, fault_window, &storage_config.endpoint, @@ -869,7 +869,7 @@ mod tests { /// experiment. #[test] fn a_storage_finding_does_not_change_the_termination_reason() { - let mut outage = StorageOutageReport::build( + let mut outage = StorageFaultReport::build( &[], None, "db.example", diff --git a/integration-tests/src/chaos/summary.rs b/integration-tests/src/chaos/summary.rs index 6dd277437e..51ab2ad401 100644 --- a/integration-tests/src/chaos/summary.rs +++ b/integration-tests/src/chaos/summary.rs @@ -50,7 +50,7 @@ use crate::chaos::fires::ScheduleFireReport; use crate::chaos::history::{Outcome, Phase, Stream}; -use crate::chaos::outage::StorageOutageReport; +use crate::chaos::outage::StorageFaultReport; use crate::chaos::ownership::OwnershipSample; use crate::chaos::probe::KeyProbe; use crate::chaos::reachability::ReachabilityReport; @@ -594,11 +594,23 @@ pub struct ChaosSummary { /// back. Absent for scenarios that do not. #[serde(default, skip_serializing_if = "Option::is_none")] pub rollback: Option, - /// The storage-outage account, for scenarios that take a storage dependency - /// away from every executor at once. Absent for scenarios that do not, for - /// the same reason as `scheduleFires`. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub storage_outage: Option, + /// The storage-fault account, for scenarios that break a storage + /// dependency underneath every executor at once — by taking it away or by + /// slowing it down. Absent for scenarios that do not, for the same reason + /// as `scheduleFires`. + /// + /// Serialised as `storageOutage` rather than under the field's own name. + /// The wire name predates the scenarios that degrade a store rather than + /// removing one, and every archived result and the report generator that + /// reads them use it. Renaming the field would cost a schema bump and + /// silently stop rendering the runs already in the bucket, which is a worse + /// outcome than one name that has outlived its accuracy. + #[serde( + rename = "storageOutage", + default, + skip_serializing_if = "Option::is_none" + )] + pub storage_fault: Option, /// Shard-ownership samples, in the order they were taken. Empty for /// scenarios that do not sample executor assignments. /// @@ -734,7 +746,7 @@ impl ChaosSummary { truncation: None, resurrection: None, rollback: None, - storage_outage: None, + storage_fault: None, ownership: Vec::new(), attention, notes: Vec::new(), @@ -855,10 +867,10 @@ impl ChaosSummary { /// outage-not-observed one: a partition that failed to take hold leaves /// every cell underneath it describing an undisturbed cluster, and that has /// to read as "this run tested nothing" rather than as a pass. - pub fn with_storage_outage(mut self, report: StorageOutageReport) -> Self { + pub fn with_storage_fault(mut self, report: StorageFaultReport) -> Self { self.attention.extend(report.attention_lines()); self.notes.extend(report.note_lines()); - self.storage_outage = Some(report); + self.storage_fault = Some(report); self } @@ -992,6 +1004,40 @@ mod tests { Utc.timestamp_opt(1_800_000_000 + secs, 0).unwrap() } + /// The field is `storage_fault` in Rust and `storageOutage` on disk, and + /// that mismatch is deliberate rather than an oversight. + /// + /// The wire name predates the scenarios that slow a store down instead of + /// removing one. Every result already in the bucket uses it, and so does + /// the report generator that renders them. Renaming it would stop those + /// runs rendering to buy nothing, so the `#[serde(rename)]` stays and this + /// test is what stops a later tidy-up from quietly dropping it. + #[test] + fn the_storage_fault_account_still_serialises_under_its_original_name() { + let summary = ChaosSummary::build(&[], Vec::new(), Vec::new(), None).with_storage_fault( + StorageFaultReport::build( + &[], + None, + "db.example", + crate::chaos::OutageExpectation::WholeWorkload { + quiet_floor_percent: 50.0, + }, + std::time::Duration::from_secs(120), + ), + ); + + let json = serde_json::to_value(&summary).unwrap(); + assert!( + json.get("storageOutage").is_some(), + "the on-disk name must not drift, got keys: {:?}", + json.as_object().map(|o| o.keys().collect::>()) + ); + assert!( + json.get("storageFault").is_none(), + "renaming this field silently orphans every archived result" + ); + } + fn op(op_id: u64, stream: Stream, phase: Phase, outcome: Outcome) -> OperationRecord { OperationRecord { op_id, From 0947c0badc4ca4193165c64a9c3eb903b731cd50 Mon Sep 17 00:00:00 2001 From: Kaur Matas <33095685+kmatasfp@users.noreply.github.com> Date: Thu, 27 Aug 2026 21:24:59 -0700 Subject: [PATCH 36/40] Add the S17 Redis cache latency scenario --- golem-test-framework/src/benchmark/config.rs | 2 + .../chaos_suites/cloud-chaos.yaml | 127 +++++++++++ integration-tests/src/benchmarks/all.rs | 2 + integration-tests/src/chaos/mod.rs | 199 ++++++++++++++---- integration-tests/src/chaos/outage.rs | 192 ++++++++++++++++- .../src/chaos/scenarios/storage_fault.rs | 42 ++-- 6 files changed, 509 insertions(+), 55 deletions(-) diff --git a/golem-test-framework/src/benchmark/config.rs b/golem-test-framework/src/benchmark/config.rs index a8e7f565a6..29d266d337 100644 --- a/golem-test-framework/src/benchmark/config.rs +++ b/golem-test-framework/src/benchmark/config.rs @@ -265,6 +265,8 @@ pub enum ChaosScenarioArg { /// Executors cut off from the Redis cache in front of the key-value layer, /// for longer than a caller is willing to wait. S18, + /// The same Redis cache slowed rather than removed. + S17, } /// Density subcommand action. diff --git a/integration-tests/chaos_suites/cloud-chaos.yaml b/integration-tests/chaos_suites/cloud-chaos.yaml index 6bd37f5ce1..4acd087ddf 100644 --- a/integration-tests/chaos_suites/cloud-chaos.yaml +++ b/integration-tests/chaos_suites/cloud-chaos.yaml @@ -1332,3 +1332,130 @@ scenarios: delaySecs: 5 signalTimeoutSecs: 1800 + + # --------------------------------------------------------------------------- + # S17 — Redis latency degradation (GOL-375) + # + # The fifth scenario on the storage driver, and the first that does not break + # anything. S18 took the Redis cache away; this leaves it reachable and makes + # it slow. That is a different question rather than a milder version of the + # same one: S18 asked whether the platform survives losing its worker-status + # store, and this asks whether it degrades or breaks when that store gets + # slower. "It went quiet" would be a failure here, not the expected result. + # + # GOL-375 asks for three things S18 could not give it — invocation latency + # under load, RSS peak, and pending-work evidence — and for a determination of + # whether worker-status writes block invocation completion or are best effort. + # S18 answered half of that: best effort for a resident durable agent, because + # AgentStatusFlusher takes the blob off the commit path, and blocking for + # anything crossing a lifecycle boundary. This run gets the other half, which + # is what that costs when the store is merely slow rather than gone. + # --------------------------------------------------------------------------- + - code: S17 + name: redis-cache-latency + enabled: true + + fault: + # Recorded for the archive rather than acted on: the workflow selects the + # manifest by scenario code and reads the Chaos Mesh kind out of the + # manifest itself. It compares this string only against `none`, + # `network-partition` and `pod-kill`, none of which this is. + kind: network-delay + target: worker-executor + # Every executor, as in the other four. A cache that is slow for only some + # executors is a routing fault wearing a cache's clothes. + mode: all + # Must equal phases.faultSecs; see the note on S16. + durationSecs: 300 + + phases: + # Same as the other four, so all five are comparable. + baselineSecs: 300 + # Longer than any of the others, and not for the reason S18's window is + # long. Nothing times out under a 500ms delay — an ephemeral operation + # goes from about 212ms to under a second, nowhere near the 120s attempt + # timeout — so there is no caller clock to outrun. The length is for the + # two things GOL-375 asks for that need time to develop: a memory series + # long enough to show whether the executor's dirty-status set grows, and + # a steady state rather than a transient. At the workflow's 15s sampling + # interval this is about 20 memory samples inside the window. + faultSecs: 300 + # As the others. + recoverySecs: 420 + + workload: + # Identical to S14, S16, S18 and S22, so the difference between the five + # results is the fault and not the population. + durableAgents: 200 + ephemeralAgents: 50 + # Zero for the same reason as the others: the scheduled stream comes from + # the block below. Setting both is refused at load time. + scheduledAgents: 0 + promiseAgents: 50 + quotaAgents: 0 + ratePerSec: 100 + + scheduled: + targets: 100 + intervalMillis: 2000 + leadSecs: 10 + # The scheduler keeps its own schema on Postgres, which this fault does + # not touch, so its claims and acknowledgements should be unaffected. S18 + # confirmed that: during-fault p99 was 2094ms against a 2099ms baseline. + # 240 as the other three, and it should not be approached. + leaseBudgetSecs: 240 + + storage: + # The same Redis endpoint S18 cuts. Both Aurora clusters stay reachable + # and unslowed for the whole run. + endpoint: master.golem-redis-dev.intsie.use1.cache.amazonaws.com + + expect: + # Neither of the rules the other four use. Under a delay nothing is + # supposed to fall silent, so silence is not available as evidence that + # the fault landed and time has to stand in for it. + kind: latency-degradation + + # `ephemeral` alone, for the reason S18 measured rather than predicted: + # its agents are created and torn down per operation, so every one + # crosses a lifecycle boundary, and that is one of the few things still + # reaching this cache synchronously. Under S18 it was the only stream + # that stopped. + slowed: [ephemeral] + + # Ephemeral's baseline median was 212ms in S18's run. The delay is 500ms + # applied to egress only (`direction: to`, which Chaos Mesh's webhook + # requires for netem against an external target), so one Redis round + # trip inside the operation puts it at about 712ms, or 3.4x. Two put it + # near 5.7x. + # + # 2.0 sits below the single-round-trip case with room to spare while + # still failing the run that matters: a netem rule that never applied + # leaves the stream at roughly 1.0x, and the report would otherwise be + # full of healthy numbers describing an experiment that never happened. + slowdownFloor: 2.0 + + # A delay should cost time, not work. Throughput is not expected to move + # much: the per-stream in-flight budget is MAX_IN_FLIGHT/4 = 256, and at + # 33 operations per second and 712ms each the stream sits around 23 in + # flight, so nothing queues. A stream falling below half its baseline + # therefore means the added latency broke something rather than slowed + # it — a timeout fired, a pool drained — and that is worth failing on. + servingFloorPercent: 50 + + # Recorded, not asserted. Expected to be near zero: nothing was ever + # unreachable, so there is no reconnect to wait for and no backlog to + # drain, only the last few in-flight operations finishing at their slower + # pace. + recoveryBudgetSecs: 60 + + retryPolicy: + # Identical to the other four. Not load-bearing here the way it is in S18, + # because nothing should time out and so nothing should retry — but a + # retry count above zero would itself be a finding, and leaving the policy + # the same is what makes that comparable. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 diff --git a/integration-tests/src/benchmarks/all.rs b/integration-tests/src/benchmarks/all.rs index d437083cbf..526a4c7af0 100644 --- a/integration-tests/src/benchmarks/all.rs +++ b/integration-tests/src/benchmarks/all.rs @@ -602,6 +602,7 @@ async fn run_chaos( ChaosScenarioArg::S22 => chaos::ScenarioCode::S22, ChaosScenarioArg::S14 => chaos::ScenarioCode::S14, ChaosScenarioArg::S18 => chaos::ScenarioCode::S18, + ChaosScenarioArg::S17 => chaos::ScenarioCode::S17, }; let config = suite .scenario(code, allow_disabled) @@ -653,6 +654,7 @@ async fn run_chaos( } code @ (chaos::ScenarioCode::S14 | chaos::ScenarioCode::S16 + | chaos::ScenarioCode::S17 | chaos::ScenarioCode::S18 | chaos::ScenarioCode::S22) => { chaos::scenarios::storage_fault::run( diff --git a/integration-tests/src/chaos/mod.rs b/integration-tests/src/chaos/mod.rs index 35bd53e1f0..bec68df77c 100644 --- a/integration-tests/src/chaos/mod.rs +++ b/integration-tests/src/chaos/mod.rs @@ -103,6 +103,9 @@ pub enum ScenarioCode { /// Executors cut off from the Redis cache that fronts the key-value layer, /// for longer than a caller is willing to wait. S18, + /// The same Redis cache still reachable but slowed, to ask whether the + /// platform degrades or breaks when its worker-status store gets slower. + S17, } impl ScenarioCode { @@ -123,13 +126,14 @@ impl ScenarioCode { ScenarioCode::S22 => "S22", ScenarioCode::S14 => "S14", ScenarioCode::S18 => "S18", + ScenarioCode::S17 => "S17", } } /// Every scenario this driver implements. The suite YAML is checked against /// this list, so a scenario cannot be enabled in YAML without code behind /// it, nor implemented without an operational switch in front of it. - pub const ALL: [ScenarioCode; 15] = [ + pub const ALL: [ScenarioCode; 16] = [ ScenarioCode::S1, ScenarioCode::S3, ScenarioCode::S5, @@ -143,6 +147,7 @@ impl ScenarioCode { ScenarioCode::S13, ScenarioCode::S14, ScenarioCode::S16, + ScenarioCode::S17, ScenarioCode::S18, ScenarioCode::S22, ]; @@ -772,6 +777,38 @@ pub enum OutageExpectation { /// says it is. serving_floor_percent: f64, }, + /// The store is still reachable, only slow. Nothing is expected to fall + /// silent at all, so the evidence the fault landed is latency rather than + /// absence. S15 and S17. + /// + /// Worth being explicit that this is a different *question*, not a milder + /// version of the other two. Those ask whether the platform survives losing + /// a store. This asks whether it degrades or breaks when one gets slower, + /// and "it went quiet" would be a failure here rather than the expected + /// result. + LatencyDegradation { + /// The streams whose latency must rise for the injected delay to have + /// landed. Their slowdown plays the part silence plays elsewhere: it is + /// the only positive evidence the run has that anything happened. + slowed: Vec, + /// The least each named stream's during-fault median latency must be as + /// a multiple of its own before-fault median. + /// + /// A multiple rather than a millisecond figure, so the threshold does + /// not have to be re-derived per stream. Streams differ by an order of + /// magnitude at rest — S18's baselines ran from 56ms on `durable` to + /// 240ms on `ephemeral` — and a fixed ceiling would be slack for one + /// and impossible for the other. + slowdown_floor: f64, + /// The least of its own baseline rate *every* stream must still hold, + /// including the slowed ones. + /// + /// The claim that separates degradation from breakage. A delay should + /// make work take longer, not stop it, so a stream falling to nothing + /// here means the added latency cost more than time — a timeout fired, + /// a pool drained, a queue filled — and that is worth failing on. + serving_floor_percent: f64, + }, } impl OutageExpectation { @@ -785,6 +822,26 @@ impl OutageExpectation { quiet_floor_percent, .. } => *quiet_floor_percent, + // Nothing is expected to go quiet, so there is no floor to apply + // and reporting one would invite a reader to check it. + OutageExpectation::LatencyDegradation { .. } => 0.0, + } + } + + /// Whether this stream is one whose slowdown would prove a delay landed. + pub fn expects_slowdown(&self, stream: Stream) -> bool { + match self { + OutageExpectation::LatencyDegradation { slowed, .. } => slowed.contains(&stream), + _ => false, + } + } + + /// The multiple of its own baseline median a slowed stream must reach, or + /// `None` where the expectation makes no claim about latency. + pub fn slowdown_floor(&self) -> Option { + match self { + OutageExpectation::LatencyDegradation { slowdown_floor, .. } => Some(*slowdown_floor), + _ => None, } } @@ -796,6 +853,7 @@ impl OutageExpectation { match self { OutageExpectation::WholeWorkload { .. } => true, OutageExpectation::PartialWorkload { silenced, .. } => silenced.contains(&stream), + OutageExpectation::LatencyDegradation { .. } => false, } } @@ -807,6 +865,10 @@ impl OutageExpectation { OutageExpectation::PartialWorkload { serving_floor_percent, .. + } + | OutageExpectation::LatencyDegradation { + serving_floor_percent, + .. } => Some(*serving_floor_percent), } } @@ -1082,6 +1144,68 @@ impl ScenarioConfig { } } + /// A floor of 100% can never be met: a stream is only quiet between the + /// answers it did give, and an operation submitted just before the heal + /// confirms just after it, inside the window it was submitted in. Every run + /// would then report the outage as not observed, and the one verdict that + /// exists to catch a fault which never landed would be stuck on. A floor of + /// zero is the opposite failure: every run passes, including the ones where + /// nothing was ever cut off. + fn check_quiet_floor(&self, floor: f64) -> anyhow::Result<()> { + if !(0.0..100.0).contains(&floor) || floor <= 0.0 { + anyhow::bail!( + "chaos scenario {}: expect.quietFloorPercent is {floor}, which is not a share \ + of the fault window a real outage could be judged by", + self.code + ); + } + Ok(()) + } + + fn check_serving_floor(&self, floor: f64) -> anyhow::Result<()> { + if !(0.0..=100.0).contains(&floor) || floor <= 0.0 { + anyhow::bail!( + "chaos scenario {}: expect.servingFloorPercent is {floor}, which is not a share \ + of a stream's own baseline it could be held to", + self.code + ); + } + Ok(()) + } + + /// The streams whose behaviour is the run's only positive evidence that the + /// fault landed at all. + /// + /// Naming one the workload never drives leaves the run with no such + /// evidence and nothing to complain about — the exact failure the floors + /// exist to prevent, reintroduced one level up. Refused at load time so it + /// costs a build rather than a maintenance window. + fn check_evidence_streams( + &self, + field: &str, + streams: &[Stream], + evidence: &str, + ) -> anyhow::Result<()> { + if streams.is_empty() { + anyhow::bail!( + "chaos scenario {}: expect.{field} is empty, so nothing in the run would show \ + whether the fault landed at all", + self.code + ); + } + for stream in streams { + if !self.drives_stream(*stream) { + anyhow::bail!( + "chaos scenario {}: expect.{field} names `{stream}`, which this scenario's \ + workload never drives, so its {evidence} during the fault would prove \ + nothing", + self.code + ); + } + } + Ok(()) + } + pub fn require_storage(&self) -> anyhow::Result<&StorageConfig> { let config = self.storage.as_ref().ok_or_else(|| { anyhow::anyhow!( @@ -1096,45 +1220,43 @@ impl ScenarioConfig { // that exists to catch a fault which never landed would be stuck on. A // floor of zero is the opposite failure: every run passes, including // the ones where nothing was ever cut off. - let quiet_floor = config.expect.quiet_floor_percent(); - if !(0.0..100.0).contains(&quiet_floor) || quiet_floor <= 0.0 { - anyhow::bail!( - "chaos scenario {}: expect.quietFloorPercent is {quiet_floor}, which is not a \ - share of the fault window a real outage could be judged by", - self.code - ); - } - if let Some(serving_floor) = config.expect.serving_floor_percent() - && (!(0.0..=100.0).contains(&serving_floor) || serving_floor <= 0.0) - { - anyhow::bail!( - "chaos scenario {}: expect.servingFloorPercent is {serving_floor}, which is not a \ - share of a stream's own baseline it could be held to", - self.code - ); - } - // A partial cut proves it landed by naming the streams it stops, so a - // name the workload never drives leaves the run with no evidence and no - // complaint — the exact failure the quiet floor exists to prevent, - // reintroduced one level up. Refused here rather than at judge time so - // it costs a build rather than a maintenance window. - if let OutageExpectation::PartialWorkload { silenced, .. } = &config.expect { - if silenced.is_empty() { - anyhow::bail!( - "chaos scenario {}: expect.silenced is empty, so nothing in the run would \ - show whether the cut landed at all", - self.code - ); + // Checked per variant rather than through the accessors, because the + // variants do not share a set of knobs and a single pass over "whatever + // is present" would silently skip whichever one this scenario actually + // relies on. + match &config.expect { + OutageExpectation::WholeWorkload { + quiet_floor_percent, + } => { + self.check_quiet_floor(*quiet_floor_percent)?; } - for stream in silenced { - if !self.drives_stream(*stream) { + OutageExpectation::PartialWorkload { + silenced, + quiet_floor_percent, + serving_floor_percent, + } => { + self.check_quiet_floor(*quiet_floor_percent)?; + self.check_serving_floor(*serving_floor_percent)?; + self.check_evidence_streams("silenced", silenced, "silence")?; + } + OutageExpectation::LatencyDegradation { + slowed, + slowdown_floor, + serving_floor_percent, + } => { + // A floor of 1.0 or less asks a stream to be no slower than it + // already was, which every run satisfies including one where + // the delay never applied. That is the same hole a zero quiet + // floor leaves, in the units this variant uses. + if !slowdown_floor.is_finite() || *slowdown_floor <= 1.0 { anyhow::bail!( - "chaos scenario {}: expect.silenced names `{stream}`, which this \ - scenario's workload never drives, so its silence during the fault \ - would prove nothing", + "chaos scenario {}: expect.slowdownFloor is {slowdown_floor}, and a \ + factor at or below 1.0 is met by a run where the delay never applied", self.code ); } + self.check_serving_floor(*serving_floor_percent)?; + self.check_evidence_streams("slowed", slowed, "slowdown")?; } } if config.endpoint.trim().is_empty() { @@ -1604,6 +1726,7 @@ mod tests { assert_eq!(ScenarioCode::parse("s16"), Some(ScenarioCode::S16)); assert_eq!(ScenarioCode::parse("s14"), Some(ScenarioCode::S14)); assert_eq!(ScenarioCode::parse("s18"), Some(ScenarioCode::S18)); + assert_eq!(ScenarioCode::parse("s17"), Some(ScenarioCode::S17)); assert_eq!(ScenarioCode::parse("S99"), None); } @@ -1649,7 +1772,11 @@ mod tests { entry.require_workload().unwrap(); entry.require_rollback().unwrap(); } - ScenarioCode::S14 | ScenarioCode::S16 | ScenarioCode::S18 | ScenarioCode::S22 => { + ScenarioCode::S14 + | ScenarioCode::S16 + | ScenarioCode::S17 + | ScenarioCode::S18 + | ScenarioCode::S22 => { entry.require_workload().unwrap(); entry.require_scheduled().unwrap(); entry.require_storage().unwrap(); diff --git a/integration-tests/src/chaos/outage.rs b/integration-tests/src/chaos/outage.rs index 0661bc0f3c..b4fa2e9eae 100644 --- a/integration-tests/src/chaos/outage.rs +++ b/integration-tests/src/chaos/outage.rs @@ -140,6 +140,13 @@ pub enum OutageViolation { /// everything went quiet — would have called that its cleanest possible /// pass. UnexpectedStall, + /// A stream a delay was aimed at ran no slower than it did at rest. + /// + /// The latency equivalent of [`Self::OutageNotObserved`]. Where a cut + /// proves it landed by silence, a delay can only prove it by time, and a + /// netem rule that failed to apply leaves a run full of healthy numbers and + /// no error anywhere — the worst artifact this suite can produce. + SlowdownNotObserved, /// A stream that was confirming operations before the outage confirmed /// nothing at all after the heal. StreamNeverRecovered, @@ -150,6 +157,7 @@ impl OutageViolation { match self { OutageViolation::OutageNotObserved => "outage-not-observed", OutageViolation::UnexpectedStall => "unexpected-stall", + OutageViolation::SlowdownNotObserved => "slowdown-not-observed", OutageViolation::StreamNeverRecovered => "stream-never-recovered", } } @@ -318,6 +326,16 @@ pub struct StorageFaultReport { /// than from an assumption about faults. #[serde(default, skip_serializing_if = "Option::is_none")] pub least_serving_stream_percent: Option, + /// The least any stream *expected to slow down* did, as a multiple of its + /// own before-fault median latency. `None` unless the expectation is + /// [`OutageExpectation::LatencyDegradation`]. + /// + /// A multiple of the stream's own baseline rather than a millisecond + /// figure, because the streams differ by an order of magnitude at rest and + /// a single absolute threshold would be slack for one and unreachable for + /// another. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub least_slowdown_factor: Option, pub cells: Vec, /// What the outage began underneath, per stream. Empty for a run that never /// learned when the fault was. @@ -466,6 +484,7 @@ impl StorageFaultReport { share_of_baseline_percent: None, quietest_stream_percent: None, least_serving_stream_percent: None, + least_slowdown_factor: None, cells, caught_in_flight: caught_in_flight(records, fault), recovery: Vec::new(), @@ -582,6 +601,44 @@ impl StorageFaultReport { } } + // The latency half, before the serving one, because a delay that never + // applied makes everything below it uninteresting: the streams would + // all be serving normally and the run would read as a clean pass of an + // experiment that never happened. + if let Some(slowdown_floor) = self.expect.slowdown_floor() { + let mut slowdowns: Vec<(Stream, f64)> = served_before + .iter() + .copied() + .filter(|stream| self.expect.expects_slowdown(*stream)) + .filter_map(|stream| { + // Each stream against its own before-fault median. Medians + // rather than means because a handful of retried operations + // drag a mean far enough to hide whether the typical + // operation moved at all, which is the question. + let baseline = self.median_ms(stream, Window::BeforeFault)?; + let during = self.median_ms(stream, Window::DuringFault)?; + (baseline > 0.0).then_some((stream, round2(during / baseline))) + }) + .collect(); + slowdowns.sort_by(|a, b| a.1.total_cmp(&b.1)); + + if let Some(&(stream, least)) = slowdowns.first() { + self.least_slowdown_factor = Some(least); + if least < slowdown_floor { + self.findings.push(OutageFinding { + violation: OutageViolation::SlowdownNotObserved, + stream: Some(stream), + detail: format!( + "{stream} was expected to slow down while {} was delayed and ran at \ + {least}x its own baseline median against a {slowdown_floor}x floor \ + — so this run has no evidence the delay reached the executors", + self.endpoint + ), + }); + } + } + } + let Some(serving_floor) = self.expect.serving_floor_percent() else { return; }; @@ -707,6 +764,15 @@ impl StorageFaultReport { .collect() } + /// One stream's median latency in a window, if it has a cell with samples. + fn median_ms(&self, stream: Stream, window: Window) -> Option { + self.cells + .iter() + .find(|c| c.stream == stream && c.window == window) + .filter(|c| c.latency.count > 0) + .map(|c| c.latency.p50_ms as f64) + } + /// Context a reader needs in order to read the cells, which is not itself a /// problem. pub fn note_lines(&self) -> Vec { @@ -715,13 +781,23 @@ impl StorageFaultReport { // Says which streams the numbers are about, because under a partial // cut they are about a subset and a reader who assumes otherwise // draws the opposite conclusion from the same figure. - let quiet = match self.quietest_stream_percent { - Some(q) => format!( + // Under a latency expectation nothing is meant to go quiet, so the + // quiet figure is absent by design rather than missing, and the + // slowdown takes its place as what says the fault landed. + let quiet = match (self.quietest_stream_percent, self.least_slowdown_factor) { + (_, Some(factor)) => format!( + "the streams expected to slow down ran at least {factor}x their own baseline \ + median (floor {}x)", + self.expect.slowdown_floor().unwrap_or_default() + ), + (Some(q), None) => format!( "the streams expected to stop were silent for at least {q}% of the fault \ window (floor {}%)", self.expect.quiet_floor_percent() ), - None => "no stream had a before-fault baseline to be judged against".to_string(), + (None, None) => { + "no stream had a before-fault baseline to be judged against".to_string() + } }; let serving = match ( self.least_serving_stream_percent, @@ -1034,6 +1110,54 @@ mod tests { records } + const SLOWDOWN_FLOOR: f64 = 2.0; + + /// A workload where every stream keeps serving and `ephemeral` is the one + /// the delay is aimed at. `ephemeral_ms` is what its operations take during + /// the fault, against a 200ms baseline. + fn latency_history(ephemeral_ms: u64, ephemeral_during: usize) -> Vec { + let mut records = Vec::new(); + for second in 1..=300 { + records.push(timed(Stream::Durable, -second, 50)); + records.push(timed(Stream::Ephemeral, -second, 200)); + } + for i in 0..180 { + records.push(timed(Stream::Durable, i as i64, 50)); + } + for i in 0..ephemeral_during { + records.push(timed(Stream::Ephemeral, i as i64, ephemeral_ms)); + } + for second in 181..=420 { + records.push(timed(Stream::Durable, second, 50)); + records.push(timed(Stream::Ephemeral, second, 200)); + } + records + } + + /// A confirmed operation that took a stated time, so a window's median is + /// something the test controls rather than something it inherits. + fn timed(stream: Stream, offset_secs: i64, duration_ms: u64) -> OperationRecord { + let mut record = op(stream, offset_secs, Outcome::Confirmed); + record.duration_ms = duration_ms; + record.completed_at = + Some(record.submitted_at + TimeDelta::milliseconds(duration_ms as i64)); + record + } + + fn build_latency(records: &[OperationRecord]) -> StorageFaultReport { + StorageFaultReport::build( + records, + Some(fault()), + ENDPOINT, + OutageExpectation::LatencyDegradation { + slowed: vec![Stream::Ephemeral], + slowdown_floor: SLOWDOWN_FLOOR, + serving_floor_percent: SERVING_FLOOR, + }, + Duration::from_secs(120), + ) + } + fn build_partial(records: &[OperationRecord]) -> StorageFaultReport { StorageFaultReport::build( records, @@ -1222,6 +1346,68 @@ mod tests { ); } + /// The delay landed and the platform degraded rather than broke, which is + /// the outcome S15 and S17 are written to confirm. + #[test] + fn a_slowed_stream_that_kept_serving_is_the_expected_outcome() { + // 200ms at rest, 1s under the delay, and still completing throughout. + let report = build_latency(&latency_history(1_000, 180)); + + assert!( + report.findings.is_empty(), + "slower but still serving is what a delay is supposed to produce, got {:?}", + report.findings + ); + assert_eq!( + report.least_slowdown_factor, + Some(5.0), + "1000ms against a 200ms baseline is 5x" + ); + assert!( + report.quietest_stream_percent.is_none(), + "nothing is expected to go quiet under a delay, so there is no quiet verdict to draw" + ); + } + + /// The netem rule that never applied. Without this the run is full of + /// healthy numbers and no error anywhere, which is the worst artifact this + /// suite can produce — the same failure `outage-not-observed` exists to + /// catch, in the units a delay is measured in. + #[test] + fn a_delay_that_did_not_reach_the_executors_is_a_finding() { + // 240ms against a 200ms baseline: 1.2x, well under the 2x floor. + let report = build_latency(&latency_history(240, 180)); + + assert!( + report + .findings + .iter() + .any(|f| f.violation == OutageViolation::SlowdownNotObserved + && f.stream == Some(Stream::Ephemeral)), + "a stream that barely moved cannot evidence a delay, got {:?}", + report.findings + ); + } + + /// A delay is supposed to cost time, not work. A slowed stream that stops + /// entirely means the added latency broke something — a timeout fired, a + /// pool drained — and that is a different and worse outcome than slowness. + #[test] + fn a_delay_that_stopped_a_stream_rather_than_slowing_it_is_a_finding() { + // Slow enough to clear the slowdown floor, but only a trickle of + // operations got through, so it degraded past degradation. + let report = build_latency(&latency_history(1_000, 2)); + + assert!( + report + .findings + .iter() + .any(|f| f.violation == OutageViolation::UnexpectedStall), + "a stream serving 2 operations where it served 180 has stopped, got {:?}", + report.findings + ); + } + /// A stream the run barely drives cannot flip the verdict on its own. It is /// judged on how long it was silent, not on its rate against a baseline, so /// a trickle reads as the near-total silence it is rather than as a stream diff --git a/integration-tests/src/chaos/scenarios/storage_fault.rs b/integration-tests/src/chaos/scenarios/storage_fault.rs index 17f0812dd9..7276e3073b 100644 --- a/integration-tests/src/chaos/scenarios/storage_fault.rs +++ b/integration-tests/src/chaos/scenarios/storage_fault.rs @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Storage outage: the shared choreography behind S14, S16, S18 and S22. +//! Storage fault: the shared choreography behind S14, S16, S17, S18 and S22. //! -//! All four codes run this module. They differ in which store is taken away and -//! for how long, both of which are suite settings, and in what a reader should -//! expect of the result: +//! All five codes run this module. They differ in which store the fault is +//! aimed at, what it does to that store and for how long, all of which are +//! suite settings, and in what a reader should expect of the result: //! //! * **S16** (GOL-379) cuts the key-value cluster for the length of an AWS //! storage failover, about a minute. The key-value retry budget covers that, @@ -35,10 +35,16 @@ //! than past any platform budget. There is no budget to exhaust: golem-dev //! configures that client to retry forever. The claim is that a stall which //! outlasts the caller does not turn one operation into two. -//! -//! The driver is the same in all four because the difference is one of +//! * **S17** (GOL-375) leaves that same cache reachable and makes it slow. The +//! only scenario here that breaks nothing, and a different question rather +//! than a milder version of S18's: not whether the platform survives losing +//! its worker-status store, but whether it degrades or breaks when that store +//! gets slower. Going quiet would be a failure here rather than the expected +//! result. +//! +//! The driver is the same in all five because the difference is one of //! expectation, not of choreography. Nothing below asserts on which outcome -//! happened: the account it produces answers all four questions, and the +//! happened: the account it produces answers all five questions, and the //! oracles that fail the build — the scheduled-fire account and the //! exactly-once account — are the ones every one of them shares. //! @@ -74,16 +80,20 @@ //! commit anything durable at all, which is the opposite arrangement: the //! platform knows exactly what it is doing and cannot record any of it. //! -//! The **Redis cache** that S18 cuts is not a cluster at all but the front half -//! of the key-value layer. `NamespaceRoutedKeyValueStorage` sends the `Worker`, +//! The **Redis cache** that S18 and S17 aim at is not a cluster at all but the +//! front half of the key-value layer. `NamespaceRoutedKeyValueStorage` sends the `Worker`, //! `AgentStatus` and `AgentStatusCheckpoint` namespaces to it and everything -//! else to Postgres, so S18 removes exactly the part S16 leaves standing. Both -//! Aurora clusters stay reachable throughout. -//! -//! No stream is a control group under any of them. A durable increment needs -//! the running-workers set before it can start, the worker-status cache to -//! resolve its mode, and the oplog before it can finish, so `durable` degrades -//! under all three cuts and must not be read as untouched. +//! else to Postgres, so S18 removes exactly the part S16 leaves standing and +//! S17 slows the same part instead. Both Aurora clusters stay reachable +//! throughout either. +//! +//! No stream is a control group under the Aurora cuts. A durable increment +//! needs the running-workers set before it can start, the worker-status cache +//! to resolve its mode, and the oplog before it can finish, so `durable` +//! degrades under both of those and must not be read as untouched. The two +//! Redis scenarios are the exception, and deliberately so: there the streams +//! that keep working are evidence rather than noise, which is why they are +//! judged by their own expectations. See [`crate::chaos::OutageExpectation`]. //! //! ## The control is the baseline, not another pod //! From b6e6c0af8e49bd48089c3009f9dfce6bae8dc6e2 Mon Sep 17 00:00:00 2001 From: Kaur Matas <33095685+kmatasfp@users.noreply.github.com> Date: Fri, 28 Aug 2026 13:33:23 -0700 Subject: [PATCH 37/40] Add the S15 key-value PostgreSQL latency scenario --- golem-test-framework/src/benchmark/config.rs | 3 + .../chaos_suites/cloud-chaos.yaml | 180 ++++++++++++++++++ integration-tests/src/benchmarks/all.rs | 2 + integration-tests/src/chaos/mod.rs | 107 ++++++++++- integration-tests/src/chaos/outage.rs | 171 ++++++++++++++++- .../src/chaos/scenarios/storage_fault.rs | 34 +++- 6 files changed, 484 insertions(+), 13 deletions(-) diff --git a/golem-test-framework/src/benchmark/config.rs b/golem-test-framework/src/benchmark/config.rs index 29d266d337..3ceab0a70d 100644 --- a/golem-test-framework/src/benchmark/config.rs +++ b/golem-test-framework/src/benchmark/config.rs @@ -267,6 +267,9 @@ pub enum ChaosScenarioArg { S18, /// The same Redis cache slowed rather than removed. S17, + /// The key-value PostgreSQL cluster slowed rather than removed. S17's + /// mirror on the other half of the split key-value layer. + S15, } /// Density subcommand action. diff --git a/integration-tests/chaos_suites/cloud-chaos.yaml b/integration-tests/chaos_suites/cloud-chaos.yaml index 4acd087ddf..cb7acabdba 100644 --- a/integration-tests/chaos_suites/cloud-chaos.yaml +++ b/integration-tests/chaos_suites/cloud-chaos.yaml @@ -1435,6 +1435,19 @@ scenarios: # full of healthy numbers describing an experiment that never happened. slowdownFloor: 2.0 + # The other three, and the claim that makes the 6.59x above attributable + # to this cache rather than to a slower cluster. Their agents are + # resident and durable, so `AgentStatusFlusher` keeps the status blob off + # the commit path and nothing they do crosses Redis synchronously — and + # the first run bore that out exactly, at 61ms, 100ms and 91ms in all + # three windows. S15 makes the mirror claim with the two lists swapped. + steady: [durable, promise, scheduled] + + # Room for the spill of sharing executors and a runtime with a stream + # that is six times slower than usual, and nothing like the multiple a + # real dependency on this cache would produce. Measured at 1.00x. + steadyCeiling: 1.5 + # A delay should cost time, not work. Throughput is not expected to move # much: the per-stream in-flight budget is MAX_IN_FLIGHT/4 = 256, and at # 33 operations per second and 712ms each the stream sits around 23 in @@ -1459,3 +1472,170 @@ scenarios: delaySecs: 5 signalTimeoutSecs: 1800 + + # --------------------------------------------------------------------------- + # S15 — key-value PostgreSQL latency degradation (GOL-374) + # + # S17 with the two halves of the key-value layer swapped. That fault slows the + # Redis cache and this one slows the Aurora cluster behind it, and because + # `NamespaceRoutedKeyValueStorage` splits the namespaces between them, the two + # runs should move opposite sets of streams. Reading them side by side is the + # point: each is the other's control. + # + # ── Which streams actually reach this cluster ────────────────────────────── + # + # Worth spelling out, because "the workload is promise- and schedule-heavy" + # undersells it and the durable stream is the surprise. + # + # * durable — `AgentStatusFlusher::on_status_changed` updates the + # `RunningWorkers` recovery index *synchronously on the hot path*, and + # only when the tracking predicate flips. An idle agent invoked once goes + # Idle → Running → Idle, so the predicate flips twice and the invocation + # waits on two writes to this cluster. The status blob itself is off the + # commit path; the recovery index deliberately is not. + # * promise — `create` is one `set_if_not_exists` and `complete` is an + # `exists` followed by another, all in the `Promise` namespace, which + # routes here. + # * scheduled — the emitter's `schedule_poll_at` registers into the + # scheduler's own schema, which lives on this same cluster under + # `GOLEM__SCHEDULER_STORAGE__CONFIG__HOST`. + # * ephemeral — reaches none of it. `on_status_changed` returns immediately + # for an ephemeral agent, and its status goes to the Redis cache, so this + # is the one stream the delay should not touch. Under S17 it was the only + # one that did. + # + # ── What the pool arithmetic says, which is the risk this run carries ────── + # + # The executors hold 64 connections to the key-value cluster and 32 to the + # scheduler schema, per pod. Under a 500ms egress delay every statement costs + # one delayed round trip, so a pool of 64 clears about 128 statements a second + # and one of 32 about 64. + # + # Demand, from S17's measured rates over two executors: durable 33.9/s × 2 + # writes and promise 33.9/s × 3 gives about 85 statements per second per pod + # against the 128 the key-value pool can clear, and scheduled 50.8/s × 2 gives + # about 51 against the 64 the scheduler pool can. Both fit, neither by much. + # + # If either tips over, sqlx queues the acquire and gives up after its default + # 30s — and the key-value path has no retry policy in front of it, so + # `PoolTimedOut` reaches an `unwrap_or_else(panic!)` and takes the executor + # with it. That is the S16 finding (golemcloud/golem PR "keyvalue storage + # retries", still open), so an executor lost here is not a new defect but a + # second, sharper measurement of a known one: it would mean the panic is + # reachable from latency alone, without anything ever becoming unreachable. + # --------------------------------------------------------------------------- + - code: S15 + name: keyvalue-postgres-latency + enabled: true + + fault: + # As S17. Recorded for the archive rather than acted on: the workflow + # picks the manifest by scenario code and reads the Chaos Mesh kind out of + # the manifest itself. + kind: network-delay + target: worker-executor + # Every executor. A store that is slow for only some of them is a routing + # fault wearing a database's clothes. + mode: all + # Must equal phases.faultSecs; see the note on S16. + durationSecs: 300 + + phases: + # S17's, unchanged, so the two delays differ only in what they aim at. + baselineSecs: 300 + # 300s for S17's reasons and one more. Nothing here should approach the + # 120s attempt timeout, so there is no caller clock to outrun; the length + # is for a memory series long enough to have a slope and for a steady + # state rather than a transient. The one more is the pool arithmetic + # above: a pool that is going to saturate does it by filling, and filling + # takes time that a 60s window does not give it. + faultSecs: 300 + recoverySecs: 420 + + workload: + # Identical to the other five, deliberately, even though GOL-374 asks for + # "promise- and schedule-heavy" work and a reweighted population would + # press the pools harder. Two of the six storage runs are a matched pair + # and comparability is what makes the pair worth anything; and the durable + # stream turns out to be the heaviest user of this cluster anyway, at two + # synchronous writes per invocation. Deliberately overloading the pool is + # a different experiment — S22 is to S16 what that would be to this — and + # it should be its own scenario rather than a thumb on this one's scale. + durableAgents: 200 + ephemeralAgents: 50 + scheduledAgents: 0 + promiseAgents: 50 + quotaAgents: 0 + ratePerSec: 100 + + scheduled: + targets: 100 + intervalMillis: 2000 + leadSecs: 10 + # As the others. Unlike S17 the scheduler is *inside* this delay, so its + # claims and acknowledgements each gain a round trip — but 240s of lease + # against a claim that costs milliseconds plus 500ms is not a number this + # run can approach, and a lease expiry here would be a real finding rather + # than a threshold that was set too tight. + leaseBudgetSecs: 240 + + storage: + # The key-value Aurora cluster, which is also where the scheduler keeps + # its schema. The same endpoint S16 and S22 cut. Neither Redis nor the + # indexed-oplog cluster is touched: slowing a second store would leave + # every finding unattributable. + endpoint: golem-postgres-dev-keyvalue.cluster-cgfyoqmjq7tc.us-east-1.rds.amazonaws.com + + expect: + # As S17. Under a delay nothing is supposed to fall silent, so silence + # is not available as evidence that the fault landed and time has to + # stand in for it. + kind: latency-degradation + + # The three streams that reach this cluster synchronously, per the + # walk-through above. All three should move by a lot: durable's baseline + # median was 61ms and it waits on two writes, so about 1061ms or 17x; + # promise's was 91ms against three, so about 1591ms or 17x; scheduled's + # was 100ms against one, so about 600ms or 6x. + slowed: [durable, promise, scheduled] + + # 2.0, as S17, and for the same reason rather than for the arithmetic + # above: the floor exists to fail a run where the netem rule never + # applied, which leaves every stream near 1.0x. Setting it near the + # predicted 6x would instead be a second, unannounced assertion that the + # prediction is right, and the prediction is the thing under test. + slowdownFloor: 2.0 + + # The control, and the mirror image of S17's list. An ephemeral agent + # never enters the recovery index and its status goes to Redis, so if + # this stream slows with the others, the delay reached something wider + # than this cluster and the run cannot say what it measured. + steady: [ephemeral] + + # As S17. Ephemeral is the noisiest stream in the suite at rest — its + # p99 runs to seven times its median — but the check is on medians, and + # the median has held at 202-212ms across every storage run so far. + steadyCeiling: 1.5 + + # A delay should cost time, not work. At 33.9 operations per second and + # about 1.6s each, promise sits around 54 of its 256 in-flight budget, + # and the other streams less, so nothing queues at the driver and + # throughput should hold. A stream falling below half its baseline means + # the latency broke something instead — and here that has a named + # candidate, which is the pool timeout above. + servingFloorPercent: 50 + + # Recorded, not asserted, and expected to be near zero: nothing was ever + # unreachable, so there is no reconnect to wait for. If an executor was + # lost to the pool timeout this will be the number that shows it, as the + # replacement has to reload its agents before it can serve. + recoveryBudgetSecs: 60 + + retryPolicy: + # Identical to the other five. Nothing should time out and so nothing + # should retry; a retry count above zero is itself a finding. + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 diff --git a/integration-tests/src/benchmarks/all.rs b/integration-tests/src/benchmarks/all.rs index 526a4c7af0..f3340dff9b 100644 --- a/integration-tests/src/benchmarks/all.rs +++ b/integration-tests/src/benchmarks/all.rs @@ -603,6 +603,7 @@ async fn run_chaos( ChaosScenarioArg::S14 => chaos::ScenarioCode::S14, ChaosScenarioArg::S18 => chaos::ScenarioCode::S18, ChaosScenarioArg::S17 => chaos::ScenarioCode::S17, + ChaosScenarioArg::S15 => chaos::ScenarioCode::S15, }; let config = suite .scenario(code, allow_disabled) @@ -653,6 +654,7 @@ async fn run_chaos( chaos::scenarios::s9::run(&config, &manifest, &deps, &signals, &outputs).await } code @ (chaos::ScenarioCode::S14 + | chaos::ScenarioCode::S15 | chaos::ScenarioCode::S16 | chaos::ScenarioCode::S17 | chaos::ScenarioCode::S18 diff --git a/integration-tests/src/chaos/mod.rs b/integration-tests/src/chaos/mod.rs index bec68df77c..f097eb2f22 100644 --- a/integration-tests/src/chaos/mod.rs +++ b/integration-tests/src/chaos/mod.rs @@ -106,6 +106,9 @@ pub enum ScenarioCode { /// The same Redis cache still reachable but slowed, to ask whether the /// platform degrades or breaks when its worker-status store gets slower. S17, + /// The key-value PostgreSQL cluster still reachable but slowed. S17's + /// mirror on the other half of the split key-value layer. + S15, } impl ScenarioCode { @@ -127,13 +130,14 @@ impl ScenarioCode { ScenarioCode::S14 => "S14", ScenarioCode::S18 => "S18", ScenarioCode::S17 => "S17", + ScenarioCode::S15 => "S15", } } /// Every scenario this driver implements. The suite YAML is checked against /// this list, so a scenario cannot be enabled in YAML without code behind /// it, nor implemented without an operational switch in front of it. - pub const ALL: [ScenarioCode; 16] = [ + pub const ALL: [ScenarioCode; 17] = [ ScenarioCode::S1, ScenarioCode::S3, ScenarioCode::S5, @@ -146,6 +150,7 @@ impl ScenarioCode { ScenarioCode::S12, ScenarioCode::S13, ScenarioCode::S14, + ScenarioCode::S15, ScenarioCode::S16, ScenarioCode::S17, ScenarioCode::S18, @@ -800,6 +805,38 @@ pub enum OutageExpectation { /// 240ms on `ephemeral` — and a fixed ceiling would be slack for one /// and impossible for the other. slowdown_floor: f64, + /// The streams the delay is expected *not* to reach, and whose + /// steadiness is therefore the run's evidence that it reached only what + /// it aimed at. + /// + /// The latency counterpart to what + /// [`Self::PartialWorkload::serving_floor_percent`] does for a cut, and + /// it earns its place for the same reason. Both halves of the key-value + /// layer are delayed by a scenario in this suite, and each predicts a + /// different split: S17 slows the Redis half, where only a lifecycle + /// boundary crosses synchronously, so `ephemeral` moves and the other + /// three do not; S15 slows the PostgreSQL half, where the running-workers + /// recovery index, the promise keys and the scheduler's own schema all + /// live, so those three move and `ephemeral` does not. Naming the far + /// side turns each run into a test of that routing rather than a + /// measurement taken on faith. + /// + /// Optional, and empty means the run asserts nothing about the streams + /// it did not name. + #[serde(default)] + steady: Vec, + /// The most a steady stream's during-fault median may be as a multiple + /// of its own before-fault median. + /// + /// Not 1.0. A stream on the far side of the delay still shares + /// executors, connection pools and a tokio runtime with the streams on + /// the near side, so some spill is expected and is not itself a finding. + /// The number has to sit above that spill and below the multiple a real + /// dependency would produce, and those are far apart: S17 measured 6.59x + /// on the stream it delayed and no movement at all — 61ms, 100ms and + /// 91ms, identical across all three windows — on the three it did not. + #[serde(default = "default_steady_ceiling")] + steady_ceiling: f64, /// The least of its own baseline rate *every* stream must still hold, /// including the slowed ones. /// @@ -811,6 +848,14 @@ pub enum OutageExpectation { }, } +/// Applied when a `latency-degradation` expectation names steady streams +/// without saying how steady. Loose enough that ordinary run-to-run spread on +/// an undelayed stream does not trip it, which matters because the finding it +/// raises says the routing is wrong. +fn default_steady_ceiling() -> f64 { + 1.5 +} + impl OutageExpectation { /// The quiet floor, which both variants carry and apply to a different set. pub fn quiet_floor_percent(&self) -> f64 { @@ -845,6 +890,27 @@ impl OutageExpectation { } } + /// Whether this stream is one the delay is expected not to reach. + pub fn expects_steady(&self, stream: Stream) -> bool { + match self { + OutageExpectation::LatencyDegradation { steady, .. } => steady.contains(&stream), + _ => false, + } + } + + /// The multiple of its own baseline median a steady stream may not exceed, + /// or `None` where the expectation names no steady streams to hold to it. + pub fn steady_ceiling(&self) -> Option { + match self { + OutageExpectation::LatencyDegradation { + steady, + steady_ceiling, + .. + } if !steady.is_empty() => Some(*steady_ceiling), + _ => None, + } + } + /// Whether this stream is one whose silence would prove the cut landed. /// /// `WholeWorkload` says yes to everything, which is what makes it the @@ -1242,6 +1308,8 @@ impl ScenarioConfig { OutageExpectation::LatencyDegradation { slowed, slowdown_floor, + steady, + steady_ceiling, serving_floor_percent, } => { // A floor of 1.0 or less asks a stream to be no slower than it @@ -1257,6 +1325,41 @@ impl ScenarioConfig { } self.check_serving_floor(*serving_floor_percent)?; self.check_evidence_streams("slowed", slowed, "slowdown")?; + // The steady list is optional, so it is checked only when + // present — but a present one carries the same two ways of + // being useless as the slowed list, plus one of its own. + for stream in steady { + if !self.drives_stream(*stream) { + anyhow::bail!( + "chaos scenario {}: expect.steady names `{stream}`, which this \ + scenario's workload never drives, so its steadiness during the \ + fault would prove nothing", + self.code + ); + } + if slowed.contains(stream) { + anyhow::bail!( + "chaos scenario {}: expect names `{stream}` as both slowed and \ + steady, so the run would demand the same stream both move and \ + stay put", + self.code + ); + } + } + // A ceiling at or below 1.0 asks an undelayed stream to run no + // slower than its own baseline median, which run-to-run spread + // alone breaks. The finding it would then raise says the + // platform routes a namespace somewhere other than where the + // source says, and that is far too strong a claim to make on + // noise. + if !steady.is_empty() && (!steady_ceiling.is_finite() || *steady_ceiling <= 1.0) { + anyhow::bail!( + "chaos scenario {}: expect.steadyCeiling is {steady_ceiling}, and a \ + factor at or below 1.0 is broken by ordinary run-to-run spread on a \ + stream the delay never reached", + self.code + ); + } } } if config.endpoint.trim().is_empty() { @@ -1727,6 +1830,7 @@ mod tests { assert_eq!(ScenarioCode::parse("s14"), Some(ScenarioCode::S14)); assert_eq!(ScenarioCode::parse("s18"), Some(ScenarioCode::S18)); assert_eq!(ScenarioCode::parse("s17"), Some(ScenarioCode::S17)); + assert_eq!(ScenarioCode::parse("s15"), Some(ScenarioCode::S15)); assert_eq!(ScenarioCode::parse("S99"), None); } @@ -1773,6 +1877,7 @@ mod tests { entry.require_rollback().unwrap(); } ScenarioCode::S14 + | ScenarioCode::S15 | ScenarioCode::S16 | ScenarioCode::S17 | ScenarioCode::S18 diff --git a/integration-tests/src/chaos/outage.rs b/integration-tests/src/chaos/outage.rs index b4fa2e9eae..40a2100cd0 100644 --- a/integration-tests/src/chaos/outage.rs +++ b/integration-tests/src/chaos/outage.rs @@ -66,15 +66,29 @@ //! So `ephemeral` stops and `durable`, `scheduled` and `promise` carry on, and //! the streams still answering are evidence rather than noise. //! +//! S17 and S15 delay the two halves rather than cutting them, and the same +//! split decides which streams move. Under S17 that is `ephemeral` alone, which +//! its run confirmed at 6.59x against three streams that did not shift by a +//! millisecond. Under S15 it is the other three: a durable invocation flips its +//! agent between tracked and untracked in the running-workers index and waits +//! for the write, a promise operation reads and writes promise keys, and +//! registering a schedule writes to the scheduler's schema — all on the +//! PostgreSQL half — while an ephemeral agent is excluded from that index by +//! `AgentStatusFlusher::on_status_changed` and never touches it. +//! //! ### What fails the run //! -//! Three things, and all are statements about the experiment rather than about +//! Five things, and all are statements about the experiment rather than about //! latency: //! //! * [`OutageViolation::OutageNotObserved`] — a stream the cut was supposed to //! stop kept working, so the fault did not land where the run says it did. //! * [`OutageViolation::UnexpectedStall`] — a stream the cut was *not* supposed //! to touch stopped anyway. +//! * [`OutageViolation::SlowdownNotObserved`] — a stream a delay was aimed at +//! ran no slower than at rest, so the netem rule proved nothing. +//! * [`OutageViolation::UnexpectedSlowdown`] — a stream a delay was *not* aimed +//! at slowed with it, which says the routing is not what the source says. //! * [`OutageViolation::StreamNeverRecovered`] — a stream that was working //! before the outage produced nothing at all after the heal. //! @@ -147,6 +161,16 @@ pub enum OutageViolation { /// netem rule that failed to apply leaves a run full of healthy numbers and /// no error anywhere — the worst artifact this suite can produce. SlowdownNotObserved, + /// A stream a delay was expected to leave alone slowed down with it. + /// + /// The counterpart to [`Self::SlowdownNotObserved`], and the same argument + /// [`Self::UnexpectedStall`] makes for a cut. A delay aimed at one half of + /// the key-value layer is a claim about routing: these namespaces go to the + /// store being slowed and those go elsewhere. A stream on the far side + /// moving with it says the claim is wrong — and a rule that only looked for + /// slowdown would have read that as its cleanest possible pass, because + /// everything the run aimed at did indeed get slower. + UnexpectedSlowdown, /// A stream that was confirming operations before the outage confirmed /// nothing at all after the heal. StreamNeverRecovered, @@ -158,6 +182,7 @@ impl OutageViolation { OutageViolation::OutageNotObserved => "outage-not-observed", OutageViolation::UnexpectedStall => "unexpected-stall", OutageViolation::SlowdownNotObserved => "slowdown-not-observed", + OutageViolation::UnexpectedSlowdown => "unexpected-slowdown", OutageViolation::StreamNeverRecovered => "stream-never-recovered", } } @@ -336,6 +361,15 @@ pub struct StorageFaultReport { /// another. #[serde(default, skip_serializing_if = "Option::is_none")] pub least_slowdown_factor: Option, + /// The most any stream the delay was expected *not* to reach moved, as a + /// multiple of its own before-fault median. `None` unless the expectation + /// names steady streams. + /// + /// Read together with `least_slowdown_factor`, the pair is the whole + /// experiment in two numbers: how far the streams behind the delayed store + /// moved, and how far the ones that should not be behind it did. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub greatest_steady_factor: Option, pub cells: Vec, /// What the outage began underneath, per stream. Empty for a run that never /// learned when the fault was. @@ -485,6 +519,7 @@ impl StorageFaultReport { quietest_stream_percent: None, least_serving_stream_percent: None, least_slowdown_factor: None, + greatest_steady_factor: None, cells, caught_in_flight: caught_in_flight(records, fault), recovery: Vec::new(), @@ -639,6 +674,40 @@ impl StorageFaultReport { } } + // The far side of the same claim. A delay aimed at one store predicts + // both which streams move and which do not, and only the second half + // can catch a delay that landed somewhere wider than the run says. + if let Some(steady_ceiling) = self.expect.steady_ceiling() { + let mut steadies: Vec<(Stream, f64)> = served_before + .iter() + .copied() + .filter(|stream| self.expect.expects_steady(*stream)) + .filter_map(|stream| { + let baseline = self.median_ms(stream, Window::BeforeFault)?; + let during = self.median_ms(stream, Window::DuringFault)?; + (baseline > 0.0).then_some((stream, round2(during / baseline))) + }) + .collect(); + steadies.sort_by(|a, b| b.1.total_cmp(&a.1)); + + if let Some(&(stream, most)) = steadies.first() { + self.greatest_steady_factor = Some(most); + if most > steady_ceiling { + self.findings.push(OutageFinding { + violation: OutageViolation::UnexpectedSlowdown, + stream: Some(stream), + detail: format!( + "{stream} does not depend on {} and was expected to run at its own \ + pace through the delay, but ran at {most}x its own baseline median \ + against a {steady_ceiling}x ceiling — the storage this stream \ + actually reaches is not what the routing says it is", + self.endpoint + ), + }); + } + } + } + let Some(serving_floor) = self.expect.serving_floor_percent() else { return; }; @@ -799,6 +868,18 @@ impl StorageFaultReport { "no stream had a before-fault baseline to be judged against".to_string() } }; + // The far side of a delay, when the scenario named one. Printed + // next to the slowdown rather than left in the cells, because the + // two numbers only mean anything together: a run where everything + // moved by the same multiple measured a slower cluster, not a + // slower store. + let steady = match (self.greatest_steady_factor, self.expect.steady_ceiling()) { + (Some(most), Some(ceiling)) => format!( + ", the streams expected to be left alone ran at no more than {most}x theirs \ + (ceiling {ceiling}x)" + ), + _ => String::new(), + }; let serving = match ( self.least_serving_stream_percent, self.expect.serving_floor_percent(), @@ -810,7 +891,7 @@ impl StorageFaultReport { _ => String::new(), }; lines.push(format!( - "Storage outage: {quiet}{serving} while {} was unreachable, and the workload held \ + "Storage outage: {quiet}{steady}{serving} while {} was unreachable, and the workload held \ {share}% of its baseline throughput across that window", self.endpoint )); @@ -1111,18 +1192,30 @@ mod tests { } const SLOWDOWN_FLOOR: f64 = 2.0; + const STEADY_CEILING: f64 = 1.5; /// A workload where every stream keeps serving and `ephemeral` is the one /// the delay is aimed at. `ephemeral_ms` is what its operations take during /// the fault, against a 200ms baseline. fn latency_history(ephemeral_ms: u64, ephemeral_during: usize) -> Vec { + latency_history_with(ephemeral_ms, ephemeral_during, 50) + } + + /// As [`latency_history`], with the far-side stream's during-fault duration + /// under the test's control too, so a run where the delay reached more than + /// it was aimed at can be built. + fn latency_history_with( + ephemeral_ms: u64, + ephemeral_during: usize, + durable_during_ms: u64, + ) -> Vec { let mut records = Vec::new(); for second in 1..=300 { records.push(timed(Stream::Durable, -second, 50)); records.push(timed(Stream::Ephemeral, -second, 200)); } for i in 0..180 { - records.push(timed(Stream::Durable, i as i64, 50)); + records.push(timed(Stream::Durable, i as i64, durable_during_ms)); } for i in 0..ephemeral_during { records.push(timed(Stream::Ephemeral, i as i64, ephemeral_ms)); @@ -1152,6 +1245,26 @@ mod tests { OutageExpectation::LatencyDegradation { slowed: vec![Stream::Ephemeral], slowdown_floor: SLOWDOWN_FLOOR, + steady: Vec::new(), + steady_ceiling: STEADY_CEILING, + serving_floor_percent: SERVING_FLOOR, + }, + Duration::from_secs(120), + ) + } + + /// The same delay, with the claim S15 and S17 both make about the streams + /// on the far side of the store written down. + fn build_latency_steady(records: &[OperationRecord]) -> StorageFaultReport { + StorageFaultReport::build( + records, + Some(fault()), + ENDPOINT, + OutageExpectation::LatencyDegradation { + slowed: vec![Stream::Ephemeral], + slowdown_floor: SLOWDOWN_FLOOR, + steady: vec![Stream::Durable], + steady_ceiling: STEADY_CEILING, serving_floor_percent: SERVING_FLOOR, }, Duration::from_secs(120), @@ -1408,6 +1521,58 @@ mod tests { ); } + /// Both halves of a delay's claim, held at once: the stream behind the + /// slowed store moved and the stream that is not behind it did not. This is + /// the shape S17 measured and the one S15 predicts with the streams + /// swapped, and it is what makes either run attributable to the store + /// rather than to a slower cluster. + #[test] + fn a_steady_stream_that_stayed_steady_is_what_makes_the_slowdown_attributable() { + let report = build_latency_steady(&latency_history_with(1_000, 180, 50)); + + assert!( + report.findings.is_empty(), + "one stream slower and the other unchanged is the expected outcome, got {:?}", + report.findings + ); + assert_eq!(report.least_slowdown_factor, Some(5.0)); + assert_eq!( + report.greatest_steady_factor, + Some(1.0), + "50ms against a 50ms baseline has not moved" + ); + } + + /// The finding a rule that only looks for slowdown cannot raise. Everything + /// the delay was aimed at did get slower, so the slowdown check passes and + /// the run reads as clean — while a stream the routing says is nowhere near + /// the delayed store moved with it, which means the routing is wrong or the + /// rule landed wider than the manifest says. + #[test] + fn a_stream_the_delay_should_not_have_touched_slowing_with_it_is_a_finding() { + // The delayed stream still clears its floor at 5x, so nothing else in + // the report objects. + let report = build_latency_steady(&latency_history_with(1_000, 180, 550)); + + assert!( + report + .findings + .iter() + .any(|f| f.violation == OutageViolation::UnexpectedSlowdown + && f.stream == Some(Stream::Durable)), + "11x on a stream that does not depend on the delayed store is a finding, got {:?}", + report.findings + ); + assert!( + !report + .findings + .iter() + .any(|f| f.violation == OutageViolation::SlowdownNotObserved), + "the aimed-at stream did slow down, so that check should be quiet: {:?}", + report.findings + ); + } + /// A stream the run barely drives cannot flip the verdict on its own. It is /// judged on how long it was silent, not on its rate against a baseline, so /// a trickle reads as the near-total silence it is rather than as a stream diff --git a/integration-tests/src/chaos/scenarios/storage_fault.rs b/integration-tests/src/chaos/scenarios/storage_fault.rs index 7276e3073b..85d240c020 100644 --- a/integration-tests/src/chaos/scenarios/storage_fault.rs +++ b/integration-tests/src/chaos/scenarios/storage_fault.rs @@ -12,9 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Storage fault: the shared choreography behind S14, S16, S17, S18 and S22. +//! Storage fault: the shared choreography behind S14, S15, S16, S17, S18 and +//! S22. //! -//! All five codes run this module. They differ in which store the fault is +//! All six codes run this module. They differ in which store the fault is //! aimed at, what it does to that store and for how long, all of which are //! suite settings, and in what a reader should expect of the result: //! @@ -41,8 +42,14 @@ //! its worker-status store, but whether it degrades or breaks when that store //! gets slower. Going quiet would be a failure here rather than the expected //! result. -//! -//! The driver is the same in all five because the difference is one of +//! * **S15** (GOL-374) slows the key-value cluster instead, so it is to S16 +//! what S17 is to S18 and the mirror image of S17 at the same time. The +//! claim is the same one — degradation rather than breakage — but on the +//! opposite set of streams, and it carries a risk the Redis delay does not: +//! this side's connection pools are bounded and its failure path is a panic. +//! See the suite entry for the arithmetic. +//! +//! The driver is the same in all six because the difference is one of //! expectation, not of choreography. Nothing below asserts on which outcome //! happened: the account it produces answers all five questions, and the //! oracles that fail the build — the scheduled-fire account and the @@ -87,13 +94,22 @@ //! S17 slows the same part instead. Both Aurora clusters stay reachable //! throughout either. //! -//! No stream is a control group under the Aurora cuts. A durable increment +//! S15 aims at the back half of that same layer, which is the key-value cluster +//! again, and the split decides its streams too. `durable` waits on the +//! `RunningWorkers` recovery index, which `AgentStatusFlusher::on_status_changed` +//! updates synchronously whenever an agent crosses between tracked and +//! untracked; `promise` reads and writes promise keys; the scheduler registers +//! into its own schema on the same cluster. `ephemeral` reaches none of it, and +//! is the one stream that should not move. +//! +//! No stream is a control group under the Aurora *cuts*. A durable increment //! needs the running-workers set before it can start, the worker-status cache //! to resolve its mode, and the oplog before it can finish, so `durable` -//! degrades under both of those and must not be read as untouched. The two -//! Redis scenarios are the exception, and deliberately so: there the streams -//! that keep working are evidence rather than noise, which is why they are -//! judged by their own expectations. See [`crate::chaos::OutageExpectation`]. +//! degrades under both of those and must not be read as untouched. The three +//! scenarios aimed at one half of the key-value layer are the exception, and +//! deliberately so: there the streams that keep working — or keep their pace — +//! are evidence rather than noise, which is why they are judged by their own +//! expectations. See [`crate::chaos::OutageExpectation`]. //! //! ## The control is the baseline, not another pod //! From 590b88d4edb4a28572d6399e041ffdb982edb1af Mon Sep 17 00:00:00 2001 From: Kaur Matas <33095685+kmatasfp@users.noreply.github.com> Date: Fri, 28 Aug 2026 14:53:42 -0700 Subject: [PATCH 38/40] Separate the serving floor from the slowdown it follows from --- .../chaos_suites/cloud-chaos.yaml | 68 ++++++++------ integration-tests/src/chaos/outage.rs | 91 ++++++++++++++++--- 2 files changed, 121 insertions(+), 38 deletions(-) diff --git a/integration-tests/chaos_suites/cloud-chaos.yaml b/integration-tests/chaos_suites/cloud-chaos.yaml index cb7acabdba..998a054294 100644 --- a/integration-tests/chaos_suites/cloud-chaos.yaml +++ b/integration-tests/chaos_suites/cloud-chaos.yaml @@ -1504,25 +1504,33 @@ scenarios: # is the one stream the delay should not touch. Under S17 it was the only # one that did. # - # ── What the pool arithmetic says, which is the risk this run carries ────── + # ── What the first run measured, which was not what it predicted ────────── # - # The executors hold 64 connections to the key-value cluster and 32 to the - # scheduler schema, per pod. Under a 500ms egress delay every statement costs - # one delayed round trip, so a pool of 64 clears about 128 statements a second - # and one of 32 about 64. + # Run 33210769609. The prediction here was 17x on durable and promise and 6x + # on scheduled, from counting round trips, with a note that the connection + # pools would fit "neither by much". The measurement was 644x, 939x and + # 1640x: durable went from 61ms to 39s, promise from 92ms to 86s, scheduled + # from 62ms to 102s. A 500ms delay came out three orders of magnitude larger + # at the caller. # - # Demand, from S17's measured rates over two executors: durable 33.9/s × 2 - # writes and promise 33.9/s × 3 gives about 85 statements per second per pod - # against the 128 the key-value pool can clear, and scheduled 50.8/s × 2 gives - # about 51 against the 64 the scheduler pool can. Both fit, neither by much. + # So this is not a degradation scenario in practice. Counting round trips + # gives the cost of one operation running alone; it says nothing about what + # happens when every operation is doing the same thing to a pool of 64 + # connections, and queueing is the whole answer. Nothing was lost — every key + # exactly once, every schedule fired once, 300/300 agents consistent, every + # stream serving again within 4s — but the platform did not stay usable. # - # If either tips over, sqlx queues the acquire and gives up after its default - # 30s — and the key-value path has no retry policy in front of it, so - # `PoolTimedOut` reaches an `unwrap_or_else(panic!)` and takes the executor - # with it. That is the S16 finding (golemcloud/golem PR "keyvalue storage - # retries", still open), so an executor lost here is not a new defect but a - # second, sharper measurement of a known one: it would mean the panic is - # reachable from latency alone, without anything ever becoming unreachable. + # Two predictions that were wrong in an informative direction: + # + # * No executor was lost. `PoolTimedOut` never reached the panic, so the + # S16 finding was not reproduced from latency alone. The queue built in + # front of the pool rather than inside its acquire timeout. + # * `ephemeral`, which reaches none of this cluster, ran 235x slower. 46 + # seconds is ninety 500ms round trips and the stream makes none, so it was + # queueing behind the three streams that do. Whatever gate that is, it is + # not the per-account concurrency semaphore — golem-dev sets + # `MAX_CONCURRENT_AGENTS_PER_EXECUTOR` to the unlimited sentinel, so that + # path is bypassed. See the runbook. # --------------------------------------------------------------------------- - code: S15 name: keyvalue-postgres-latency @@ -1593,10 +1601,9 @@ scenarios: kind: latency-degradation # The three streams that reach this cluster synchronously, per the - # walk-through above. All three should move by a lot: durable's baseline - # median was 61ms and it waits on two writes, so about 1061ms or 17x; - # promise's was 91ms against three, so about 1591ms or 17x; scheduled's - # was 100ms against one, so about 600ms or 6x. + # walk-through above. Measured at 644x, 939x and 1640x — see the note + # above for why the round-trip arithmetic underestimated that by two + # orders of magnitude. slowed: [durable, promise, scheduled] # 2.0, as S17, and for the same reason rather than for the arithmetic @@ -1614,15 +1621,22 @@ scenarios: # As S17. Ephemeral is the noisiest stream in the suite at rest — its # p99 runs to seven times its median — but the check is on medians, and - # the median has held at 202-212ms across every storage run so far. + # the median has held at 196-212ms across every storage run so far. + # + # The first run broke this at 235x, and deliberately left as it is: the + # ceiling did its job. Raising it to accommodate the result would turn + # the one number that caught the coupling into a number that describes + # it. steadyCeiling: 1.5 - # A delay should cost time, not work. At 33.9 operations per second and - # about 1.6s each, promise sits around 54 of its 256 in-flight budget, - # and the other streams less, so nothing queues at the driver and - # throughput should hold. A stream falling below half its baseline means - # the latency broke something instead — and here that has a named - # candidate, which is the pool timeout above. + # A delay should cost time, not work. Held at 50 for comparability, but + # the first run showed this floor cannot be read on its own once the + # in-flight budget saturates: throughput is concurrency over latency, so + # a stream 900x slower cannot hold its rate however healthy it is. + # Promise came in at 11.9% of baseline while carrying a hundred times + # more work in flight than at rest. `outage.rs` now suppresses the + # finding when the stream's own slowdown accounts for the shortfall, + # which is the only case where the two floors were measuring one fact. servingFloorPercent: 50 # Recorded, not asserted, and expected to be near zero: nothing was ever diff --git a/integration-tests/src/chaos/outage.rs b/integration-tests/src/chaos/outage.rs index 40a2100cd0..61304edf70 100644 --- a/integration-tests/src/chaos/outage.rs +++ b/integration-tests/src/chaos/outage.rs @@ -167,9 +167,19 @@ pub enum OutageViolation { /// [`Self::UnexpectedStall`] makes for a cut. A delay aimed at one half of /// the key-value layer is a claim about routing: these namespaces go to the /// store being slowed and those go elsewhere. A stream on the far side - /// moving with it says the claim is wrong — and a rule that only looked for - /// slowdown would have read that as its cleanest possible pass, because - /// everything the run aimed at did indeed get slower. + /// moving with it says something is wrong with that picture — and a rule + /// that only looked for slowdown would have read it as its cleanest + /// possible pass, because everything the run aimed at did indeed get + /// slower. + /// + /// It says *something* rather than *what*, and S15's first run is why the + /// wording is careful. `ephemeral` came out at 234.83x there, which cannot + /// be a route it takes itself: 46 seconds is ninety 500ms round trips, and + /// the stream makes none. It was queueing behind the three streams that do, + /// through a gate the executor shares across an account. So the three + /// readings are that the routing is wrong, that the fault landed wider than + /// the manifest names, or that the platform couples streams that the + /// storage layer keeps apart — and the third is not the least interesting. UnexpectedSlowdown, /// A stream that was confirming operations before the outage confirmed /// nothing at all after the heal. @@ -699,8 +709,8 @@ impl StorageFaultReport { detail: format!( "{stream} does not depend on {} and was expected to run at its own \ pace through the delay, but ran at {most}x its own baseline median \ - against a {steady_ceiling}x ceiling — the storage this stream \ - actually reaches is not what the routing says it is", + against a {steady_ceiling}x ceiling — so either it reaches that \ + storage after all, or it is queueing behind the streams that do", self.endpoint ), }); @@ -734,17 +744,52 @@ impl StorageFaultReport { if let Some(&(stream, least)) = serving.first() { self.least_serving_stream_percent = Some(least); - if least < serving_floor { - self.findings.push(OutageFinding { - violation: OutageViolation::UnexpectedStall, - stream: Some(stream), - detail: format!( + // Under a delay the two floors are not independent, and treating + // them as though they were reports the same fact twice. + // + // Throughput is concurrency over latency, and the workload offers a + // fixed in-flight budget per stream. So once a stream's latency + // rises far enough to saturate that budget, its throughput *must* + // fall in proportion, and the serving floor is then measuring the + // slowdown the run already reported rather than anything new. S15's + // first run made this concrete: promise held 11.88% of baseline + // while running 939x slower, which is 108 times more work in flight + // than at rest, not a stall. + // + // The invariant that separates them: share × slowdown is the ratio + // of during-fault concurrency to baseline concurrency. At or above + // 100% the stream carried more work than it did at rest and did not + // stop, whatever its throughput did. Below it, work is genuinely + // being lost, which is what this finding exists for. + let explained_by_slowdown = self + .expect + .slowdown_floor() + .and(self.median_ms(stream, Window::BeforeFault)) + .zip(self.median_ms(stream, Window::DuringFault)) + .map(|(before, during)| before > 0.0 && least * (during / before) >= 100.0) + .unwrap_or(false); + if least < serving_floor && !explained_by_slowdown { + let detail = if self.expect.slowdown_floor().is_some() { + format!( + "{stream} was expected to slow down while {} was delayed rather than \ + stop, but held only {least}% of its own baseline against a \ + {serving_floor}% floor, and its own latency does not account for the \ + shortfall — so the delay cost this stream work and not only time", + self.endpoint + ) + } else { + format!( "{stream} does not depend on {} and was expected to carry on through \ the cut, but held only {least}% of its own baseline against a \ {serving_floor}% floor — the storage this stream actually reaches is \ not what the routing says it is", self.endpoint - ), + ) + }; + self.findings.push(OutageFinding { + violation: OutageViolation::UnexpectedStall, + stream: Some(stream), + detail, }); } } @@ -1521,6 +1566,30 @@ mod tests { ); } + /// The throughput drop a delay forces on its own, which is not a second + /// finding. The workload offers each stream a fixed in-flight budget, so a + /// stream running a hundred times slower cannot hold its baseline rate no + /// matter how healthy it is — and reporting that as a stall restates the + /// slowdown under another name. S15's first run hit exactly this: promise + /// held 11.88% of baseline at 939x, which is a hundred times more work in + /// flight than at rest. + #[test] + fn a_stream_too_slow_to_hold_its_rate_has_not_stopped() { + // 100x slower, and serving 5 operations where it served 180. That is + // 2.8% of baseline, far under the 50% floor — and 2.8% x 100 is 278% of + // its baseline concurrency, so it carried more work rather than less. + let report = build_latency(&latency_history(20_000, 5)); + + assert!( + !report + .findings + .iter() + .any(|f| f.violation == OutageViolation::UnexpectedStall), + "a stream whose own latency accounts for its rate has not stalled, got {:?}", + report.findings + ); + } + /// Both halves of a delay's claim, held at once: the stream behind the /// slowed store moved and the stream that is not behind it did not. This is /// the shape S17 measured and the one S15 predicts with the streams From d7188b99a3185832a408f6a36eb7f04ec1bb9731 Mon Sep 17 00:00:00 2001 From: Kaur Matas <33095685+kmatasfp@users.noreply.github.com> Date: Fri, 28 Aug 2026 16:22:38 -0700 Subject: [PATCH 39/40] Add the S15 elimination series --- golem-test-framework/src/benchmark/config.rs | 6 + .../chaos_suites/cloud-chaos.yaml | 236 ++++++++++++++++++ integration-tests/src/benchmarks/all.rs | 6 + integration-tests/src/chaos/mod.rs | 100 +++++++- integration-tests/src/chaos/outage.rs | 8 + integration-tests/src/chaos/result.rs | 3 + 6 files changed, 357 insertions(+), 2 deletions(-) diff --git a/golem-test-framework/src/benchmark/config.rs b/golem-test-framework/src/benchmark/config.rs index 3ceab0a70d..7eca7e55d6 100644 --- a/golem-test-framework/src/benchmark/config.rs +++ b/golem-test-framework/src/benchmark/config.rs @@ -270,6 +270,12 @@ pub enum ChaosScenarioArg { /// The key-value PostgreSQL cluster slowed rather than removed. S17's /// mirror on the other half of the split key-value layer. S15, + /// S15 driving `ephemeral` alone: the control for the elimination series. + S15A, + /// S15A plus `durable`. + S15B, + /// S15B plus `promise`. + S15C, } /// Density subcommand action. diff --git a/integration-tests/chaos_suites/cloud-chaos.yaml b/integration-tests/chaos_suites/cloud-chaos.yaml index 998a054294..758728331b 100644 --- a/integration-tests/chaos_suites/cloud-chaos.yaml +++ b/integration-tests/chaos_suites/cloud-chaos.yaml @@ -1653,3 +1653,239 @@ scenarios: delaySecs: 5 signalTimeoutSecs: 1800 + + # --------------------------------------------------------------------------- + # S15A / S15B / S15C — the S15 elimination series + # + # S15's first run left one thing unexplained. `ephemeral` slowed 235x under a + # delay aimed at a cluster it never touches, and the search for a shared gate + # inside the executor came up empty: worker-service was exonerated by the + # traces, agent creation ran in single-digit milliseconds, memory and + # filesystem pools had headroom, the per-account concurrency semaphore is + # bypassed on golem-dev, the tokio global queue was flat at zero with the busy + # ratio *falling*, and neither the oplog cluster nor Redis moved. Everything + # was parked, nothing was competing, and no mechanism in the source accounts + # for it. + # + # So the next step is elimination rather than more forensics: hold the fault + # and the ephemeral stream fixed, and add one stream at a time until ephemeral + # starts to suffer. Whichever addition does it names the interaction, and the + # code reading has somewhere to start. + # + # S15A ephemeral the control: nothing should move + # S15B ephemeral + durable durable is the heaviest user + # S15C ephemeral + durable + promise + # S15 ephemeral + durable + promise + scheduled already run: 235x + # + # ── Why ratePerSec differs between them, which is not a knob being fiddled ── + # + # `workload::start` divides `ratePerSec` evenly across the streams that have + # agents, so a constant here would give ephemeral 100/s alone and 33/s in a + # threesome, and the comparison would be between three different ephemeral + # workloads rather than between three fault environments. Each entry sets the + # rate that holds ephemeral at the ~34/s S15 measured it at. + # + # One confound cannot be removed the same way. `MAX_IN_FLIGHT / active.len()` + # gives ephemeral a budget of 1024, 512 and 341 across the three, against 256 + # in S15, and that ceiling is a constant in the driver rather than a suite + # setting. It bounds throughput, not latency, and latency is what this series + # reads, so it is recorded rather than corrected. + # + # ── Why the scheduled block is present with zero targets ─────────────────── + # + # `storage_fault::run` requires the block. Zero targets means no registrations + # and no fires, so the scheduled stream is not driven and `drives_stream` + # reports it as absent, which is what keeps it out of the expectations below. + # --------------------------------------------------------------------------- + - code: S15A + name: keyvalue-postgres-latency-ephemeral-only + enabled: true + + fault: + kind: network-delay + target: worker-executor + mode: all + # The same file S15 applies, not a copy of it. Four manifests that must + # stay identical by hand is four chances for the fault to diverge from the + # run it is being compared against. + manifest: networkchaos-s15.yaml + durationSecs: 300 + + phases: + # S15's exactly. Shorter windows would be cheaper and the effect is large + # enough to survive them, but the whole value of this series is that its + # ephemeral numbers are comparable with S15's, and a median taken over a + # different window length is a different measurement. + baselineSecs: 300 + faultSecs: 300 + recoverySecs: 420 + + workload: + durableAgents: 0 + ephemeralAgents: 50 + scheduledAgents: 0 + promiseAgents: 0 + quotaAgents: 0 + # One active stream, so ephemeral gets all of it. + ratePerSec: 34 + + scheduled: + targets: 0 + intervalMillis: 2000 + leadSecs: 10 + leaseBudgetSecs: 240 + + storage: + endpoint: golem-postgres-dev-keyvalue.cluster-cgfyoqmjq7tc.us-east-1.rds.amazonaws.com + + expect: + kind: latency-degradation + + # Empty, and the only entry in the suite that is. Nothing this workload + # drives reaches the delayed cluster, so there is no stream whose + # slowdown could evidence the fault. That makes this a control rather + # than an experiment, and it means the run cannot fail on + # `slowdown-not-observed`: it cannot tell a delay that landed and did + # not matter from one that never applied. + # + # The evidence has to come from outside the workload, and it does. The + # scheduler polls its own schema on this cluster whether or not anything + # is scheduled, so `db_success_seconds{svc="scheduler_storage"}` moves + # under the fault regardless of what the workload does. Check it before + # believing a clean result here. + slowed: [] + + # Required by the schema and inert here, because the floor is only ever + # applied to the streams `slowed` names and it names none. Left at the + # series' value rather than at something arbitrary, so a later edit that + # gives this run a slowed stream inherits a floor that means what it + # does everywhere else. + slowdownFloor: 2.0 + + # The whole question. If ephemeral holds its 196ms median with nothing + # else running, then the 235x in S15 came from the company it was + # keeping and not from the fault. + steady: [ephemeral] + steadyCeiling: 1.5 + + servingFloorPercent: 50 + + recoveryBudgetSecs: 60 + + retryPolicy: + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + + - code: S15B + name: keyvalue-postgres-latency-plus-durable + enabled: true + + fault: + kind: network-delay + target: worker-executor + mode: all + manifest: networkchaos-s15.yaml + durationSecs: 300 + + phases: + baselineSecs: 300 + faultSecs: 300 + recoverySecs: 420 + + workload: + # S15's durable population, unchanged, so the load this adds is the load + # S15 had rather than a scaled-down version of it. + durableAgents: 200 + ephemeralAgents: 50 + scheduledAgents: 0 + promiseAgents: 0 + quotaAgents: 0 + # Two active streams, so 34/s each. + ratePerSec: 68 + + scheduled: + targets: 0 + intervalMillis: 2000 + leadSecs: 10 + leaseBudgetSecs: 240 + + storage: + endpoint: golem-postgres-dev-keyvalue.cluster-cgfyoqmjq7tc.us-east-1.rds.amazonaws.com + + expect: + kind: latency-degradation + # Durable waits on two synchronous writes to this cluster per + # invocation, so it should move a long way. S15 measured 644x. + slowed: [durable] + slowdownFloor: 2.0 + # And this is the reading that matters. If ephemeral moves here, durable + # alone is enough to do it and the interaction is between those two. + steady: [ephemeral] + steadyCeiling: 1.5 + servingFloorPercent: 50 + + recoveryBudgetSecs: 60 + + retryPolicy: + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 + + - code: S15C + name: keyvalue-postgres-latency-plus-promise + enabled: true + + fault: + kind: network-delay + target: worker-executor + mode: all + manifest: networkchaos-s15.yaml + durationSecs: 300 + + phases: + baselineSecs: 300 + faultSecs: 300 + recoverySecs: 420 + + workload: + durableAgents: 200 + ephemeralAgents: 50 + scheduledAgents: 0 + promiseAgents: 50 + quotaAgents: 0 + # Three active streams, so 33.3/s each — the rate S15 ran them at. + ratePerSec: 100 + + scheduled: + targets: 0 + intervalMillis: 2000 + leadSecs: 10 + leaseBudgetSecs: 240 + + storage: + endpoint: golem-postgres-dev-keyvalue.cluster-cgfyoqmjq7tc.us-east-1.rds.amazonaws.com + + expect: + kind: latency-degradation + # Promise makes the most calls of any stream: three of its own, plus the + # two its durable agent pays on each of the two invocations an operation + # takes. S15 measured 939x. + slowed: [durable, promise] + slowdownFloor: 2.0 + steady: [ephemeral] + steadyCeiling: 1.5 + servingFloorPercent: 50 + + recoveryBudgetSecs: 60 + + retryPolicy: + transportOnly: true + maxRetries: 1 + delaySecs: 5 + + signalTimeoutSecs: 1800 diff --git a/integration-tests/src/benchmarks/all.rs b/integration-tests/src/benchmarks/all.rs index f3340dff9b..bac31f25be 100644 --- a/integration-tests/src/benchmarks/all.rs +++ b/integration-tests/src/benchmarks/all.rs @@ -604,6 +604,9 @@ async fn run_chaos( ChaosScenarioArg::S18 => chaos::ScenarioCode::S18, ChaosScenarioArg::S17 => chaos::ScenarioCode::S17, ChaosScenarioArg::S15 => chaos::ScenarioCode::S15, + ChaosScenarioArg::S15A => chaos::ScenarioCode::S15A, + ChaosScenarioArg::S15B => chaos::ScenarioCode::S15B, + ChaosScenarioArg::S15C => chaos::ScenarioCode::S15C, }; let config = suite .scenario(code, allow_disabled) @@ -655,6 +658,9 @@ async fn run_chaos( } code @ (chaos::ScenarioCode::S14 | chaos::ScenarioCode::S15 + | chaos::ScenarioCode::S15A + | chaos::ScenarioCode::S15B + | chaos::ScenarioCode::S15C | chaos::ScenarioCode::S16 | chaos::ScenarioCode::S17 | chaos::ScenarioCode::S18 diff --git a/integration-tests/src/chaos/mod.rs b/integration-tests/src/chaos/mod.rs index f097eb2f22..c639914b47 100644 --- a/integration-tests/src/chaos/mod.rs +++ b/integration-tests/src/chaos/mod.rs @@ -109,6 +109,13 @@ pub enum ScenarioCode { /// The key-value PostgreSQL cluster still reachable but slowed. S17's /// mirror on the other half of the split key-value layer. S15, + /// S15 driving `ephemeral` alone. The control: that stream reaches none of + /// the delayed cluster, so nothing should move. + S15A, + /// S15A plus `durable`, the heaviest user of the delayed cluster. + S15B, + /// S15B plus `promise`. The remaining step, `scheduled`, is S15 itself. + S15C, } impl ScenarioCode { @@ -131,13 +138,16 @@ impl ScenarioCode { ScenarioCode::S18 => "S18", ScenarioCode::S17 => "S17", ScenarioCode::S15 => "S15", + ScenarioCode::S15A => "S15A", + ScenarioCode::S15B => "S15B", + ScenarioCode::S15C => "S15C", } } /// Every scenario this driver implements. The suite YAML is checked against /// this list, so a scenario cannot be enabled in YAML without code behind /// it, nor implemented without an operational switch in front of it. - pub const ALL: [ScenarioCode; 17] = [ + pub const ALL: [ScenarioCode; 20] = [ ScenarioCode::S1, ScenarioCode::S3, ScenarioCode::S5, @@ -151,6 +161,9 @@ impl ScenarioCode { ScenarioCode::S13, ScenarioCode::S14, ScenarioCode::S15, + ScenarioCode::S15A, + ScenarioCode::S15B, + ScenarioCode::S15C, ScenarioCode::S16, ScenarioCode::S17, ScenarioCode::S18, @@ -242,6 +255,17 @@ pub struct FaultConfig { /// half the executors" is only meaningful next to how many that was. #[serde(default, skip_serializing_if = "Option::is_none")] pub target_count: Option, + /// The Chaos Mesh manifest to apply, by file name, when it is not the one + /// the workflow would find from the scenario code. + /// + /// Exists for the elimination variants. S15A, S15B and S15C inject exactly + /// the fault S15 injects and differ only in which streams the workload + /// drives, so copying `networkchaos-s15.yaml` three times would make four + /// files that must be kept identical by hand — and a run whose fault + /// silently diverged from the run it is being compared against is worse + /// than no run. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub manifest: Option, pub duration_secs: u64, } @@ -1324,7 +1348,32 @@ impl ScenarioConfig { ); } self.check_serving_floor(*serving_floor_percent)?; - self.check_evidence_streams("slowed", slowed, "slowdown")?; + // A run may name no slowed streams at all, but only if it names + // steady ones — and then it is a control rather than an + // experiment. S15A drives `ephemeral` alone, which reaches none + // of the delayed store, so *nothing* in its workload should + // move and there is no slowdown available as evidence that the + // fault landed. + // + // The cost is real and worth stating: such a run cannot fail on + // `slowdown-not-observed`, so it cannot tell a delay that + // landed and did not matter from one that never applied. That + // evidence has to come from outside the workload, and for S15A + // it does — the scheduler polls its own schema on the delayed + // cluster whether or not anything is scheduled, so + // `db_success_seconds{svc="scheduler_storage"}` moves under the + // fault regardless. The runbook says to check it. + if slowed.is_empty() && steady.is_empty() { + anyhow::bail!( + "chaos scenario {}: expect.slowed and expect.steady are both empty, so \ + nothing in the run would show whether the fault landed or whether it \ + mattered", + self.code + ); + } + if !slowed.is_empty() { + self.check_evidence_streams("slowed", slowed, "slowdown")?; + } // The steady list is optional, so it is checked only when // present — but a present one carries the same two ways of // being useless as the slowed list, plus one of its own. @@ -1536,6 +1585,7 @@ mod tests { target: "shard-manager".to_string(), mode: "one".to_string(), target_count: None, + manifest: None, duration_secs: 60, }, phases: PhaseConfig { @@ -1581,6 +1631,7 @@ mod tests { target: "worker-executor".to_string(), mode: "one".to_string(), target_count: None, + manifest: None, duration_secs: 60, }, phases: PhaseConfig { @@ -1655,6 +1706,7 @@ mod tests { target: "worker-executor".to_string(), mode: "all".to_string(), target_count: None, + manifest: None, duration_secs: 180, }, phases: PhaseConfig { @@ -1724,6 +1776,45 @@ mod tests { ); } + /// A control run: no stream is expected to slow, and the one named steady + /// is what the run is actually asking about. Allowed, because a series that + /// isolates an interaction needs a run with nothing in it to interact. + #[test] + fn a_latency_run_that_names_only_steady_streams_is_allowed() { + let mut config = storage_config("db.example", 50.0, 0, true); + config.workload.as_mut().unwrap().ephemeral_agents = 10; + config.storage.as_mut().unwrap().expect = OutageExpectation::LatencyDegradation { + slowed: Vec::new(), + slowdown_floor: 2.0, + steady: vec![Stream::Ephemeral], + steady_ceiling: 1.5, + serving_floor_percent: 50.0, + }; + assert!( + config.require_storage().is_ok(), + "a run may assert that nothing moved, as long as it names what should not have" + ); + } + + /// Both lists empty is the one shape that says nothing at all: no stream + /// would show the fault landing, and none would show it mattering either. + #[test] + fn a_latency_run_that_names_no_streams_at_all_is_refused() { + let mut config = storage_config("db.example", 50.0, 0, true); + config.storage.as_mut().unwrap().expect = OutageExpectation::LatencyDegradation { + slowed: Vec::new(), + slowdown_floor: 2.0, + steady: Vec::new(), + steady_ceiling: 1.5, + serving_floor_percent: 50.0, + }; + let error = config.require_storage().unwrap_err().to_string(); + assert!( + error.contains("expect.slowed and expect.steady are both empty"), + "the message has to say which pair is missing, got: {error}" + ); + } + /// Without an endpoint the archived result could not say which storage the /// run took away. #[test] @@ -1831,6 +1922,8 @@ mod tests { assert_eq!(ScenarioCode::parse("s18"), Some(ScenarioCode::S18)); assert_eq!(ScenarioCode::parse("s17"), Some(ScenarioCode::S17)); assert_eq!(ScenarioCode::parse("s15"), Some(ScenarioCode::S15)); + assert_eq!(ScenarioCode::parse("s15a"), Some(ScenarioCode::S15A)); + assert_eq!(ScenarioCode::parse("s15c"), Some(ScenarioCode::S15C)); assert_eq!(ScenarioCode::parse("S99"), None); } @@ -1878,6 +1971,9 @@ mod tests { } ScenarioCode::S14 | ScenarioCode::S15 + | ScenarioCode::S15A + | ScenarioCode::S15B + | ScenarioCode::S15C | ScenarioCode::S16 | ScenarioCode::S17 | ScenarioCode::S18 diff --git a/integration-tests/src/chaos/outage.rs b/integration-tests/src/chaos/outage.rs index 61304edf70..ec7b0778af 100644 --- a/integration-tests/src/chaos/outage.rs +++ b/integration-tests/src/chaos/outage.rs @@ -899,6 +899,14 @@ impl StorageFaultReport { // quiet figure is absent by design rather than missing, and the // slowdown takes its place as what says the fault landed. let quiet = match (self.quietest_stream_percent, self.least_slowdown_factor) { + // A control run: the expectation is a delay, but it names no + // stream that should feel it, so there is no slowdown to report + // and its absence is the result rather than missing data. + (None, None) if self.expect.slowdown_floor().is_some() => { + "no stream in this run was expected to slow down, so whether the delay \ + landed has to be read from the storage metrics rather than from here" + .to_string() + } (_, Some(factor)) => format!( "the streams expected to slow down ran at least {factor}x their own baseline \ median (floor {}x)", diff --git a/integration-tests/src/chaos/result.rs b/integration-tests/src/chaos/result.rs index c8b149582b..5cc3c99518 100644 --- a/integration-tests/src/chaos/result.rs +++ b/integration-tests/src/chaos/result.rs @@ -245,6 +245,7 @@ mod tests { target: "shard-manager".to_string(), mode: "one".to_string(), target_count: None, + manifest: None, duration_secs: 60, }, workload: Some(WorkloadConfig { @@ -996,6 +997,7 @@ mod sample_artifact { target: "shard-manager".to_string(), mode: "one".to_string(), target_count: None, + manifest: None, duration_secs: 60, }, workload: Some(WorkloadConfig { @@ -1208,6 +1210,7 @@ mod sample_artifact { target: "worker-executor".to_string(), mode: "one".to_string(), target_count: None, + manifest: None, duration_secs: 60, }, workload: None, From e5f7f3c2a36f7254a841548a06b5bccaa0fe6390 Mon Sep 17 00:00:00 2001 From: Kaur Matas <33095685+kmatasfp@users.noreply.github.com> Date: Fri, 28 Aug 2026 16:54:22 -0700 Subject: [PATCH 40/40] Skip the scheduled fire gate when a run drives no targets --- .../src/chaos/scenarios/storage_fault.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/integration-tests/src/chaos/scenarios/storage_fault.rs b/integration-tests/src/chaos/scenarios/storage_fault.rs index 85d240c020..8db2e7bde9 100644 --- a/integration-tests/src/chaos/scenarios/storage_fault.rs +++ b/integration-tests/src/chaos/scenarios/storage_fault.rs @@ -426,8 +426,18 @@ pub async fn run( // this scenario is about, and a platform that accepted every registration // and ran none of them would otherwise reach read-back and report a // flawless account of a mechanism that never worked. - let sampled = sample_fire_count(code, &ctx, &targets).await; - if sampled == 0 { + // + // Only when the run drives the stream at all. A scenario that registers + // nothing has no fire to observe, so holding it to this gate aborts it + // during the baseline and before the fault is ever injected. That is what + // happened to S15A's first run, and it cost a cluster run to learn. + let drives_scheduled = config.drives_stream(Stream::Scheduled); + let sampled = if drives_scheduled { + sample_fire_count(code, &ctx, &targets).await + } else { + 0 + }; + if drives_scheduled && sampled == 0 { warn!( "{code}: {baseline_operations} operations confirmed and no scheduled action has fired" );