Chaos testing phase 3

golem-test-framework/src/benchmark/config.rs

@@ -242,6 +242,34 @@ pub enum ChaosScenarioArg {
     S12,
     /// Rolling executor restarts under load.
     S13,
+    /// Executor pod kill while scheduled actions are between claim and fire.
+    S10,
+    /// Executor pod kill while agents are suspended on promises being completed.
+    S11,
+    /// Executor cut off from worker-service while it keeps its shards.
+    S3,
+    /// Executor pod kill while agents are having their state reverted.
+    S7,
+    /// Executor pod kill while agents are being deleted.
+    S6,
+    /// Executor pod kill while a component rollback is in flight.
+    S9,
+    /// Executors cut off from the key-value PostgreSQL cluster for about as
+    /// long as an AWS storage failover takes.
+    S16,
+    /// The same cut, held for longer than the key-value retry budget.
+    S22,
+    /// Executors cut off from the indexed-oplog PostgreSQL cluster for the
+    /// length of a writer failover.
+    S14,
+    /// Executors cut off from the Redis cache in front of the key-value layer,
+    /// for longer than a caller is willing to wait.
+    S18,
+    /// The same Redis cache slowed rather than removed.
+    S17,
+    /// The key-value PostgreSQL cluster slowed rather than removed. S17's
+    /// mirror on the other half of the split key-value layer.
+    S15,
 }
 
 /// Density subcommand action.

golem-test-framework/src/benchmark/results.rs

@@ -525,6 +525,17 @@ pub struct RunMetadata {
     /// Container image tag of the deployed `worker-service`.
     #[serde(skip_serializing_if = "Option::is_none", default)]
     pub worker_service_image_tag: Option<String>,
+    /// Image digest of the deployed `worker-executor`, when the manifest pins
+    /// one. A tag can be moved; the digest is what actually identifies the
+    /// build a run tested, which is the whole point of recording it.
+    #[serde(skip_serializing_if = "Option::is_none", default)]
+    pub worker_executor_image_digest: Option<String>,
+    /// Image digest of the deployed `registry-service`.
+    #[serde(skip_serializing_if = "Option::is_none", default)]
+    pub registry_service_image_digest: Option<String>,
+    /// Image digest of the deployed `worker-service`.
+    #[serde(skip_serializing_if = "Option::is_none", default)]
+    pub worker_service_image_digest: Option<String>,
     /// Aurora ACU capacity for the main (`golem_dev`) cluster at run start.
     #[serde(skip_serializing_if = "Option::is_none", default)]
     pub aurora_acu_main: Option<f64>,
@@ -575,6 +586,9 @@ impl RunMetadata {
             worker_executor_image_tag: env_str("GOLEM_BENCH_WORKER_EXECUTOR_IMAGE_TAG"),
             registry_service_image_tag: env_str("GOLEM_BENCH_REGISTRY_SERVICE_IMAGE_TAG"),
             worker_service_image_tag: env_str("GOLEM_BENCH_WORKER_SERVICE_IMAGE_TAG"),
+            worker_executor_image_digest: env_str("GOLEM_BENCH_WORKER_EXECUTOR_IMAGE_DIGEST"),
+            registry_service_image_digest: env_str("GOLEM_BENCH_REGISTRY_SERVICE_IMAGE_DIGEST"),
+            worker_service_image_digest: env_str("GOLEM_BENCH_WORKER_SERVICE_IMAGE_DIGEST"),
             aurora_acu_main: env_f64("GOLEM_BENCH_AURORA_ACU_MAIN"),
             aurora_acu_indexed: env_f64("GOLEM_BENCH_AURORA_ACU_INDEXED"),
             aurora_acu_keyvalue: env_f64("GOLEM_BENCH_AURORA_ACU_KEYVALUE"),

integration-tests/src/benchmarks/all.rs

@@ -592,6 +592,18 @@ async fn run_chaos(
                 ChaosScenarioArg::S5 => chaos::ScenarioCode::S5,
                 ChaosScenarioArg::S12 => chaos::ScenarioCode::S12,
                 ChaosScenarioArg::S13 => chaos::ScenarioCode::S13,
+                ChaosScenarioArg::S10 => chaos::ScenarioCode::S10,
+                ChaosScenarioArg::S11 => chaos::ScenarioCode::S11,
+                ChaosScenarioArg::S3 => chaos::ScenarioCode::S3,
+                ChaosScenarioArg::S7 => chaos::ScenarioCode::S7,
+                ChaosScenarioArg::S6 => chaos::ScenarioCode::S6,
+                ChaosScenarioArg::S9 => chaos::ScenarioCode::S9,
+                ChaosScenarioArg::S16 => chaos::ScenarioCode::S16,
+                ChaosScenarioArg::S22 => chaos::ScenarioCode::S22,
+                ChaosScenarioArg::S14 => chaos::ScenarioCode::S14,
+                ChaosScenarioArg::S18 => chaos::ScenarioCode::S18,
+                ChaosScenarioArg::S17 => chaos::ScenarioCode::S17,
+                ChaosScenarioArg::S15 => chaos::ScenarioCode::S15,
             };
             let config = suite
                 .scenario(code, allow_disabled)
@@ -623,6 +635,35 @@ async fn run_chaos(
                 chaos::ScenarioCode::S13 => {
                     chaos::scenarios::s13::run(&config, &manifest, &deps, &signals, &outputs).await
                 }
+                chaos::ScenarioCode::S10 => {
+                    chaos::scenarios::s10::run(&config, &manifest, &deps, &signals, &outputs).await
+                }
+                chaos::ScenarioCode::S11 => {
+                    chaos::scenarios::s11::run(&config, &manifest, &deps, &signals, &outputs).await
+                }
+                chaos::ScenarioCode::S3 => {
+                    chaos::scenarios::s3::run(&config, &manifest, &deps, &signals, &outputs).await
+                }
+                chaos::ScenarioCode::S7 => {
+                    chaos::scenarios::s7::run(&config, &manifest, &deps, &signals, &outputs).await
+                }
+                chaos::ScenarioCode::S6 => {
+                    chaos::scenarios::s6::run(&config, &manifest, &deps, &signals, &outputs).await
+                }
+                chaos::ScenarioCode::S9 => {
+                    chaos::scenarios::s9::run(&config, &manifest, &deps, &signals, &outputs).await
+                }
+                code @ (chaos::ScenarioCode::S14
+                | chaos::ScenarioCode::S15
+                | chaos::ScenarioCode::S16
+                | chaos::ScenarioCode::S17
+                | chaos::ScenarioCode::S18
+                | chaos::ScenarioCode::S22) => {
+                    chaos::scenarios::storage_fault::run(
+                        code, &config, &manifest, &deps, &signals, &outputs,
+                    )
+                    .await
+                }
             };
 
             deps.kill_all().await;
@@ -855,7 +896,7 @@ async fn run_density(
 
             // Emit the cell result as a single-result suite so the JSON shape
             // matches cloud-perf (BenchmarkSuiteResultCollection), and the
-            // buildspec can upload it directly to S3.
+            // buildspec can upload it directly to golem-bench-results.
             let mut suite_result = BenchmarkSuiteResult::new(&format!("density-{section}"));
             suite_result.add(result);
             if let Some(run_id) = cloud_bench_run_id() {

integration-tests/src/benchmarks/density/agent.rs

@@ -185,7 +185,8 @@ pub struct CellConfig {
 }
 
 impl CellConfig {
-    /// The cell's name, used in S3 paths and result identifiers. Encodes the
+    /// The cell's name, used in `golem-bench-results` keys and result identifiers.
+    /// Encodes the
     /// full axis set in human-readable terms (no cryptic single letters).
     pub fn cell_name(&self) -> String {
         let mut parts = vec![

integration-tests/src/benchmarks/density/prep.rs

@@ -20,7 +20,8 @@
 //!
 //! The buildspec runs prep exactly once at suite start (against a freshly-wiped
 //! cluster) and writes the resulting [`PrepManifest`] to a file that is
-//! uploaded to S3 and passed to every per-cell invocation via
+//! uploaded to the `golem-bench-results` bucket and passed to every per-cell
+//! invocation via
 //! `--prep-manifest`. The manifest carries the account token and all component
 //! IDs so per-cell invocations need no by-name lookup and no re-tokenization —
 //! this is also the resume mechanism: a resumed run reloads the same manifest.